From 60171093c55c4dd3b7985ac28b87ff8099a2f5bb Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Fri, 17 Nov 2023 14:43:57 +0100 Subject: [PATCH 1/5] Update version to 4.0.0-beta.143 --- app/Jobs/ContainerStatusJob.php | 2 +- config/sentry.php | 2 +- config/version.php | 2 +- versions.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/Jobs/ContainerStatusJob.php b/app/Jobs/ContainerStatusJob.php index 2762b8b54..16ab460fe 100644 --- a/app/Jobs/ContainerStatusJob.php +++ b/app/Jobs/ContainerStatusJob.php @@ -266,7 +266,7 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted $this->server->team->notify(new ContainerStopped($containerName, $this->server, $url)); } } catch (\Throwable $e) { - send_internal_notification('ContainerStatusJob failed with: ' . $e->getMessage()); + send_internal_notification("ContainerStatusJob failed on ({$this->server->id}) with: " . $e->getMessage()); ray($e->getMessage()); handleError($e); } diff --git a/config/sentry.php b/config/sentry.php index 5655b866b..d1111bbd4 100644 --- a/config/sentry.php +++ b/config/sentry.php @@ -7,7 +7,7 @@ return [ // The release version of your application // Example with dynamic git hash: trim(exec('git --git-dir ' . base_path('.git') . ' log --pretty="%h" -n1 HEAD')) - 'release' => '4.0.0-beta.142', + 'release' => '4.0.0-beta.143', // When left empty or `null` the Laravel environment will be used 'environment' => config('app.env'), diff --git a/config/version.php b/config/version.php index 8b7d1633c..318484b79 100644 --- a/config/version.php +++ b/config/version.php @@ -1,3 +1,3 @@ Date: Fri, 17 Nov 2023 14:46:04 +0100 Subject: [PATCH 2/5] Fix server readiness check in ContainerStatusJob and ServerStatusJob --- app/Jobs/ContainerStatusJob.php | 4 +++- app/Jobs/ServerStatusJob.php | 5 +++-- app/Models/Server.php | 10 +++++++--- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/app/Jobs/ContainerStatusJob.php b/app/Jobs/ContainerStatusJob.php index 16ab460fe..ba3c40a03 100644 --- a/app/Jobs/ContainerStatusJob.php +++ b/app/Jobs/ContainerStatusJob.php @@ -39,7 +39,9 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted { ray("checking container statuses for {$this->server->id}"); try { - $this->server->checkServerRediness(); + if (!$this->server->isServerReady()) { + throw new \Exception('Server is not ready.'); + }; $containers = instant_remote_process(["docker container ls -q"], $this->server); if (!$containers) { return; diff --git a/app/Jobs/ServerStatusJob.php b/app/Jobs/ServerStatusJob.php index 7d591bc83..cfe61598f 100644 --- a/app/Jobs/ServerStatusJob.php +++ b/app/Jobs/ServerStatusJob.php @@ -34,8 +34,9 @@ class ServerStatusJob implements ShouldQueue, ShouldBeEncrypted { ray("checking server status for {$this->server->id}"); try { - $this->server->checkServerRediness(); - $this->cleanup(notify: false); + if ($this->server->isServerReady()) { + $this->cleanup(notify: false); + } } catch (\Throwable $e) { send_internal_notification('ServerStatusJob failed with: ' . $e->getMessage()); ray($e->getMessage()); diff --git a/app/Models/Server.php b/app/Models/Server.php index 77b80e680..e52b2280c 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -128,13 +128,15 @@ class Server extends BaseModel } return false; } - public function checkServerRediness() + public function isServerReady() { $serverUptimeCheckNumber = $this->unreachable_count; $serverUptimeCheckNumberMax = 3; $currentTime = now()->timestamp; $runtime5Minutes = 1 * 60; + + $isReady = false; // Run for 1 minutes max and check every 5 seconds for 3 times while ($currentTime + $runtime5Minutes > now()->timestamp) { if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) { @@ -165,7 +167,8 @@ class Server extends BaseModel $db->update(['status' => 'exited']); } } - throw new \Exception('Server is not reachable.'); + $isReady = false; + break; } $result = $this->validateConnection(); ray('validateConnection: ' . $result); @@ -177,9 +180,10 @@ class Server extends BaseModel Sleep::for(5)->seconds(); return; } + $isReady = true; break; } - return true; + return $isReady; } public function getDiskUsage() { From 4c2b3df861edb7fdea6164e7997cd752dd88c784 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Fri, 17 Nov 2023 14:56:39 +0100 Subject: [PATCH 3/5] Update server runtime and comments --- app/Models/Server.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/Models/Server.php b/app/Models/Server.php index e52b2280c..f00f772a8 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -134,11 +134,11 @@ class Server extends BaseModel $serverUptimeCheckNumberMax = 3; $currentTime = now()->timestamp; - $runtime5Minutes = 1 * 60; + $runtime = 30; $isReady = false; - // Run for 1 minutes max and check every 5 seconds for 3 times - while ($currentTime + $runtime5Minutes > now()->timestamp) { + // Run for 30 seconds max and check every 5 seconds for 3 times + while ($currentTime + $runtime > now()->timestamp) { if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) { if ($this->unreachable_notification_sent === false) { ray('Server unreachable, sending notification...'); From 9ce3b43e0906704ef319449613ea971253191974 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Fri, 17 Nov 2023 15:11:29 +0100 Subject: [PATCH 4/5] Add Team model and merge servers with own servers --- app/Console/Kernel.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 7f8c86b94..fa195aafd 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -11,6 +11,7 @@ use App\Jobs\ServerStatusJob; use App\Models\InstanceSettings; use App\Models\ScheduledDatabaseBackup; use App\Models\Server; +use App\Models\Team; use Illuminate\Console\Scheduling\Schedule; use Illuminate\Foundation\Console\Kernel as ConsoleKernel; @@ -52,6 +53,8 @@ class Kernel extends ConsoleKernel { if (isCloud()) { $servers = Server::all()->whereNotNull('team.subscription')->where('team.subscription.stripe_trial_already_ended', false)->where('ip', '!=', '1.2.3.4'); + $own = Team::find(0)->servers; + $servers = $servers->merge($own); } else { $servers = Server::all()->where('ip', '!=', '1.2.3.4'); } From b0c96e64c9d2bef59ddb6a490f14e62ae30aed96 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Fri, 17 Nov 2023 15:18:08 +0100 Subject: [PATCH 5/5] Fix server unreachable notification count --- app/Jobs/ContainerStatusJob.php | 2 +- app/Notifications/Server/Unreachable.php | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/Jobs/ContainerStatusJob.php b/app/Jobs/ContainerStatusJob.php index ba3c40a03..1c66ef53f 100644 --- a/app/Jobs/ContainerStatusJob.php +++ b/app/Jobs/ContainerStatusJob.php @@ -40,7 +40,7 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted ray("checking container statuses for {$this->server->id}"); try { if (!$this->server->isServerReady()) { - throw new \Exception('Server is not ready.'); + return; }; $containers = instant_remote_process(["docker container ls -q"], $this->server); if (!$containers) { diff --git a/app/Notifications/Server/Unreachable.php b/app/Notifications/Server/Unreachable.php index ae100b804..07f801fe8 100644 --- a/app/Notifications/Server/Unreachable.php +++ b/app/Notifications/Server/Unreachable.php @@ -43,7 +43,7 @@ class Unreachable extends Notification implements ShouldQueue public function toMail(): MailMessage { $mail = new MailMessage(); - $mail->subject("Coolify: Server ({$this->server->name}) is unreachable after trying to connect to it 5 times"); + $mail->subject("Coolify: Server ({$this->server->name}) is unreachable after trying to connect to it 3 times"); $mail->view('emails.server-lost-connection', [ 'name' => $this->server->name, ]); @@ -52,13 +52,13 @@ class Unreachable extends Notification implements ShouldQueue public function toDiscord(): string { - $message = "Coolify: Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: We automatically try to revive your server. If your server is back online, we will automatically turn on all automations & integrations."; + $message = "Coolify: Server '{$this->server->name}' is unreachable after trying to connect to it 3 times. All automations & integrations are turned off! Please check your server! IMPORTANT: We automatically try to revive your server. If your server is back online, we will automatically turn on all automations & integrations."; return $message; } public function toTelegram(): array { return [ - "message" => "Coolify: Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: We automatically try to revive your server. If your server is back online, we will automatically turn on all automations & integrations." + "message" => "Coolify: Server '{$this->server->name}' is unreachable after trying to connect to it 3 times. All automations & integrations are turned off! Please check your server! IMPORTANT: We automatically try to revive your server. If your server is back online, we will automatically turn on all automations & integrations." ]; } }