From f4cb7cea211e1c79895cccc75aced72015ec0a1e Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Fri, 3 May 2024 10:22:28 +0200 Subject: [PATCH] fix: better server vlaidation fix: remove unnecessary coolify.yaml from cloud hosted servers --- app/Console/Kernel.php | 6 +-- app/Jobs/ServerStatusJob.php | 13 ++++- app/Models/Server.php | 99 ++++++++++++++++-------------------- 3 files changed, 60 insertions(+), 58 deletions(-) diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 734063b2e..1f7006aaa 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -72,13 +72,13 @@ private function check_resources($schedule) $containerServers = $servers->where('settings.is_swarm_worker', false)->where('settings.is_build_server', false); } foreach ($containerServers as $server) { - $schedule->job(new ContainerStatusJob($server))->everyTwoMinutes()->onOneServer(); + $schedule->job(new ContainerStatusJob($server))->everyMinute()->onOneServer(); if ($server->isLogDrainEnabled()) { - $schedule->job(new CheckLogDrainContainerJob($server))->everyTwoMinutes()->onOneServer(); + $schedule->job(new CheckLogDrainContainerJob($server))->everyMinute()->onOneServer(); } } foreach ($servers as $server) { - $schedule->job(new ServerStatusJob($server))->everyTwoMinutes()->onOneServer(); + $schedule->job(new ServerStatusJob($server))->everyMinute()->onOneServer(); } } private function instance_auto_update($schedule) diff --git a/app/Jobs/ServerStatusJob.php b/app/Jobs/ServerStatusJob.php index 31683d097..1383e1f5a 100644 --- a/app/Jobs/ServerStatusJob.php +++ b/app/Jobs/ServerStatusJob.php @@ -38,11 +38,12 @@ public function uniqueId(): int public function handle() { if (!$this->server->isServerReady($this->tries)) { - throw new \RuntimeException('Server is not ready.'); + return "Server is not ready yet."; }; try { if ($this->server->isFunctional()) { $this->cleanup(notify: false); + $this->removeCoolifyYaml(); } } catch (\Throwable $e) { send_internal_notification('ServerStatusJob failed with: ' . $e->getMessage()); @@ -50,6 +51,16 @@ public function handle() return handleError($e); } } + private function removeCoolifyYaml() + { + // This will remote the coolify.yaml file from the server as it is not needed on cloud servers + if (isCloud()) { + $file = $this->server->proxyPath() . "/dynamic/coolify.yaml"; + return instant_remote_process([ + "rm -f $file", + ], $this->server, false); + } + } public function cleanup(bool $notify = false): void { $this->disk_usage = $this->server->getDiskUsage(); diff --git a/app/Models/Server.php b/app/Models/Server.php index bda044320..8f66f44e7 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -10,6 +10,7 @@ use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Storage; +use Illuminate\Support\Sleep; use Spatie\SchemalessAttributes\Casts\SchemalessAttributes; use Spatie\SchemalessAttributes\SchemalessAttributesTrait; use Illuminate\Support\Str; @@ -239,7 +240,7 @@ public function setupDynamicProxyConfiguration() $dynamic_config_path = $this->proxyPath() . "/dynamic"; if ($this->proxyType() === 'TRAEFIK_V2') { $file = "$dynamic_config_path/coolify.yaml"; - if (empty($settings->fqdn)) { + if (empty($settings->fqdn) || isCloud()) { instant_remote_process([ "rm -f $file", ], $this); @@ -467,63 +468,53 @@ public function isServerReady(int $tries = 3) if ($this->skipServer()) { return false; } - $serverUptimeCheckNumber = $this->unreachable_count; - if ($this->unreachable_count < $tries) { - $serverUptimeCheckNumber = $this->unreachable_count + 1; - } - if ($this->unreachable_count > $tries) { - $serverUptimeCheckNumber = $tries; - } - - $serverUptimeCheckNumberMax = $tries; - - // ray('server: ' . $this->name); - // ray('serverUptimeCheckNumber: ' . $serverUptimeCheckNumber); - // ray('serverUptimeCheckNumberMax: ' . $serverUptimeCheckNumberMax); - - ['uptime' => $uptime] = $this->validateConnection(); - if ($uptime) { - if ($this->unreachable_notification_sent === true) { - $this->update(['unreachable_notification_sent' => false]); - } - return true; - } else { - if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) { - // Reached max number of retries - if ($this->unreachable_notification_sent === false) { - ray('Server unreachable, sending notification...'); - $this->team?->notify(new Unreachable($this)); - $this->update(['unreachable_notification_sent' => true]); + $checkIteration = 1; + $isServerReady = false; + while ($checkIteration < $tries) { + ['uptime' => $uptime] = $this->validateConnection(); + if ($uptime) { + if ($this->unreachable_notification_sent === true) { + $this->update(['unreachable_notification_sent' => false]); } - if ($this->settings->is_reachable === true) { - $this->settings()->update([ - 'is_reachable' => false, - ]); - } - - foreach ($this->applications() as $application) { - $application->update(['status' => 'exited']); - } - foreach ($this->databases() as $database) { - $database->update(['status' => 'exited']); - } - foreach ($this->services()->get() as $service) { - $apps = $service->applications()->get(); - $dbs = $service->databases()->get(); - foreach ($apps as $app) { - $app->update(['status' => 'exited']); - } - foreach ($dbs as $db) { - $db->update(['status' => 'exited']); - } - } - } else { - $this->update([ - 'unreachable_count' => $this->unreachable_count + 1, + $this->settings()->update([ + 'is_reachable' => true, ]); + $isServerReady = true; + break; + } else { + ray('Server is not ready yet.'); + $checkIteration++; + Sleep::for(10)->seconds(); } - return false; } + if ($isServerReady) { + return $isServerReady; + } + if ($this->unreachable_notification_sent === false) { + ray('Server unreachable, sending notification...'); + $this->team?->notify(new Unreachable($this)); + $this->update(['unreachable_notification_sent' => true]); + } + $this->settings()->update([ + 'is_reachable' => false, + ]); + foreach ($this->applications() as $application) { + $application->update(['status' => 'exited']); + } + foreach ($this->databases() as $database) { + $database->update(['status' => 'exited']); + } + foreach ($this->services()->get() as $service) { + $apps = $service->applications()->get(); + $dbs = $service->databases()->get(); + foreach ($apps as $app) { + $app->update(['status' => 'exited']); + } + foreach ($dbs as $db) { + $db->update(['status' => 'exited']); + } + } + return false; } public function getDiskUsage() {