diff --git a/app/Jobs/ContainerStatusJob.php b/app/Jobs/ContainerStatusJob.php index c0d0e7ad9..bacfcb739 100644 --- a/app/Jobs/ContainerStatusJob.php +++ b/app/Jobs/ContainerStatusJob.php @@ -17,33 +17,37 @@ use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\Middleware\WithoutOverlapping; use Illuminate\Queue\SerializesModels; use Illuminate\Support\Arr; -use Illuminate\Support\Sleep; class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; + public $tries = 5; + public function backoff(): int + { + return isDev() ? 1 : 5; + } public function middleware(): array { - return [(new WithoutOverlapping($this->server->id))->dontRelease()]; + return [(new WithoutOverlapping($this->server->uuid))]; } public function uniqueId(): int { - return $this->server->id; + return $this->server->uuid; } public function __construct(public Server $server) { - $this->handle(); + // $this->handle(); } public function handle() { + if (!$this->server->isServerReady($this->tries)) { + throw new \RuntimeException('Server is not ready.'); + }; try { - if (!$this->server->isServerReady()) { - return; - }; if ($this->server->isSwarm()) { $containers = instant_remote_process(["docker service inspect $(docker service ls -q) --format '{{json .}}'"], $this->server, false); $containerReplicates = instant_remote_process(["docker service ls --format '{{json .}}'"], $this->server, false); diff --git a/app/Models/Server.php b/app/Models/Server.php index 04638ba53..7bd791d11 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -12,7 +12,6 @@ use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Support\Carbon; use Illuminate\Support\Collection; use Illuminate\Support\Facades\Process; -use Illuminate\Support\Sleep; use Spatie\SchemalessAttributes\Casts\SchemalessAttributes; use Spatie\SchemalessAttributes\SchemalessAttributesTrait; use Illuminate\Support\Str; @@ -150,30 +149,35 @@ class Server extends BaseModel } return false; } - public function isServerReady() + public function isServerReady($tries) { - $serverUptimeCheckNumber = $this->unreachable_count; - $serverUptimeCheckNumberMax = 8; + $serverUptimeCheckNumber = $this->unreachable_count + 1; + $serverUptimeCheckNumberMax = $tries ?? 3; - $currentTime = now()->timestamp; - $runtime = 50; + ray('server: ' . $this->name); + ray('serverUptimeCheckNumber: ' . $serverUptimeCheckNumber); + ray('serverUptimeCheckNumberMax: ' . $serverUptimeCheckNumberMax); - $isReady = false; - // Run for 50 seconds max and check every 5 seconds for 8 times - while ($currentTime + $runtime > now()->timestamp) { - ray('serverUptimeCheckNumber: ' . $serverUptimeCheckNumber); + $result = $this->validateConnection(); + if ($result) { + if ($this->unreachable_notification_sent === true) { + $this->update(['unreachable_notification_sent' => false]); + } + return true; + } else { if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) { + // Reached max number of retries if ($this->unreachable_notification_sent === false) { ray('Server unreachable, sending notification...'); $this->team?->notify(new Unreachable($this)); $this->update(['unreachable_notification_sent' => true]); } - $this->settings()->update([ - 'is_reachable' => false, - ]); - $this->update([ - 'unreachable_count' => 0, - ]); + if ($this->settings->is_reachable === true) { + $this->settings()->update([ + 'is_reachable' => false, + ]); + } + foreach ($this->applications() as $application) { $application->update(['status' => 'exited']); } @@ -190,23 +194,13 @@ class Server extends BaseModel $db->update(['status' => 'exited']); } } - $isReady = false; - break; - } - $result = $this->validateConnection(); - // ray('validateConnection: ' . $result); - if (!$result) { - $serverUptimeCheckNumber++; + } else { $this->update([ - 'unreachable_count' => $serverUptimeCheckNumber, + 'unreachable_count' => $this->unreachable_count + 1, ]); - Sleep::for(5)->seconds(); - continue; } - $isReady = true; - break; + return false; } - return $isReady; } public function getDiskUsage() { diff --git a/app/Notifications/Server/Revived.php b/app/Notifications/Server/Revived.php index 400ef8377..3e5a99425 100644 --- a/app/Notifications/Server/Revived.php +++ b/app/Notifications/Server/Revived.php @@ -54,7 +54,7 @@ class Revived extends Notification implements ShouldQueue public function toDiscord(): string { - $message = "Coolify: Server '{$this->server->name}' revived. All automations & integrations are turned on again!"; + $message = "Coolify: Server '{$this->server->name}' revived. All automations & integrations are turned on again!"; return $message; } public function toTelegram(): array