From 467471f54ad2216e324e2943861a307bdbf8c860 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Fri, 17 Nov 2023 14:46:04 +0100 Subject: [PATCH] Fix server readiness check in ContainerStatusJob and ServerStatusJob --- app/Jobs/ContainerStatusJob.php | 4 +++- app/Jobs/ServerStatusJob.php | 5 +++-- app/Models/Server.php | 10 +++++++--- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/app/Jobs/ContainerStatusJob.php b/app/Jobs/ContainerStatusJob.php index 16ab460fe..ba3c40a03 100644 --- a/app/Jobs/ContainerStatusJob.php +++ b/app/Jobs/ContainerStatusJob.php @@ -39,7 +39,9 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted { ray("checking container statuses for {$this->server->id}"); try { - $this->server->checkServerRediness(); + if (!$this->server->isServerReady()) { + throw new \Exception('Server is not ready.'); + }; $containers = instant_remote_process(["docker container ls -q"], $this->server); if (!$containers) { return; diff --git a/app/Jobs/ServerStatusJob.php b/app/Jobs/ServerStatusJob.php index 7d591bc83..cfe61598f 100644 --- a/app/Jobs/ServerStatusJob.php +++ b/app/Jobs/ServerStatusJob.php @@ -34,8 +34,9 @@ class ServerStatusJob implements ShouldQueue, ShouldBeEncrypted { ray("checking server status for {$this->server->id}"); try { - $this->server->checkServerRediness(); - $this->cleanup(notify: false); + if ($this->server->isServerReady()) { + $this->cleanup(notify: false); + } } catch (\Throwable $e) { send_internal_notification('ServerStatusJob failed with: ' . $e->getMessage()); ray($e->getMessage()); diff --git a/app/Models/Server.php b/app/Models/Server.php index 77b80e680..e52b2280c 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -128,13 +128,15 @@ class Server extends BaseModel } return false; } - public function checkServerRediness() + public function isServerReady() { $serverUptimeCheckNumber = $this->unreachable_count; $serverUptimeCheckNumberMax = 3; $currentTime = now()->timestamp; $runtime5Minutes = 1 * 60; + + $isReady = false; // Run for 1 minutes max and check every 5 seconds for 3 times while ($currentTime + $runtime5Minutes > now()->timestamp) { if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) { @@ -165,7 +167,8 @@ class Server extends BaseModel $db->update(['status' => 'exited']); } } - throw new \Exception('Server is not reachable.'); + $isReady = false; + break; } $result = $this->validateConnection(); ray('validateConnection: ' . $result); @@ -177,9 +180,10 @@ class Server extends BaseModel Sleep::for(5)->seconds(); return; } + $isReady = true; break; } - return true; + return $isReady; } public function getDiskUsage() {