From 933ec5741d2821356e163de8ef65f8c2312b5677 Mon Sep 17 00:00:00 2001
From: Andras Bacsai <andras.bacsai@gmail.com>
Date: Mon, 9 Oct 2023 12:07:42 +0200
Subject: [PATCH] fix: server unreachable count

---
 app/Jobs/ContainerStatusJob.php               | 47 +++++++++++--------
 ...08_111819_add_server_unreachable_count.php | 28 +++++++++++
 2 files changed, 56 insertions(+), 19 deletions(-)
 create mode 100644 database/migrations/2023_10_08_111819_add_server_unreachable_count.php

diff --git a/app/Jobs/ContainerStatusJob.php b/app/Jobs/ContainerStatusJob.php
index 8156939e5..72e8cb4dd 100644
--- a/app/Jobs/ContainerStatusJob.php
+++ b/app/Jobs/ContainerStatusJob.php
@@ -46,28 +46,37 @@ public function handle()
         try {
             ray("checking server status for {$this->server->name}");
             // ray()->clearAll();
-            $serverUptimeCheckNumber = 0;
+            $serverUptimeCheckNumber = $this->server->unreachable_count;
             $serverUptimeCheckNumberMax = 3;
-            while (true) {
-                ray('checking # ' . $serverUptimeCheckNumber);
-                if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) {
-                    if ($this->server->unreachable_email_sent === false) {
-                        ray('Server unreachable, sending notification...');
-                        // $this->server->team->notify(new Unreachable($this->server));
-                    }
-                    $this->server->settings()->update([
-                        'is_reachable' => false,
-                    ]);
+
+            ray('checking # ' . $serverUptimeCheckNumber);
+            if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) {
+                if ($this->server->unreachable_email_sent === false) {
+                    ray('Server unreachable, sending notification...');
+                    // $this->server->team->notify(new Unreachable($this->server));
                     $this->server->update(['unreachable_email_sent' => true]);
-                    return;
                 }
-                $result = $this->server->validateConnection();
-                if ($result) {
-                    break;
-                }
-                $serverUptimeCheckNumber++;
-                sleep(5);
+                $this->server->settings()->update([
+                    'is_reachable' => false,
+                    'unreachable_count' => 0,
+                ]);
+                return;
             }
+            $result = $this->server->validateConnection();
+            if ($result) {
+                $this->server->settings()->update([
+                    'is_reachable' => true,
+                    'unreachable_count' => 0,
+                ]);
+            } else {
+                $serverUptimeCheckNumber++;
+                $this->server->settings()->update([
+                    'is_reachable' => false,
+                    'unreachable_count' => $serverUptimeCheckNumber,
+                ]);
+                return;
+            }
+
             if (data_get($this->server, 'unreachable_email_sent') === true) {
                 ray('Server is reachable again, sending notification...');
                 // $this->server->team->notify(new Revived($this->server));
@@ -82,7 +91,7 @@ public function handle()
                     'is_usable' => true
                 ]);
             }
-            $this->server->validateDockerEngine(true);
+            // $this->server->validateDockerEngine(true);
             $containers = instant_remote_process(["docker container ls -q"], $this->server);
             if (!$containers) {
                 return;
diff --git a/database/migrations/2023_10_08_111819_add_server_unreachable_count.php b/database/migrations/2023_10_08_111819_add_server_unreachable_count.php
new file mode 100644
index 000000000..b6a0d710a
--- /dev/null
+++ b/database/migrations/2023_10_08_111819_add_server_unreachable_count.php
@@ -0,0 +1,28 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        Schema::table('servers', function (Blueprint $table) {
+            $table->integer('unreachable_count')->default(0);
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::table('servers', function (Blueprint $table) {
+            $table->dropColumn('unreachable_count');
+        });
+    }
+};