From 61dbc81765a0e6a884e4161690626ee76cd86dc9 Mon Sep 17 00:00:00 2001
From: Andras Bacsai <andras.bacsai@gmail.com>
Date: Fri, 23 Feb 2024 21:51:43 +0100
Subject: [PATCH] feat: delay container/server jobs

---
 .../Commands/CleanupUnreachableServers.php    |  5 ++--
 app/Console/Kernel.php                        | 28 +++++++++++++++++--
 2 files changed, 28 insertions(+), 5 deletions(-)

diff --git a/app/Console/Commands/CleanupUnreachableServers.php b/app/Console/Commands/CleanupUnreachableServers.php
index 8bbd27d64..b63dc1d36 100644
--- a/app/Console/Commands/CleanupUnreachableServers.php
+++ b/app/Console/Commands/CleanupUnreachableServers.php
@@ -8,15 +8,16 @@
 class CleanupUnreachableServers extends Command
 {
     protected $signature = 'cleanup:unreachable-servers';
-    protected $description = 'Cleanup Unreachable Servers (3 days)';
+    protected $description = 'Cleanup Unreachable Servers (7 days)';
 
     public function handle()
     {
         echo "Running unreachable server cleanup...\n";
-        $servers = Server::where('unreachable_count', 3)->where('unreachable_notification_sent', true)->where('updated_at', '<', now()->subDays(3))->get();
+        $servers = Server::where('unreachable_count', 3)->where('unreachable_notification_sent', true)->where('updated_at', '<', now()->subDays(7))->get();
         if ($servers->count() > 0) {
             foreach ($servers as $server) {
                 echo "Cleanup unreachable server ($server->id) with name $server->name";
+                send_internal_notification("Server $server->name is unreachable for 7 days. Cleaning up...");
                 $server->update([
                     'ip' => '1.2.3.4'
                 ]);
diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php
index 76fcbfc1f..636df5d36 100644
--- a/app/Console/Kernel.php
+++ b/app/Console/Kernel.php
@@ -68,13 +68,35 @@ private function check_resources($schedule)
             $containerServers = $servers->where('settings.is_swarm_worker', false)->where('settings.is_build_server', false);
         }
         foreach ($containerServers as $server) {
-            $schedule->job(new ContainerStatusJob($server))->everyMinute()->onOneServer();
+            // $schedule->job(new ContainerStatusJob($server))->everyMinute()->onOneServer();
+            $schedule
+                ->call(function () use ($server) {
+                    $randomSeconds = rand(1, 40);
+                    $job = new ContainerStatusJob($server);
+                    $job->delay($randomSeconds);
+                    ray('dispatching container status job in ' . $randomSeconds . ' seconds');
+                    dispatch($job);
+                })->name('container-status-' . $server->id)->everyMinute()->onOneServer();
             if ($server->isLogDrainEnabled()) {
-                $schedule->job(new CheckLogDrainContainerJob($server))->everyMinute()->onOneServer();
+                $schedule
+                ->call(function () use ($server) {
+                    $randomSeconds = rand(1, 40);
+                    $job = new CheckLogDrainContainerJob($server);
+                    $job->delay($randomSeconds);
+                    dispatch($job);
+                })->name('log-drain-container-check-' . $server->id)->everyMinute()->onOneServer();
+                // $schedule->job(new CheckLogDrainContainerJob($server))->everyMinute()->onOneServer();
             }
         }
         foreach ($servers as $server) {
-            $schedule->job(new ServerStatusJob($server))->everyMinute()->onOneServer();
+            $schedule
+            ->call(function () use ($server) {
+                $randomSeconds = rand(1, 40);
+                $job = new ServerStatusJob($server);
+                $job->delay($randomSeconds);
+                dispatch($job);
+            })->name('server-status-job-' . $server->id)->everyMinute()->onOneServer();
+            // $schedule->job(new ServerStatusJob($server))->everyMinute()->onOneServer();
         }
     }
     private function instance_auto_update($schedule)