fix: better unreachable/revived server statuses

app/Console/Commands/Cloud.php

@@ -0,0 +1,33 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Models\Server;
+use Illuminate\Console\Command;
+
+class Cloud extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'cloud:unused-servers';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Get Unused Servers from Cloud';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        Server::all()->whereNotNull('team.subscription')->where('team.subscription.stripe_trial_already_ended',true)->each(function($server){
+            $this->info($server->name);
+        });
+    }
+}

app/Console/Kernel.php

@@ -48,7 +48,11 @@ class Kernel extends ConsoleKernel
     }
     private function check_resources($schedule)
     {
-        $servers = Server::all()->where('settings.is_usable', true)->where('settings.is_reachable', true);
+        if (isCloud()) {
+            $servers = Server::all()->whereNotNull('team.subscription')->where('team.subscription.stripe_trial_already_ended', false);
+        } else {
+            $servers = Server::all();
+        }
         foreach ($servers as $server) {
             $schedule->job(new ContainerStatusJob($server))->everyMinute()->onOneServer();
         }

app/Jobs/ContainerStatusJob.php

@@ -7,6 +7,7 @@ use App\Models\ApplicationPreview;
 use App\Models\Server;
 use App\Notifications\Container\ContainerRestarted;
 use App\Notifications\Container\ContainerStopped;
+use App\Notifications\Server\Revived;
 use App\Notifications\Server\Unreachable;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldBeEncrypted;
@@ -51,14 +52,20 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted
     public function handle(): void
     {
         try {
+            ray("checking server status for {$this->server->name}");
             // ray()->clearAll();
             $serverUptimeCheckNumber = 0;
             $serverUptimeCheckNumberMax = 3;
             while (true) {
                 if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) {
                     send_internal_notification('Server unreachable: ' . $this->server->name);
-                    // $this->server->settings()->update(['is_reachable' => false]);
+                    if ($this->server->unreachable_email_sent === false) {
-                    // $this->server->team->notify(new Unreachable($this->server));
+                        $this->server->team->notify(new Unreachable($this->server));
+                    }
+                    $this->server->settings()->update([
+                        'is_reachable' => false,
+                    ]);
+                    $this->server->update(['unreachable_email_sent' => true]);
                     return;
                 }
                 $result = $this->checkServerConnection();
@@ -68,6 +75,20 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted
                 $serverUptimeCheckNumber++;
                 sleep(5);
             }
+            if (data_get($this->server, 'unreachable_email_sent') === true) {
+                $this->server->team->notify(new Revived($this->server));
+                $this->server->update(['unreachable_email_sent' => false]);
+            }
+            if (
+                data_get($this->server, 'settings.is_reachable') === false ||
+                data_get($this->server, 'settings.is_usable') === false
+            ) {
+                $this->server->settings()->update([
+                    'is_reachable' => true,
+                    'is_usable' => true
+                ]);
+            }
+
             $containers = instant_remote_process(["docker container ls -q"], $this->server);
             if (!$containers) {
                 return;

app/Models/Server.php

@@ -8,6 +8,7 @@ use Illuminate\Database\Eloquent\Builder;
 use Illuminate\Database\Eloquent\Casts\Attribute;
 use Spatie\SchemalessAttributes\Casts\SchemalessAttributes;
 use Spatie\SchemalessAttributes\SchemalessAttributesTrait;
+use Illuminate\Support\Str;
 
 class Server extends BaseModel
 {
@@ -15,6 +16,11 @@ class Server extends BaseModel
 
     protected static function booted()
     {
+        static::saved(function ($server) {
+            $server->ip = Str::of($server->ip)->trim();
+            $server->user = Str::of($server->user)->trim();
+        });
+
         static::created(function ($server) {
             ServerSetting::create([
                 'server_id' => $server->id,

app/Notifications/Server/Revived.php

@@ -0,0 +1,49 @@
+<?php
+
+namespace App\Notifications\Server;
+
+use App\Models\Server;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Notifications\Messages\MailMessage;
+use Illuminate\Notifications\Notification;
+
+class Revived extends Notification implements ShouldQueue
+{
+    use Queueable;
+
+    public $tries = 1;
+    public function __construct(public Server $server)
+    {
+        if ($this->server->unreachable_email_sent === false) {
+            return;
+        }
+    }
+
+    public function via(object $notifiable): array
+    {
+        return setNotificationChannels($notifiable, 'status_changes');
+    }
+
+    public function toMail(): MailMessage
+    {
+        $mail = new MailMessage();
+        $mail->subject("✅ Server ({$this->server->name}) revived.");
+        $mail->view('emails.server-revived', [
+            'name' => $this->server->name,
+        ]);
+        return $mail;
+    }
+
+    public function toDiscord(): string
+    {
+        $message = "✅ Server '{$this->server->name}' revived. All automations & integrations are turned on again!";
+        return $message;
+    }
+    public function toTelegram(): array
+    {
+        return [
+            "message" => "✅ Server '{$this->server->name}' revived. All automations & integrations are turned on again!"
+        ];
+    }
+}

app/Notifications/Server/Unreachable.php

@@ -35,13 +35,13 @@ class Unreachable extends Notification implements ShouldQueue
 
     public function toDiscord(): string
     {
-        $message = "⛔ Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: You have to validate your server again after you fix the issue.";
+        $message = "⛔ Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: We automatically try to revive your server. If your server is back online, we will automatically turn on all automations & integrations.";
         return $message;
     }
     public function toTelegram(): array
     {
         return [
-            "message" => "⛔ Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: You have to validate your server again after you fix the issue."
+            "message" => "⛔ Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: We automatically try to revive your server. If your server is back online, we will automatically turn on all automations & integrations."
         ];
     }
 }

bootstrap/helpers/remoteProcess.php

@@ -7,6 +7,8 @@ use App\Models\Application;
 use App\Models\ApplicationDeploymentQueue;
 use App\Models\PrivateKey;
 use App\Models\Server;
+use App\Notifications\Server\Revived;
+use App\Notifications\Server\Unreachable;
 use Carbon\Carbon;
 use Illuminate\Database\Eloquent\Model;
 use Illuminate\Support\Collection;
@@ -85,7 +87,7 @@ function generateSshCommand(Server $server, string $command, bool $isMux = true)
     if ($isMux && config('coolify.mux_enabled')) {
         $ssh_command .= '-o ControlMaster=auto -o ControlPersist=1m -o ControlPath=/var/www/html/storage/app/ssh/mux/%h_%p_%r ';
     }
-    if (data_get($server,'settings.is_cloudflare_tunnel')) {
+    if (data_get($server, 'settings.is_cloudflare_tunnel')) {
         $ssh_command .= '-o ProxyCommand="/usr/local/bin/cloudflared access ssh --hostname %h" ';
     }
     $command = "PATH=\$PATH:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/host/usr/local/sbin:/host/usr/local/bin:/host/usr/sbin:/host/usr/bin:/host/sbin:/host/bin && $command";
@@ -122,13 +124,14 @@ function instant_remote_process(Collection|array $command, Server $server, $thro
     }
     return $output;
 }
-function excludeCertainErrors(string $errorOutput, ?int $exitCode = null) {
+function excludeCertainErrors(string $errorOutput, ?int $exitCode = null)
+{
     $ignoredErrors = collect([
         'Permission denied (publickey',
         'Could not resolve hostname',
     ]);
     $ignored = false;
-    foreach ($ignoredErrors as $ignoredError)  {
+    foreach ($ignoredErrors as $ignoredError) {
         if (Str::contains($errorOutput, $ignoredError)) {
             $ignored = true;
             break;
@@ -183,6 +186,9 @@ function validateServer(Server $server, bool $throwError = false)
         $uptime = instant_remote_process(['uptime'], $server, $throwError);
         if (!$uptime) {
             $server->settings->is_reachable = false;
+            $server->team->notify(new Unreachable($server));
+            $server->unreachable_email_sent = true;
+            $server->save();
             return [
                 "uptime" => null,
                 "dockerVersion" => null,
@@ -203,6 +209,11 @@ function validateServer(Server $server, bool $throwError = false)
             $server->settings->is_usable = false;
         } else {
             $server->settings->is_usable = true;
+            if (data_get($server, 'unreachable_email_sent') === true) {
+                $server->team->notify(new Revived($server));
+                $server->unreachable_email_sent = false;
+                $server->save();
+            }
         }
         return [
             "uptime" => $uptime,
@@ -213,7 +224,9 @@ function validateServer(Server $server, bool $throwError = false)
         $server->settings->is_usable = false;
         throw $e;
     } finally {
-        if (data_get($server, 'settings')) $server->settings->save();
+        if (data_get($server, 'settings')) {
+            $server->settings->save();
+        }
     }
 }
 

bootstrap/helpers/shared.php

@@ -310,6 +310,7 @@ function send_internal_notification(string $message): void
         $baseUrl = config('app.name');
         $team = Team::find(0);
         $team->notify(new GeneralNotification("👀 {$baseUrl}: " . $message));
+        ray("👀 {$baseUrl}: " . $message);
     } catch (\Throwable $e) {
         ray($e->getMessage());
     }

database/migrations/2023_09_23_111819_add_server_emails.php

@@ -0,0 +1,31 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        Schema::table('servers', function (Blueprint $table) {
+            $table->boolean('unreachable_email_sent')->default(false);
+            $table->dropColumn('unreachable_count');
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::table('servers', function (Blueprint $table) {
+            $table->dropColumn('unreachable_email_sent');
+            $table->integer('unreachable_count')->default(0);
+        });
+
+    }
+};

resources/views/emails/server-lost-connection.blade.php

@@ -4,7 +4,7 @@ Coolify cannot connect to your server ({{$name}}). Please check your server and
 
 All automations & integrations are turned off!
 
-IMPORTANT: You have to validate your server again after you fix the issue.
+IMPORTANT: We automatically try to revive your server. If your server is back online, we will automatically turn on all automations & integrations.
 
 If you have any questions, please contact us.
 

resources/views/emails/server-revived.blade.php

@@ -0,0 +1,6 @@
+<x-emails.layout>
+
+Your server ({{$name}}) was offline for a while, but it is back online now. All automations & integrations are turned on again.
+
+</x-emails.layout>
+