feat: monitor server connection

app/Console/Kernel.php

@@ -3,13 +3,12 @@
 namespace App\Console;
 
 use App\Jobs\CheckResaleLicenseJob;
-use App\Jobs\CheckResaleLicenseKeys;
 use App\Jobs\CleanupInstanceStuffsJob;
 use App\Jobs\DatabaseBackupJob;
 use App\Jobs\DockerCleanupJob;
-use App\Jobs\InstanceApplicationsStatusJob;
 use App\Jobs\InstanceAutoUpdateJob;
 use App\Jobs\ProxyCheckJob;
+use App\Jobs\ResourceStatusJob;
 use App\Models\ScheduledDatabaseBackup;
 use Illuminate\Console\Scheduling\Schedule;
 use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
@@ -21,7 +20,7 @@ protected function schedule(Schedule $schedule): void
         //        $schedule->call(fn() => $this->check_scheduled_backups($schedule))->everyTenSeconds();
         if (is_dev()) {
             $schedule->command('horizon:snapshot')->everyMinute();
-            $schedule->job(new InstanceApplicationsStatusJob)->everyMinute();
+            $schedule->job(new ResourceStatusJob)->everyMinute();
             $schedule->job(new ProxyCheckJob)->everyFiveMinutes();
             $schedule->job(new CleanupInstanceStuffsJob)->everyMinute();
 
@@ -31,7 +30,7 @@ protected function schedule(Schedule $schedule): void
         } else {
             $schedule->command('horizon:snapshot')->everyFiveMinutes();
             $schedule->job(new CleanupInstanceStuffsJob)->everyMinute();
-            $schedule->job(new InstanceApplicationsStatusJob)->everyMinute();
+            $schedule->job(new ResourceStatusJob)->everyMinute();
             $schedule->job(new CheckResaleLicenseJob)->hourly();
             $schedule->job(new ProxyCheckJob)->everyFiveMinutes();
             $schedule->job(new DockerCleanupJob)->everyTenMinutes();
@@ -49,7 +48,10 @@ private function check_scheduled_backups($schedule)
             return;
         }
         foreach ($scheduled_backups as $scheduled_backup) {
-            if (!$scheduled_backup->enabled) continue;
+            if (!$scheduled_backup->enabled) {
+                continue;
+            }
+
             if (isset(VALID_CRON_STRINGS[$scheduled_backup->frequency])) {
                 $scheduled_backup->frequency = VALID_CRON_STRINGS[$scheduled_backup->frequency];
             }

app/Http/Livewire/PrivateKey/Change.php

@@ -43,7 +43,7 @@ public function changePrivateKey()
                 $this->private_key->private_key .= "\n";
             }
             $this->private_key->save();
-            refreshPrivateKey($this->private_key);
+            refresh_server_connection($this->private_key);
         } catch (\Exception $e) {
             return general_error_handler(err: $e, that: $this);
         }

app/Http/Livewire/Server/PrivateKey.php

@@ -17,7 +17,7 @@ public function setPrivateKey($private_key_id)
         $this->server->update([
             'private_key_id' => $private_key_id
         ]);
-        refreshPrivateKey($this->server->privateKey);
+        refresh_server_connection($this->server->privateKey);
         $this->server->refresh();
         $this->checkConnection();
     }

app/Jobs/ContainerStatusJob.php

@@ -35,7 +35,7 @@ public function handle(): void
     {
         try {
             $status = get_container_status(server: $this->resource->destination->server, container_id: $this->container_name, throwError: false);
-            if ($this->resource->status === 'running' && $status === 'stopped') {
+            if ($this->resource->status === 'running' && $status !== 'running') {
                 $this->resource->environment->project->team->notify(new StatusChanged($this->resource));
             }
 

app/Jobs/ResourceStatusJob.php

@@ -10,7 +10,7 @@
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
 
-class InstanceApplicationsStatusJob implements ShouldQueue, ShouldBeUnique
+class ResourceStatusJob implements ShouldQueue, ShouldBeUnique
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 

app/Notifications/Server/NotReachable.php

@@ -0,0 +1,58 @@
+<?php
+
+namespace App\Notifications\Server;
+
+use App\Models\Server;
+use App\Notifications\Channels\DiscordChannel;
+use App\Notifications\Channels\EmailChannel;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Notifications\Messages\MailMessage;
+use Illuminate\Notifications\Notification;
+use Illuminate\Support\Str;
+
+class NotReachable extends Notification implements ShouldQueue
+{
+    use Queueable;
+
+    public function __construct(public Server $server)
+    {
+
+    }
+
+    public function via(object $notifiable): array
+    {
+        $channels = [];
+        $isEmailEnabled = data_get($notifiable, 'smtp_enabled');
+        $isDiscordEnabled = data_get($notifiable, 'discord_enabled');
+        $isSubscribedToEmailEvent = data_get($notifiable, 'smtp_notifications_status_changes');
+        $isSubscribedToDiscordEvent = data_get($notifiable, 'discord_notifications_status_changes');
+
+        // if ($isEmailEnabled && $isSubscribedToEmailEvent) {
+        //     $channels[] = EmailChannel::class;
+        // }
+        if ($isDiscordEnabled && $isSubscribedToDiscordEvent) {
+            $channels[] = DiscordChannel::class;
+        }
+        return $channels;
+    }
+
+    public function toMail(): MailMessage
+    {
+        $mail = new MailMessage();
+        // $fqdn = $this->fqdn;
+        $mail->subject("⛔ Server '{$this->server->name}' is unreachable");
+        // $mail->view('emails.application-status-changes', [
+        //     'name' => $this->application_name,
+        //     'fqdn' => $fqdn,
+        //     'application_url' => $this->application_url,
+        // ]);
+        return $mail;
+    }
+
+    public function toDiscord(): string
+    {
+        $message = '⛔ Server \'' . $this->server->name . '\' is unreachable (could be a temporary issue). If you receive this more than twice in a row, please check your server.';
+        return $message;
+    }
+}

bootstrap/helpers/docker.php

@@ -8,8 +8,8 @@ function format_docker_command_output_to_json($rawOutput): Collection
     $outputLines = explode(PHP_EOL, $rawOutput);
 
     return collect($outputLines)
-        ->reject(fn ($line) => empty($line))
+        ->reject(fn($line) => empty($line))
-        ->map(fn ($outputLine) => json_decode($outputLine, true, flags: JSON_THROW_ON_ERROR));
+        ->map(fn($outputLine) => json_decode($outputLine, true, flags: JSON_THROW_ON_ERROR));
 }
 
 function format_docker_labels_to_json($rawOutput): Collection
@@ -17,7 +17,7 @@ function format_docker_labels_to_json($rawOutput): Collection
     $outputLines = explode(PHP_EOL, $rawOutput);
 
     return collect($outputLines)
-        ->reject(fn ($line) => empty($line))
+        ->reject(fn($line) => empty($line))
         ->map(function ($outputLine) {
             $outputArray = explode(',', $outputLine);
             return collect($outputArray)
@@ -45,6 +45,7 @@ function format_docker_envs_to_json($rawOutput)
 
 function get_container_status(Server $server, string $container_id, bool $all_data = false, bool $throwError = false)
 {
+    check_server_connection($server);
     $container = instant_remote_process(["docker inspect --format '{{json .}}' {$container_id}"], $server, $throwError);
     if (!$container) {
         return 'exited';

bootstrap/helpers/remoteProcess.php

@@ -7,6 +7,7 @@
 use App\Models\ApplicationDeploymentQueue;
 use App\Models\PrivateKey;
 use App\Models\Server;
+use App\Notifications\Server\NotReachable;
 use Carbon\Carbon;
 use Illuminate\Database\Eloquent\Model;
 use Illuminate\Support\Collection;
@@ -109,8 +110,8 @@ function instant_remote_process(array $command, Server $server, $throwError = tr
     $exitCode = $process->exitCode();
     if ($exitCode !== 0) {
         if ($repeat > 1) {
+            ray("repeat: ", $repeat);
             Sleep::for(200)->milliseconds();
-            ray('executing again');
             return instant_remote_process($command, $server, $throwError, $repeat - 1);
         }
         // ray('ERROR OCCURED: ' . $process->errorOutput());
@@ -152,21 +153,22 @@ function decode_remote_command_output(?ApplicationDeploymentQueue $application_d
     return $formatted;
 }
 
-function refreshPrivateKey(PrivateKey $private_key)
+function refresh_server_connection(PrivateKey $private_key)
 {
     foreach ($private_key->servers as $server) {
         // Delete the old ssh mux file to force a new one to be created
         Storage::disk('ssh-mux')->delete($server->muxFilename());
-        if (auth()->user()->currentTeam()->id) {
+        // check if user is authenticated
-            auth()->user()->currentTeam()->privateKeys = PrivateKey::where('team_id', auth()->user()->currentTeam()->id)->get();
+            if (auth()?->user()?->currentTeam()->id) {
-        }
+                auth()->user()->currentTeam()->privateKeys = PrivateKey::where('team_id', auth()->user()->currentTeam()->id)->get();
+            }
     }
 }
 
 function validateServer(Server $server)
 {
     try {
-        refreshPrivateKey($server->privateKey);
+        refresh_server_connection($server->privateKey);
         $uptime = instant_remote_process(['uptime'], $server);
         if (!$uptime) {
             $uptime = 'Server not reachable.';
@@ -192,3 +194,25 @@ function validateServer(Server $server)
         $server->settings->save();
     }
 }
+
+function check_server_connection(Server $server) {
+    try {
+        refresh_server_connection($server->privateKey);
+        instant_remote_process(['uptime'], $server);
+        $server->unreachable_count = 0;
+        $server->settings->is_reachable = true;
+    } catch (\Exception $e) {
+        if ($server->unreachable_count == 2) {
+            $server->team->notify(new NotReachable($server));
+            $server->settings->is_reachable = false;
+            $server->settings->save();
+        } else {
+            $server->unreachable_count += 1;
+        }
+
+        throw $e;
+    } finally {
+        $server->settings->save();
+        $server->save();
+    }
+}

database/migrations/2023_08_15_111126_update_servers_add_unreachable_count_table.php

@@ -0,0 +1,28 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        Schema::table('servers', function (Blueprint $table) {
+            $table->integer('unreachable_count')->default(0);
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::table('servers', function (Blueprint $table) {
+            $table->dropColumn('unreachable_count');
+        });
+    }
+};

resources/views/components/server/navbar.blade.php

@@ -1,7 +1,9 @@
 <div class="pb-6">
     <div class="flex items-center gap-2">
         <h1>Server</h1>
-        <livewire:server.proxy.status :server="$server" />
+        @if ($server->settings->is_reachable)
+            <livewire:server.proxy.status :server="$server" />
+        @endif
     </div>
     <div class="subtitle ">{{ data_get($server, 'name') }}</div>
     <nav class="navbar-main">

resources/views/livewire/server/form.blade.php

@@ -10,42 +10,51 @@
                 running on.<br>Please think again.</p>
         </x-slot:modalBody>
     </x-modal>
-    @if ($server->settings->is_reachable)
+    <form wire:submit.prevent='submit' class="flex flex-col">
-        <form wire:submit.prevent='submit' class="flex flex-col">
+        <div class="flex gap-2">
-            <div class="flex gap-2">
+            <h2>General</h2>
-                <h2>General</h2>
+            @if ($server->id === 0)
-                @if ($server->id === 0)
+                <x-forms.button isModal modalId="changeLocalhost">Save</x-forms.button>
-                    <x-forms.button isModal modalId="changeLocalhost">Save</x-forms.button>
+            @else
-                @else
+                <x-forms.button type="submit">Save</x-forms.button>
-                    <x-forms.button type="submit">Save</x-forms.button>
+            @endif
-                @endif
+            <x-forms.button wire:click.prevent='validateServer'>
-            </div>
+                Validate Server
-            <div class="flex flex-col gap-2 ">
+            </x-forms.button>
-                <div class="flex flex-col w-full gap-2 lg:flex-row">
+        </div>
-                    <x-forms.input id="server.name" label="Name" required />
+        @if (!$server->settings->is_reachable)
-                    <x-forms.input id="server.description" label="Description" />
+            You can't use this server until it is validated.
-                    <x-forms.input placeholder="https://example.com" id="wildcard_domain" label="Wildcard Domain"
+        @else
-                        helper="Wildcard domain for your applications. If you set this, you will get a random generated domain for your new applications.<br><span class='font-bold text-white'>Example</span>In case you set:<span class='text-helper'>https://example.com</span>your applications will get: <span class='text-helper'>https://randomId.example.com</span>" />
+            Server validated.
-                    {{-- <x-forms.checkbox disabled type="checkbox" id="server.settings.is_part_of_swarm"
+        @endif
+        <div class="flex flex-col gap-2 pt-4">
+            <div class="flex flex-col w-full gap-2 lg:flex-row">
+                <x-forms.input id="server.name" label="Name" required />
+                <x-forms.input id="server.description" label="Description" />
+                <x-forms.input placeholder="https://example.com" id="wildcard_domain" label="Wildcard Domain"
+                    helper="Wildcard domain for your applications. If you set this, you will get a random generated domain for your new applications.<br><span class='font-bold text-white'>Example</span>In case you set:<span class='text-helper'>https://example.com</span>your applications will get: <span class='text-helper'>https://randomId.example.com</span>" />
+                {{-- <x-forms.checkbox disabled type="checkbox" id="server.settings.is_part_of_swarm"
                     label="Is it part of a Swarm cluster?" /> --}}
-                </div>
+            </div>
-                <div class="flex flex-col w-full gap-2 lg:flex-row">
+            <div class="flex flex-col w-full gap-2 lg:flex-row">
-                    @if ($server->id === 0)
+                @if ($server->id === 0)
-                        <x-forms.input id="server.ip" label="IP Address" required />
+                    <x-forms.input id="server.ip" label="IP Address" required />
-                    @else
+                @else
-                        <x-forms.input id="server.ip" label="IP Address" readonly required />
+                    <x-forms.input id="server.ip" label="IP Address" readonly required />
-                    @endif
+                @endif
-                    <div class="flex gap-2">
+                <div class="flex gap-2">
-                        <x-forms.input id="server.user" label="User" required />
+                    <x-forms.input id="server.user" label="User" required />
-                        <x-forms.input type="number" id="server.port" label="Port" required />
+                    <x-forms.input type="number" id="server.port" label="Port" required />
-                    </div>
                 </div>
             </div>
+        </div>
+        @if ($server->settings->is_reachable)
             <h3 class="py-4">Settings</h3>
             <div class="flex items-center w-64 gap-2">
                 <x-forms.input id="cleanup_after_percentage" label="Disk Cleanup threshold (%)" required
                     helper="Disk cleanup job will be executed if disk usage is more than this number." />
             </div>
+
             <h3 class="py-4">Actions</h3>
             <div class="flex items-center gap-2">
                 <x-forms.button wire:click.prevent='validateServer'>
@@ -61,26 +70,20 @@
                     </x-forms.button>
                 @endif
             </div>
-            <div class="container w-full py-4 mx-auto">
+        @endif
-                <livewire:activity-monitor header="Logs" />
+        <div class="container w-full py-4 mx-auto">
-            </div>
+            <livewire:activity-monitor header="Logs" />
-            @isset($uptime)
-                <h3 class="pb-3">Server Info</h3>
-                <div class="py-2 pb-4">
-                    <p>Uptime: {{ $uptime }}</p>
-                    @isset($dockerVersion)
-                        <p>Docker Engine {{ $dockerVersion }}</p>
-                    @endisset
-                </div>
-            @endisset
-        </form>
-    @else
-        <div class="w-full pb-4">
-            <div class="cursor-pointer box" wire:click.prevent='validateServer'>
-                Validate Server
-            </div>
         </div>
-    @endif
+        @isset($uptime)
+            <h3 class="pb-3">Server Info</h3>
+            <div class="py-2 pb-4">
+                <p>Uptime: {{ $uptime }}</p>
+                @isset($dockerVersion)
+                    <p>Docker Engine {{ $dockerVersion }}</p>
+                @endisset
+            </div>
+        @endisset
+    </form>
     <h2>Danger Zone</h2>
     <div class="">Woah. I hope you know what are you doing.</div>
     <h4 class="pt-4">Delete Server</h4>