diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 21c4de1d0..047f037d7 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -3,13 +3,12 @@ namespace App\Console; use App\Jobs\CheckResaleLicenseJob; -use App\Jobs\CheckResaleLicenseKeys; use App\Jobs\CleanupInstanceStuffsJob; use App\Jobs\DatabaseBackupJob; use App\Jobs\DockerCleanupJob; -use App\Jobs\InstanceApplicationsStatusJob; use App\Jobs\InstanceAutoUpdateJob; use App\Jobs\ProxyCheckJob; +use App\Jobs\ResourceStatusJob; use App\Models\ScheduledDatabaseBackup; use Illuminate\Console\Scheduling\Schedule; use Illuminate\Foundation\Console\Kernel as ConsoleKernel; @@ -21,7 +20,7 @@ protected function schedule(Schedule $schedule): void // $schedule->call(fn() => $this->check_scheduled_backups($schedule))->everyTenSeconds(); if (is_dev()) { $schedule->command('horizon:snapshot')->everyMinute(); - $schedule->job(new InstanceApplicationsStatusJob)->everyMinute(); + $schedule->job(new ResourceStatusJob)->everyMinute(); $schedule->job(new ProxyCheckJob)->everyFiveMinutes(); $schedule->job(new CleanupInstanceStuffsJob)->everyMinute(); @@ -31,7 +30,7 @@ protected function schedule(Schedule $schedule): void } else { $schedule->command('horizon:snapshot')->everyFiveMinutes(); $schedule->job(new CleanupInstanceStuffsJob)->everyMinute(); - $schedule->job(new InstanceApplicationsStatusJob)->everyMinute(); + $schedule->job(new ResourceStatusJob)->everyMinute(); $schedule->job(new CheckResaleLicenseJob)->hourly(); $schedule->job(new ProxyCheckJob)->everyFiveMinutes(); $schedule->job(new DockerCleanupJob)->everyTenMinutes(); @@ -49,7 +48,10 @@ private function check_scheduled_backups($schedule) return; } foreach ($scheduled_backups as $scheduled_backup) { - if (!$scheduled_backup->enabled) continue; + if (!$scheduled_backup->enabled) { + continue; + } + if (isset(VALID_CRON_STRINGS[$scheduled_backup->frequency])) { $scheduled_backup->frequency = VALID_CRON_STRINGS[$scheduled_backup->frequency]; } diff --git a/app/Http/Livewire/PrivateKey/Change.php b/app/Http/Livewire/PrivateKey/Change.php index d40e70e34..91d42bd7a 100644 --- a/app/Http/Livewire/PrivateKey/Change.php +++ b/app/Http/Livewire/PrivateKey/Change.php @@ -43,7 +43,7 @@ public function changePrivateKey() $this->private_key->private_key .= "\n"; } $this->private_key->save(); - refreshPrivateKey($this->private_key); + refresh_server_connection($this->private_key); } catch (\Exception $e) { return general_error_handler(err: $e, that: $this); } diff --git a/app/Http/Livewire/Server/PrivateKey.php b/app/Http/Livewire/Server/PrivateKey.php index 366aec85f..8f433f73d 100644 --- a/app/Http/Livewire/Server/PrivateKey.php +++ b/app/Http/Livewire/Server/PrivateKey.php @@ -17,7 +17,7 @@ public function setPrivateKey($private_key_id) $this->server->update([ 'private_key_id' => $private_key_id ]); - refreshPrivateKey($this->server->privateKey); + refresh_server_connection($this->server->privateKey); $this->server->refresh(); $this->checkConnection(); } diff --git a/app/Jobs/ContainerStatusJob.php b/app/Jobs/ContainerStatusJob.php index 083023929..f2be8b4d7 100644 --- a/app/Jobs/ContainerStatusJob.php +++ b/app/Jobs/ContainerStatusJob.php @@ -35,7 +35,7 @@ public function handle(): void { try { $status = get_container_status(server: $this->resource->destination->server, container_id: $this->container_name, throwError: false); - if ($this->resource->status === 'running' && $status === 'stopped') { + if ($this->resource->status === 'running' && $status !== 'running') { $this->resource->environment->project->team->notify(new StatusChanged($this->resource)); } diff --git a/app/Jobs/InstanceApplicationsStatusJob.php b/app/Jobs/ResourceStatusJob.php similarity index 92% rename from app/Jobs/InstanceApplicationsStatusJob.php rename to app/Jobs/ResourceStatusJob.php index fd173215b..5ac162d3f 100644 --- a/app/Jobs/InstanceApplicationsStatusJob.php +++ b/app/Jobs/ResourceStatusJob.php @@ -10,7 +10,7 @@ use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; -class InstanceApplicationsStatusJob implements ShouldQueue, ShouldBeUnique +class ResourceStatusJob implements ShouldQueue, ShouldBeUnique { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; diff --git a/app/Notifications/Server/NotReachable.php b/app/Notifications/Server/NotReachable.php new file mode 100644 index 000000000..0b5f71569 --- /dev/null +++ b/app/Notifications/Server/NotReachable.php @@ -0,0 +1,58 @@ +fqdn; + $mail->subject("⛔ Server '{$this->server->name}' is unreachable"); + // $mail->view('emails.application-status-changes', [ + // 'name' => $this->application_name, + // 'fqdn' => $fqdn, + // 'application_url' => $this->application_url, + // ]); + return $mail; + } + + public function toDiscord(): string + { + $message = '⛔ Server \'' . $this->server->name . '\' is unreachable (could be a temporary issue). If you receive this more than twice in a row, please check your server.'; + return $message; + } +} diff --git a/bootstrap/helpers/docker.php b/bootstrap/helpers/docker.php index 2787f153e..760f520e9 100644 --- a/bootstrap/helpers/docker.php +++ b/bootstrap/helpers/docker.php @@ -8,8 +8,8 @@ function format_docker_command_output_to_json($rawOutput): Collection $outputLines = explode(PHP_EOL, $rawOutput); return collect($outputLines) - ->reject(fn ($line) => empty($line)) - ->map(fn ($outputLine) => json_decode($outputLine, true, flags: JSON_THROW_ON_ERROR)); + ->reject(fn($line) => empty($line)) + ->map(fn($outputLine) => json_decode($outputLine, true, flags: JSON_THROW_ON_ERROR)); } function format_docker_labels_to_json($rawOutput): Collection @@ -17,7 +17,7 @@ function format_docker_labels_to_json($rawOutput): Collection $outputLines = explode(PHP_EOL, $rawOutput); return collect($outputLines) - ->reject(fn ($line) => empty($line)) + ->reject(fn($line) => empty($line)) ->map(function ($outputLine) { $outputArray = explode(',', $outputLine); return collect($outputArray) @@ -45,6 +45,7 @@ function format_docker_envs_to_json($rawOutput) function get_container_status(Server $server, string $container_id, bool $all_data = false, bool $throwError = false) { + check_server_connection($server); $container = instant_remote_process(["docker inspect --format '{{json .}}' {$container_id}"], $server, $throwError); if (!$container) { return 'exited'; diff --git a/bootstrap/helpers/remoteProcess.php b/bootstrap/helpers/remoteProcess.php index 3f8f567d8..f5bb61d75 100644 --- a/bootstrap/helpers/remoteProcess.php +++ b/bootstrap/helpers/remoteProcess.php @@ -7,6 +7,7 @@ use App\Models\ApplicationDeploymentQueue; use App\Models\PrivateKey; use App\Models\Server; +use App\Notifications\Server\NotReachable; use Carbon\Carbon; use Illuminate\Database\Eloquent\Model; use Illuminate\Support\Collection; @@ -109,8 +110,8 @@ function instant_remote_process(array $command, Server $server, $throwError = tr $exitCode = $process->exitCode(); if ($exitCode !== 0) { if ($repeat > 1) { + ray("repeat: ", $repeat); Sleep::for(200)->milliseconds(); - ray('executing again'); return instant_remote_process($command, $server, $throwError, $repeat - 1); } // ray('ERROR OCCURED: ' . $process->errorOutput()); @@ -152,21 +153,22 @@ function decode_remote_command_output(?ApplicationDeploymentQueue $application_d return $formatted; } -function refreshPrivateKey(PrivateKey $private_key) +function refresh_server_connection(PrivateKey $private_key) { foreach ($private_key->servers as $server) { // Delete the old ssh mux file to force a new one to be created Storage::disk('ssh-mux')->delete($server->muxFilename()); - if (auth()->user()->currentTeam()->id) { - auth()->user()->currentTeam()->privateKeys = PrivateKey::where('team_id', auth()->user()->currentTeam()->id)->get(); - } + // check if user is authenticated + if (auth()?->user()?->currentTeam()->id) { + auth()->user()->currentTeam()->privateKeys = PrivateKey::where('team_id', auth()->user()->currentTeam()->id)->get(); + } } } function validateServer(Server $server) { try { - refreshPrivateKey($server->privateKey); + refresh_server_connection($server->privateKey); $uptime = instant_remote_process(['uptime'], $server); if (!$uptime) { $uptime = 'Server not reachable.'; @@ -192,3 +194,25 @@ function validateServer(Server $server) $server->settings->save(); } } + +function check_server_connection(Server $server) { + try { + refresh_server_connection($server->privateKey); + instant_remote_process(['uptime'], $server); + $server->unreachable_count = 0; + $server->settings->is_reachable = true; + } catch (\Exception $e) { + if ($server->unreachable_count == 2) { + $server->team->notify(new NotReachable($server)); + $server->settings->is_reachable = false; + $server->settings->save(); + } else { + $server->unreachable_count += 1; + } + + throw $e; + } finally { + $server->settings->save(); + $server->save(); + } +} diff --git a/database/migrations/2023_08_15_111126_update_servers_add_unreachable_count_table.php b/database/migrations/2023_08_15_111126_update_servers_add_unreachable_count_table.php new file mode 100644 index 000000000..b6a0d710a --- /dev/null +++ b/database/migrations/2023_08_15_111126_update_servers_add_unreachable_count_table.php @@ -0,0 +1,28 @@ +integer('unreachable_count')->default(0); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('servers', function (Blueprint $table) { + $table->dropColumn('unreachable_count'); + }); + } +}; diff --git a/resources/views/components/server/navbar.blade.php b/resources/views/components/server/navbar.blade.php index 32691c44d..382e968c3 100644 --- a/resources/views/components/server/navbar.blade.php +++ b/resources/views/components/server/navbar.blade.php @@ -1,7 +1,9 @@