From 53c20e1e991b50d4d326a58ebc71a8013382319d Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 14 Sep 2023 12:45:50 +0200 Subject: [PATCH] feat: new container status checks --- app/Actions/Proxy/StartProxy.php | 16 +- app/Console/Kernel.php | 32 +--- .../Livewire/Project/Application/Heading.php | 6 +- .../Livewire/Project/Database/Heading.php | 7 +- app/Http/Livewire/Server/Form.php | 16 +- app/Http/Livewire/Settings/Configuration.php | 4 +- app/Jobs/ApplicationContainerStatusJob.php | 1 - app/Jobs/ApplicationDeploymentJob.php | 5 +- app/Jobs/ContainerStatusJob.php | 139 ++++++++++++++++++ app/Jobs/ServerDetailsCheckJob.php | 80 ---------- .../Application/DeploymentSuccess.php | 2 +- .../Container/ContainerRestarted.php | 62 ++++++++ .../Container/ContainerStopped.php | 61 ++++++++ app/Notifications/Server/NotReachable.php | 53 ------- app/Notifications/Server/Unreachable.php | 47 ++++++ .../emails/container-restarted.blade.php | 11 ++ .../views/emails/container-stopped.blade.php | 9 ++ .../emails/server-lost-connection.blade.php | 7 +- 18 files changed, 369 insertions(+), 189 deletions(-) create mode 100644 app/Jobs/ContainerStatusJob.php delete mode 100644 app/Jobs/ServerDetailsCheckJob.php create mode 100644 app/Notifications/Container/ContainerRestarted.php create mode 100644 app/Notifications/Container/ContainerStopped.php delete mode 100644 app/Notifications/Server/NotReachable.php create mode 100644 app/Notifications/Server/Unreachable.php create mode 100644 resources/views/emails/container-restarted.blade.php create mode 100644 resources/views/emails/container-stopped.blade.php diff --git a/app/Actions/Proxy/StartProxy.php b/app/Actions/Proxy/StartProxy.php index d8a14d27e..7e2878b7e 100644 --- a/app/Actions/Proxy/StartProxy.php +++ b/app/Actions/Proxy/StartProxy.php @@ -8,7 +8,7 @@ class StartProxy { - public function __invoke(Server $server): Activity + public function __invoke(Server $server, bool $async = true): Activity|string { $proxy_path = get_proxy_path(); $networks = collect($server->standaloneDockers)->map(function ($docker) { @@ -26,8 +26,7 @@ public function __invoke(Server $server): Activity $docker_compose_yml_base64 = base64_encode($configuration); $server->proxy->last_applied_settings = Str::of($docker_compose_yml_base64)->pipe('md5')->value; $server->save(); - - $activity = remote_process([ + $commands = [ "echo '####### Creating required Docker networks...'", ...$create_networks_command, "cd $proxy_path", @@ -44,8 +43,13 @@ public function __invoke(Server $server): Activity "echo '####### Starting coolify-proxy...'", 'docker compose up -d --remove-orphans', "echo '####### Proxy installed successfully...'" - ], $server); - - return $activity; + ]; + if (!$async) { + instant_remote_process($commands, $server); + return 'OK'; + } else { + $activity = remote_process($commands, $server); + return $activity; + } } } diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index ae40fec49..d15107c19 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -2,21 +2,15 @@ namespace App\Console; -use App\Enums\ProxyTypes; -use App\Jobs\ApplicationContainerStatusJob; use App\Jobs\CheckResaleLicenseJob; use App\Jobs\CleanupInstanceStuffsJob; use App\Jobs\DatabaseBackupJob; -use App\Jobs\DatabaseContainerStatusJob; use App\Jobs\DockerCleanupJob; use App\Jobs\InstanceAutoUpdateJob; -use App\Jobs\ProxyContainerStatusJob; -use App\Jobs\ServerDetailsCheckJob; -use App\Models\Application; +use App\Jobs\ContainerStatusJob; use App\Models\InstanceSettings; use App\Models\ScheduledDatabaseBackup; use App\Models\Server; -use App\Models\StandalonePostgresql; use Illuminate\Console\Scheduling\Schedule; use Illuminate\Foundation\Console\Kernel as ConsoleKernel; @@ -25,15 +19,14 @@ class Kernel extends ConsoleKernel protected function schedule(Schedule $schedule): void { if (isDev()) { - $schedule->job(new ServerDetailsCheckJob(Server::find(0)))->everyTenMinutes()->onOneServer(); + // $schedule->job(new ContainerStatusJob(Server::find(0)))->everyTenMinutes()->onOneServer(); // $schedule->command('horizon:snapshot')->everyMinute(); // $schedule->job(new CleanupInstanceStuffsJob)->everyMinute(); // $schedule->job(new CheckResaleLicenseJob)->hourly(); // $schedule->job(new DockerCleanupJob)->everyOddHour(); // $this->instance_auto_update($schedule); // $this->check_scheduled_backups($schedule); - // $this->check_resources($schedule); - // $this->check_proxies($schedule); + $this->check_resources($schedule); } else { $schedule->command('horizon:snapshot')->everyFiveMinutes(); $schedule->job(new CleanupInstanceStuffsJob)->everyTenMinutes()->onOneServer(); @@ -42,26 +35,15 @@ protected function schedule(Schedule $schedule): void $this->instance_auto_update($schedule); $this->check_scheduled_backups($schedule); $this->check_resources($schedule); - $this->check_proxies($schedule); - } - } - private function check_proxies($schedule) - { - $servers = Server::all()->where('settings.is_usable', true)->where('settings.is_reachable', true)->whereNotNull('proxy.type')->where('proxy.type', '!=', ProxyTypes::NONE->value); - foreach ($servers as $server) { - $schedule->job(new ProxyContainerStatusJob($server))->everyMinute()->onOneServer(); } } private function check_resources($schedule) { - $applications = Application::all(); - foreach ($applications as $application) { - $schedule->job(new ApplicationContainerStatusJob($application))->everyMinute()->onOneServer(); - } + $servers = Server::all()->where('settings.is_usable', true)->where('settings.is_reachable', true); + ray($servers); - $postgresqls = StandalonePostgresql::all(); - foreach ($postgresqls as $postgresql) { - $schedule->job(new DatabaseContainerStatusJob($postgresql))->everyMinute()->onOneServer(); + foreach ($servers as $server) { + $schedule->job(new ContainerStatusJob($server))->everyMinute()->onOneServer(); } } private function instance_auto_update($schedule) diff --git a/app/Http/Livewire/Project/Application/Heading.php b/app/Http/Livewire/Project/Application/Heading.php index 5b9560007..0774ad441 100644 --- a/app/Http/Livewire/Project/Application/Heading.php +++ b/app/Http/Livewire/Project/Application/Heading.php @@ -3,6 +3,7 @@ namespace App\Http\Livewire\Project\Application; use App\Jobs\ApplicationContainerStatusJob; +use App\Jobs\ContainerStatusJob; use App\Models\Application; use App\Notifications\Application\StatusChanged; use Livewire\Component; @@ -22,9 +23,8 @@ public function mount() public function check_status() { - dispatch_sync(new ApplicationContainerStatusJob( - application: $this->application, - )); + ray($this->application->destination->server); + dispatch_sync(new ContainerStatusJob($this->application->destination->server)); $this->application->refresh(); } diff --git a/app/Http/Livewire/Project/Database/Heading.php b/app/Http/Livewire/Project/Database/Heading.php index 178cbfca5..f2fe8f033 100644 --- a/app/Http/Livewire/Project/Database/Heading.php +++ b/app/Http/Livewire/Project/Database/Heading.php @@ -3,8 +3,7 @@ namespace App\Http\Livewire\Project\Database; use App\Actions\Database\StartPostgresql; -use App\Jobs\DatabaseContainerStatusJob; -use App\Notifications\Application\StatusChanged; +use App\Jobs\ContainerStatusJob; use Livewire\Component; class Heading extends Component @@ -25,9 +24,7 @@ public function activityFinished() public function check_status() { - dispatch_sync(new DatabaseContainerStatusJob( - database: $this->database, - )); + dispatch_sync(new ContainerStatusJob($this->database->destination->server)); $this->database->refresh(); } diff --git a/app/Http/Livewire/Server/Form.php b/app/Http/Livewire/Server/Form.php index 9d5c892b3..b288d3bfa 100644 --- a/app/Http/Livewire/Server/Form.php +++ b/app/Http/Livewire/Server/Form.php @@ -88,17 +88,11 @@ public function delete() public function submit() { $this->validate(); - // $validation = Validator::make($this->server->toArray(), [ - // 'ip' => [ - // 'ip' - // ], - // ]); - // if ($validation->fails()) { - // foreach ($validation->errors()->getMessages() as $key => $value) { - // $this->addError("server.{$key}", $value[0]); - // } - // return; - // } + $uniqueIPs = Server::all()->pluck('ip')->toArray(); + if (in_array($this->server->ip, $uniqueIPs)) { + $this->emit('error', 'IP address is already in use by another team.'); + return; + } $this->server->settings->wildcard_domain = $this->wildcard_domain; $this->server->settings->cleanup_after_percentage = $this->cleanup_after_percentage; $this->server->settings->save(); diff --git a/app/Http/Livewire/Settings/Configuration.php b/app/Http/Livewire/Settings/Configuration.php index 6801823fe..9bc7c20cb 100644 --- a/app/Http/Livewire/Settings/Configuration.php +++ b/app/Http/Livewire/Settings/Configuration.php @@ -2,7 +2,7 @@ namespace App\Http\Livewire\Settings; -use App\Jobs\ProxyContainerStatusJob; +use App\Jobs\ContainerStatusJob; use App\Models\InstanceSettings as ModelsInstanceSettings; use App\Models\Server; use Livewire\Component; @@ -124,7 +124,7 @@ private function setup_instance_fqdn() ]; } $this->save_configuration_to_disk($traefik_dynamic_conf, $file); - dispatch(new ProxyContainerStatusJob($this->server)); + dispatch(new ContainerStatusJob($this->server)); } } diff --git a/app/Jobs/ApplicationContainerStatusJob.php b/app/Jobs/ApplicationContainerStatusJob.php index f1d2ed7c9..ae41a5c41 100644 --- a/app/Jobs/ApplicationContainerStatusJob.php +++ b/app/Jobs/ApplicationContainerStatusJob.php @@ -4,7 +4,6 @@ use App\Models\Application; use App\Models\ApplicationPreview; -use App\Notifications\Application\StatusChanged; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldBeUnique; diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index d224afd10..ddd64b089 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -132,7 +132,9 @@ public function handle(): void $this->deploy(); } } - if ($this->application->fqdn) dispatch(new ProxyContainerStatusJob($this->server)); + if ($this->server->isProxyShouldRun()) { + dispatch(new ContainerStatusJob($this->server)); + } $this->next(ApplicationDeploymentStatus::FINISHED->value); } catch (Exception $e) { ray($e); @@ -267,6 +269,7 @@ private function health_check() "echo 'Rolling update completed.'" ], ); + $this->application->update(['status' => 'running']); break; } $counter++; diff --git a/app/Jobs/ContainerStatusJob.php b/app/Jobs/ContainerStatusJob.php new file mode 100644 index 000000000..f2c82f4bf --- /dev/null +++ b/app/Jobs/ContainerStatusJob.php @@ -0,0 +1,139 @@ +server->uuid)]; + } + + public function uniqueId(): string + { + return $this->server->uuid; + } + + private function checkServerConnection() { + ray("Checking server connection to {$this->server->ip}"); + $uptime = instant_remote_process(['uptime'], $this->server, false); + if (!is_null($uptime)) { + ray('Server is up'); + return true; + } + } + public function handle(): void + { + try { + ray()->clearAll(); + $serverUptimeCheckNumber = 0; + $serverUptimeCheckNumberMax = 5; + while (true) { + if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) { + $this->server->settings()->update(['is_reachable' => false]); + $this->server->team->notify(new Unreachable($this->server)); + return; + } + $result = $this->checkServerConnection(); + if ($result) { + break; + } + $serverUptimeCheckNumber++; + sleep(5); + } + $containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this->server); + $containers = format_docker_command_output_to_json($containers); + $applications = $this->server->applications(); + $databases = $this->server->databases(); + if ($this->server->isProxyShouldRun()) { + $foundProxyContainer = $containers->filter(function ($value, $key) { + return data_get($value, 'Name') === '/coolify-proxy'; + })->first(); + if (!$foundProxyContainer) { + resolve(StartProxy::class)($this->server, false); + $this->server->team->notify(new ContainerRestarted('coolify-proxy', $this->server)); + } + } + foreach ($applications as $application) { + $uuid = data_get($application, 'uuid'); + $foundContainer = $containers->filter(function ($value, $key) use ($uuid) { + return Str::startsWith(data_get($value, 'Name'), "/$uuid"); + })->first(); + + if ($foundContainer) { + $containerStatus = data_get($foundContainer, 'State.Status'); + $databaseStatus = data_get($application, 'status'); + if ($containerStatus !== $databaseStatus) { + $application->update(['status' => $containerStatus]); + } + } else { + $databaseStatus = data_get($application, 'status'); + if ($databaseStatus !== 'exited') { + $application->update(['status' => 'exited']); + $name = data_get($application, 'name'); + $fqdn = data_get($application, 'fqdn'); + $containerName = $name ? "$name ($fqdn)" : $fqdn; + $project = data_get($application, 'environment.project'); + $environment = data_get($application, 'environment'); + $url = base_url() . '/project/' . $project->uuid . "/" . $environment->name . "/application/" . $application->uuid; + $this->server->team->notify(new ContainerStopped($containerName, $this->server, $url)); + } + } + } + foreach ($databases as $database) { + $uuid = data_get($database, 'uuid'); + $foundContainer = $containers->filter(function ($value, $key) use ($uuid) { + return Str::startsWith(data_get($value, 'Name'), "/$uuid"); + })->first(); + + if ($foundContainer) { + $containerStatus = data_get($foundContainer, 'State.Status'); + $databaseStatus = data_get($database, 'status'); + if ($containerStatus !== $databaseStatus) { + $database->update(['status' => $containerStatus]); + } + } else { + $databaseStatus = data_get($database, 'status'); + if ($databaseStatus !== 'exited') { + $database->update(['status' => 'exited']); + $name = data_get($database, 'name'); + $containerName = $name; + $project = data_get($database, 'environment.project'); + $environment = data_get($database, 'environment'); + $url = base_url() . '/project/' . $project->uuid . "/" . $environment->name . "/database/" . $database->uuid; + $this->server->team->notify(new ContainerStopped($containerName, $this->server, $url)); + } + } + } + // TODO Monitor other containers not managed by Coolify + } catch (\Throwable $e) { + send_internal_notification('ContainerStatusJob failed with: ' . $e->getMessage()); + ray($e->getMessage()); + throw $e; + } + } +} diff --git a/app/Jobs/ServerDetailsCheckJob.php b/app/Jobs/ServerDetailsCheckJob.php deleted file mode 100644 index 8a68c66ae..000000000 --- a/app/Jobs/ServerDetailsCheckJob.php +++ /dev/null @@ -1,80 +0,0 @@ -server->uuid)]; - } - - public function uniqueId(): string - { - return $this->server->uuid; - } - - public function handle(): void - { - try { - ray()->clearAll(); - $containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this->server); - $containers = format_docker_command_output_to_json($containers); - $applications = $this->server->applications(); - // ray($applications); - // ray(format_docker_command_output_to_json($containers)); - foreach ($applications as $application) { - $uuid = data_get($application, 'uuid'); - $foundContainer = $containers->filter(function ($value, $key) use ($uuid) { - $image = data_get($value, 'Config.Image'); - return Str::startsWith($image, $uuid); - })->first(); - - if ($foundContainer) { - $containerStatus = data_get($foundContainer, 'State.Status'); - $databaseStatus = data_get($application, 'status'); - ray($containerStatus, $databaseStatus); - if ($containerStatus !== $databaseStatus) { - // $application->update(['status' => $containerStatus]); - } - } - } - // foreach ($containers as $container) { - // $labels = format_docker_labels_to_json(data_get($container,'Config.Labels')); - // $foundLabel = $labels->filter(fn ($value, $key) => Str::startsWith($key, 'coolify.applicationId')); - // if ($foundLabel->count() > 0) { - // $appFound = $applications->where('id', $foundLabel['coolify.applicationId'])->first(); - // if ($appFound) { - // $containerStatus = data_get($container, 'State.Status'); - // $databaseStatus = data_get($appFound, 'status'); - // ray($containerStatus, $databaseStatus); - // } - // } - // } - } catch (\Throwable $e) { - // send_internal_notification('ServerDetailsCheckJob failed with: ' . $e->getMessage()); - ray($e->getMessage()); - throw $e; - } - } -} diff --git a/app/Notifications/Application/DeploymentSuccess.php b/app/Notifications/Application/DeploymentSuccess.php index 0705e11ae..ea789b69a 100644 --- a/app/Notifications/Application/DeploymentSuccess.php +++ b/app/Notifications/Application/DeploymentSuccess.php @@ -53,7 +53,7 @@ public function toMail(): MailMessage $pull_request_id = data_get($this->preview, 'pull_request_id', 0); $fqdn = $this->fqdn; if ($pull_request_id === 0) { - $mail->subject("✅New version is deployed of {$this->application_name}"); + $mail->subject("✅ New version is deployed of {$this->application_name}"); } else { $fqdn = $this->preview->fqdn; $mail->subject("✅ Pull request #{$pull_request_id} of {$this->application_name} deployed successfully"); diff --git a/app/Notifications/Container/ContainerRestarted.php b/app/Notifications/Container/ContainerRestarted.php new file mode 100644 index 000000000..480fe2cb0 --- /dev/null +++ b/app/Notifications/Container/ContainerRestarted.php @@ -0,0 +1,62 @@ +subject("✅ Container ({$this->name}) has been restarted automatically on {$this->server->name}"); + $mail->view('emails.container-restarted', [ + 'containerName' => $this->name, + 'serverName' => $this->server->name, + 'url' => $this->url , + ]); + return $mail; + } + + public function toDiscord(): string + { + $message = "✅ Container ({$this->name}) has been restarted automatically on {$this->server->name}"; + return $message; + } + public function toTelegram(): array + { + $message = "✅ Container ({$this->name}) has been restarted automatically on {$this->server->name}"; + $payload = [ + "message" => $message, + ]; + if ($this->url) { + $payload['buttons'] = [ + [ + [ + "text" => "Check Proxy in Coolify", + "url" => $this->url + ] + ] + ]; + }; + return $payload; + } +} diff --git a/app/Notifications/Container/ContainerStopped.php b/app/Notifications/Container/ContainerStopped.php new file mode 100644 index 000000000..eec287751 --- /dev/null +++ b/app/Notifications/Container/ContainerStopped.php @@ -0,0 +1,61 @@ +subject("⛔ Container ({$this->name}) has been stopped on {$this->server->name}"); + $mail->view('emails.container-stopped', [ + 'containerName' => $this->name, + 'serverName' => $this->server->name, + 'url' => $this->url, + ]); + return $mail; + } + + public function toDiscord(): string + { + $message = "⛔ Container ({$this->name}) has been stopped on {$this->server->name}"; + return $message; + } + public function toTelegram(): array + { + $message = "⛔ Container ({$this->name}) has been stopped on {$this->server->name}"; + $payload = [ + "message" => $message, + ]; + if ($this->url) { + $payload['buttons'] = [ + [ + [ + "text" => "Open Application in Coolify", + "url" => $this->url + ] + ] + ]; + } + return $payload; + } +} diff --git a/app/Notifications/Server/NotReachable.php b/app/Notifications/Server/NotReachable.php deleted file mode 100644 index 672636c57..000000000 --- a/app/Notifications/Server/NotReachable.php +++ /dev/null @@ -1,53 +0,0 @@ -fqdn; - $mail->subject("⛔ Server '{$this->server->name}' is unreachable"); - // $mail->view('emails.application-status-changes', [ - // 'name' => $this->application_name, - // 'fqdn' => $fqdn, - // 'application_url' => $this->application_url, - // ]); - return $mail; - } - - public function toDiscord(): string - { - $message = '⛔ Server \'' . $this->server->name . '\' is unreachable (could be a temporary issue). If you receive this more than twice in a row, please check your server.'; - return $message; - } - public function toTelegram(): array - { - return [ - "message" => '⛔ Server \'' . $this->server->name . '\' is unreachable (could be a temporary issue). If you receive this more than twice in a row, please check your server.' - ]; - } -} diff --git a/app/Notifications/Server/Unreachable.php b/app/Notifications/Server/Unreachable.php new file mode 100644 index 000000000..ec9c11d11 --- /dev/null +++ b/app/Notifications/Server/Unreachable.php @@ -0,0 +1,47 @@ +subject("⛔ Server ({$this->server->name}) is unreachable after trying to connect to it 5 times"); + $mail->view('emails.server-lost-connection', [ + 'name' => $this->server->name, + ]); + return $mail; + } + + public function toDiscord(): string + { + $message = "⛔ Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: You have to validate your server again after you fix the issue."; + return $message; + } + public function toTelegram(): array + { + return [ + "message" => "⛔ Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: You have to validate your server again after you fix the issue." + ]; + } +} diff --git a/resources/views/emails/container-restarted.blade.php b/resources/views/emails/container-restarted.blade.php new file mode 100644 index 000000000..e3ec3371c --- /dev/null +++ b/resources/views/emails/container-restarted.blade.php @@ -0,0 +1,11 @@ + + +Container ({{ $containerName }}) has been restarted automatically on {{$serverName}}, because it was stopped unexpected. + +@if ($containerName === 'coolify-proxy') +Coolify Proxy should run on your server as you have FQDN set up in one of your resources. If you don't want to use Coolify Proxy, please remove FQDN from your resources. + +Note: The proxy should not stop unexpectedly, so please check what is going on your server. +@endif + + diff --git a/resources/views/emails/container-stopped.blade.php b/resources/views/emails/container-stopped.blade.php new file mode 100644 index 000000000..77e8a2597 --- /dev/null +++ b/resources/views/emails/container-stopped.blade.php @@ -0,0 +1,9 @@ + + +Container {{ $containerName }} has been stopped unexpected on {{$serverName}}. + +@if ($url) +Please check what is going on [here]({{ $url }}). +@endif + + diff --git a/resources/views/emails/server-lost-connection.blade.php b/resources/views/emails/server-lost-connection.blade.php index 67aa0816a..83b5e2db5 100644 --- a/resources/views/emails/server-lost-connection.blade.php +++ b/resources/views/emails/server-lost-connection.blade.php @@ -1,5 +1,10 @@ -Coolify Cloud cannot connect to your server ({{$name}}). Please check your server and make sure it is running. + +Coolify cannot connect to your server ({{$name}}). Please check your server and make sure it is running. + +All automations & integrations are turned off! + +IMPORTANT: You have to validate your server again after you fix the issue. If you have any questions, please contact us.