From 9e1a7d5d9ab46d3c79261c1b13791a41101160cd Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 7 Feb 2024 14:55:06 +0100 Subject: [PATCH] feat: multi deployments --- app/Actions/Application/StopApplication.php | 15 +- .../Application/StopApplicationOneServer.php | 38 ++ app/Actions/Shared/ComplexStatusCheck.php | 55 +++ app/Console/Kernel.php | 3 +- app/Jobs/ApplicationDeploymentJob.php | 353 ++++++++++-------- app/Jobs/ContainerStatusJob.php | 32 +- app/Jobs/ServerStatusJob.php | 9 + .../Project/Application/DeploymentNavbar.php | 4 +- app/Livewire/Project/Application/Heading.php | 50 ++- app/Livewire/Project/Shared/Destination.php | 41 +- app/Models/Application.php | 74 +++- app/Models/StandaloneMariadb.php | 36 +- app/Models/StandaloneMongodb.php | 36 +- app/Models/StandaloneMysql.php | 36 +- app/Models/StandalonePostgresql.php | 36 +- app/Models/StandaloneRedis.php | 36 +- ..._06_132748_add_additional_destinations.php | 1 + .../components/databases/navbar.blade.php | 2 +- .../views/components/status/index.blade.php | 4 +- .../components/status/restarting.blade.php | 9 +- .../views/components/status/running.blade.php | 9 +- .../views/components/status/stopped.blade.php | 4 +- .../application/configuration.blade.php | 6 +- .../project/application/general.blade.php | 2 +- .../project/application/heading.blade.php | 2 +- .../project/database/configuration.blade.php | 10 +- .../livewire/project/resource/index.blade.php | 5 +- .../project/shared/destination.blade.php | 96 +++-- 28 files changed, 714 insertions(+), 290 deletions(-) create mode 100644 app/Actions/Application/StopApplicationOneServer.php create mode 100644 app/Actions/Shared/ComplexStatusCheck.php diff --git a/app/Actions/Application/StopApplication.php b/app/Actions/Application/StopApplication.php index 3cde4a80b..601b8e991 100644 --- a/app/Actions/Application/StopApplication.php +++ b/app/Actions/Application/StopApplication.php @@ -19,8 +19,8 @@ public function handle(Application $application) $servers = collect([]); $servers->push($application->destination->server); - $application->additional_networks->map(function ($network) use ($servers) { - $servers->push($network->server); + $application->additional_servers->map(function ($server) use ($servers) { + $servers->push($server); }); foreach ($servers as $server) { if (!$server->isFunctional()) { @@ -37,18 +37,7 @@ public function handle(Application $application) ); } } - // $application->environment->project->team->notify(new StatusChanged($application)); } } - - // // Delete Preview Deployments - // $previewDeployments = $application->previews; - // foreach ($previewDeployments as $previewDeployment) { - // $containers = getCurrentApplicationContainerStatus($server, $application->id, $previewDeployment->pull_request_id); - // foreach ($containers as $container) { - // $name = str_replace('/', '', $container['Names']); - // instant_remote_process(["docker rm -f $name"], $application->destination->server, throwError: false); - // } - // } } } diff --git a/app/Actions/Application/StopApplicationOneServer.php b/app/Actions/Application/StopApplicationOneServer.php new file mode 100644 index 000000000..1945a94bd --- /dev/null +++ b/app/Actions/Application/StopApplicationOneServer.php @@ -0,0 +1,38 @@ +destination->server->isSwarm()) { + return; + } + if (!$server->isFunctional()) { + return 'Server is not functional'; + } + try { + $containers = getCurrentApplicationContainerStatus($server, $application->id, 0); + if ($containers->count() > 0) { + foreach ($containers as $container) { + $containerName = data_get($container, 'Names'); + if ($containerName) { + instant_remote_process( + ["docker rm -f {$containerName}"], + $server + ); + } + } + } + } catch (\Exception $e) { + ray($e->getMessage()); + return $e->getMessage(); + } + } +} diff --git a/app/Actions/Shared/ComplexStatusCheck.php b/app/Actions/Shared/ComplexStatusCheck.php new file mode 100644 index 000000000..7987257f2 --- /dev/null +++ b/app/Actions/Shared/ComplexStatusCheck.php @@ -0,0 +1,55 @@ +additional_servers; + $servers->push($application->destination->server); + foreach ($servers as $server) { + $is_main_server = $application->destination->server->id === $server->id; + if (!$server->isFunctional()) { + if ($is_main_server) { + $application->update(['status' => 'exited:unhealthy']); + continue; + } else { + $application->additional_servers()->updateExistingPivot($server->id, ['status' => 'exited:unhealthy']); + continue; + } + } + $container = instant_remote_process(["docker container inspect $(docker container ls -q --filter 'label=coolify.applicationId={$application->id}' --filter 'label=coolify.pullRequestId=0') --format '{{json .}}'"], $server, false); + $container = format_docker_command_output_to_json($container); + if ($container->count() === 1) { + $container = $container->first(); + $containerStatus = data_get($container, 'State.Status'); + $containerHealth = data_get($container, 'State.Health.Status', 'unhealthy'); + if ($is_main_server) { + $statusFromDb = $application->status; + if ($statusFromDb !== $containerStatus) { + $application->update(['status' => "$containerStatus:$containerHealth"]); + } + } else { + $additional_server = $application->additional_servers()->wherePivot('server_id', $server->id); + $statusFromDb = $additional_server->first()->pivot->status; + if ($statusFromDb !== $containerStatus) { + $additional_server->updateExistingPivot($server->id, ['status' => "$containerStatus:$containerHealth"]); + } + } + } else { + if ($is_main_server) { + $application->update(['status' => 'exited:unhealthy']); + continue; + } else { + $application->additional_servers()->updateExistingPivot($server->id, ['status' => 'exited:unhealthy']); + continue; + } + } + } + } +} diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index c56feb902..37266ca5d 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -4,6 +4,7 @@ use App\Jobs\CheckLogDrainContainerJob; use App\Jobs\CleanupInstanceStuffsJob; +use App\Jobs\ComplexContainerStatusJob; use App\Jobs\DatabaseBackupJob; use App\Jobs\ScheduledTaskJob; use App\Jobs\InstanceAutoUpdateJob; @@ -91,7 +92,6 @@ private function check_scheduled_backups($schedule) { $scheduled_backups = ScheduledDatabaseBackup::all(); if ($scheduled_backups->isEmpty()) { - ray('no scheduled backups'); return; } foreach ($scheduled_backups as $scheduled_backup) { @@ -117,7 +117,6 @@ private function check_scheduled_tasks($schedule) { $scheduled_tasks = ScheduledTask::all(); if ($scheduled_tasks->isEmpty()) { - ray('no scheduled tasks'); return; } foreach ($scheduled_tasks as $scheduled_task) { diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index 4edf1b537..fa828acbe 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -248,13 +248,14 @@ public function handle(): void dispatch(new ContainerStatusJob($this->server)); } // Otherwise built image needs to be pushed before from the build server. - if (!$this->use_build_server) { - if ($this->application->additional_networks->count() > 0) { - $this->push_to_docker_registry(forceFail: true); - } else { - $this->push_to_docker_registry(); - } - } + // ray($this->use_build_server); + // if (!$this->use_build_server) { + // if ($this->application->additional_servers->count() > 0) { + // $this->push_to_docker_registry(forceFail: true); + // } else { + // $this->push_to_docker_registry(); + // } + // } $this->next(ApplicationDeploymentStatus::FINISHED->value); if ($this->pull_request_id !== 0) { if ($this->application->is_github_based()) { @@ -292,155 +293,6 @@ public function handle(): void ApplicationStatusChanged::dispatch(data_get($this->application, 'environment.project.team.id')); } } - private function write_deployment_configurations() - { - if (isset($this->docker_compose_base64)) { - if ($this->use_build_server) { - $this->server = $this->original_server; - } - $readme = generate_readme_file($this->application->name, $this->application_deployment_queue->updated_at); - $composeFileName = "$this->configuration_dir/docker-compose.yml"; - if ($this->pull_request_id !== 0) { - $composeFileName = "$this->configuration_dir/docker-compose-pr-{$this->pull_request_id}.yml"; - } - $this->execute_remote_command( - [ - "mkdir -p $this->configuration_dir" - ], - [ - "echo '{$this->docker_compose_base64}' | base64 -d > $composeFileName", - ], - [ - "echo '{$readme}' > $this->configuration_dir/README.md", - ] - ); - if ($this->use_build_server) { - $this->server = $this->build_server; - } - } - } - private function push_to_docker_registry($forceFail = false) - { - if ( - $this->application->docker_registry_image_name && - $this->application->build_pack !== 'dockerimage' && - !$this->application->destination->server->isSwarm() && - !$this->restart_only && - !(str($this->saved_outputs->get('local_image_found'))->isNotEmpty() && !$this->application->isConfigurationChanged()) - ) { - try { - instant_remote_process(["docker images --format '{{json .}}' {$this->production_image_name}"], $this->server); - $this->application_deployment_queue->addLogEntry("----------------------------------------"); - $this->application_deployment_queue->addLogEntry("Pushing image to docker registry ({$this->production_image_name})."); - $this->execute_remote_command( - [ - executeInDocker($this->deployment_uuid, "docker push {$this->production_image_name}"), 'hidden' => true - ], - ); - if ($this->application->docker_registry_image_tag) { - // Tag image with latest - $this->application_deployment_queue->addLogEntry("Tagging and pushing image with latest tag."); - $this->execute_remote_command( - [ - executeInDocker($this->deployment_uuid, "docker tag {$this->production_image_name} {$this->application->docker_registry_image_name}:{$this->application->docker_registry_image_tag}"), 'ignore_errors' => true, 'hidden' => true - ], - [ - executeInDocker($this->deployment_uuid, "docker push {$this->application->docker_registry_image_name}:{$this->application->docker_registry_image_tag}"), 'ignore_errors' => true, 'hidden' => true - ], - ); - } - $this->application_deployment_queue->addLogEntry("Image pushed to docker registry."); - } catch (Exception $e) { - $this->application_deployment_queue->addLogEntry("Failed to push image to docker registry. Please check debug logs for more information."); - if ($forceFail) { - throw new RuntimeException($e->getMessage(), 69420); - } - ray($e); - } - } - } - private function generate_image_names() - { - if ($this->application->dockerfile) { - if ($this->application->docker_registry_image_name) { - $this->build_image_name = Str::lower("{$this->application->docker_registry_image_name}:build"); - $this->production_image_name = Str::lower("{$this->application->docker_registry_image_name}:latest"); - } else { - $this->build_image_name = Str::lower("{$this->application->uuid}:build"); - $this->production_image_name = Str::lower("{$this->application->uuid}:latest"); - } - } else if ($this->application->build_pack === 'dockerimage') { - $this->production_image_name = Str::lower("{$this->dockerImage}:{$this->dockerImageTag}"); - } else if ($this->pull_request_id !== 0) { - if ($this->application->docker_registry_image_name) { - $this->build_image_name = Str::lower("{$this->application->docker_registry_image_name}:pr-{$this->pull_request_id}-build"); - $this->production_image_name = Str::lower("{$this->application->docker_registry_image_name}:pr-{$this->pull_request_id}"); - } else { - $this->build_image_name = Str::lower("{$this->application->uuid}:pr-{$this->pull_request_id}-build"); - $this->production_image_name = Str::lower("{$this->application->uuid}:pr-{$this->pull_request_id}"); - } - } else { - $this->dockerImageTag = str($this->commit)->substr(0, 128); - if ($this->application->docker_registry_image_name) { - $this->build_image_name = Str::lower("{$this->application->docker_registry_image_name}:{$this->dockerImageTag}-build"); - $this->production_image_name = Str::lower("{$this->application->docker_registry_image_name}:{$this->dockerImageTag}"); - } else { - $this->build_image_name = Str::lower("{$this->application->uuid}:{$this->dockerImageTag}-build"); - $this->production_image_name = Str::lower("{$this->application->uuid}:{$this->dockerImageTag}"); - } - } - } - private function just_restart() - { - $this->application_deployment_queue->addLogEntry("Starting deployment of {$this->customRepository}:{$this->application->git_branch} to {$this->server->name}."); - $this->prepare_builder_image(); - $this->check_git_if_build_needed(); - $this->set_base_dir(); - $this->generate_image_names(); - $this->check_image_locally_or_remotely(); - if (str($this->saved_outputs->get('local_image_found'))->isNotEmpty()) { - $this->application_deployment_queue->addLogEntry("Image found ({$this->production_image_name}) with the same Git Commit SHA. Restarting container."); - $this->create_workdir(); - $this->generate_compose_file(); - $this->rolling_update(); - return; - } - throw new RuntimeException('Cannot find image anywhere. Please redeploy the application.'); - } - private function check_image_locally_or_remotely() - { - $this->execute_remote_command([ - "docker images -q {$this->production_image_name} 2>/dev/null", "hidden" => true, "save" => "local_image_found" - ]); - if (str($this->saved_outputs->get('local_image_found'))->isEmpty() && $this->application->docker_registry_image_name) { - $this->execute_remote_command([ - "docker pull {$this->production_image_name} 2>/dev/null", "ignore_errors" => true, "hidden" => true - ]); - $this->execute_remote_command([ - "docker images -q {$this->production_image_name} 2>/dev/null", "hidden" => true, "save" => "local_image_found" - ]); - } - } - private function save_environment_variables() - { - $envs = collect([]); - if ($this->pull_request_id !== 0) { - foreach ($this->application->environment_variables_preview as $env) { - $envs->push($env->key . '=' . $env->real_value); - } - } else { - foreach ($this->application->environment_variables as $env) { - $envs->push($env->key . '=' . $env->real_value); - } - } - $envs_base64 = base64_encode($envs->implode("\n")); - $this->execute_remote_command( - [ - executeInDocker($this->deployment_uuid, "echo '$envs_base64' | base64 -d > $this->workdir/.env") - ], - ); - } - private function deploy_simple_dockerfile() { if ($this->use_build_server) { @@ -461,6 +313,7 @@ private function deploy_simple_dockerfile() $this->create_workdir(); $this->application_deployment_queue->addLogEntry("No configuration changed & image found ({$this->production_image_name}) with the same Git Commit SHA. Build step skipped."); $this->generate_compose_file(); + $this->push_to_docker_registry(); $this->rolling_update(); return; } @@ -469,9 +322,9 @@ private function deploy_simple_dockerfile() $this->generate_build_env_variables(); $this->add_build_env_variables_to_dockerfile(); $this->build_image(); + $this->push_to_docker_registry(); $this->rolling_update(); } - private function deploy_dockerimage_buildpack() { $this->dockerImage = $this->application->docker_registry_image_name; @@ -576,6 +429,7 @@ private function deploy_dockerfile_buildpack() $this->create_workdir(); $this->application_deployment_queue->addLogEntry("No configuration changed & image found ({$this->production_image_name}) with the same Git Commit SHA. Build step skipped."); $this->generate_compose_file(); + $this->push_to_docker_registry(); $this->rolling_update(); return; } @@ -586,6 +440,7 @@ private function deploy_dockerfile_buildpack() $this->generate_build_env_variables(); $this->add_build_env_variables_to_dockerfile(); $this->build_image(); + $this->push_to_docker_registry(); $this->rolling_update(); } private function deploy_nixpacks_buildpack() @@ -604,6 +459,8 @@ private function deploy_nixpacks_buildpack() $this->create_workdir(); $this->application_deployment_queue->addLogEntry("No configuration changed & image found ({$this->production_image_name}) with the same Git Commit SHA. Build step skipped."); $this->generate_compose_file(); + ray('pushing to docker registry'); + $this->push_to_docker_registry(); $this->rolling_update(); return; } @@ -616,8 +473,8 @@ private function deploy_nixpacks_buildpack() $this->generate_nixpacks_confs(); $this->generate_compose_file(); $this->generate_build_env_variables(); - // $this->add_build_env_variables_to_dockerfile(); $this->build_image(); + $this->push_to_docker_registry(); $this->rolling_update(); } private function deploy_static_buildpack() @@ -636,17 +493,191 @@ private function deploy_static_buildpack() $this->create_workdir(); $this->application_deployment_queue->addLogEntry("No configuration changed & image found ({$this->production_image_name}) with the same Git Commit SHA. Build step skipped."); $this->generate_compose_file(); + $this->push_to_docker_registry(); $this->rolling_update(); return; } } $this->clone_repository(); $this->cleanup_git(); - $this->build_image(); $this->generate_compose_file(); + $this->build_image(); + $this->push_to_docker_registry(); $this->rolling_update(); } + private function write_deployment_configurations() + { + if (isset($this->docker_compose_base64)) { + if ($this->use_build_server) { + $this->server = $this->original_server; + } + $readme = generate_readme_file($this->application->name, $this->application_deployment_queue->updated_at); + $composeFileName = "$this->configuration_dir/docker-compose.yml"; + if ($this->pull_request_id !== 0) { + $composeFileName = "$this->configuration_dir/docker-compose-pr-{$this->pull_request_id}.yml"; + } + $this->execute_remote_command( + [ + "mkdir -p $this->configuration_dir" + ], + [ + "echo '{$this->docker_compose_base64}' | base64 -d > $composeFileName", + ], + [ + "echo '{$readme}' > $this->configuration_dir/README.md", + ] + ); + if ($this->use_build_server) { + $this->server = $this->build_server; + } + } + } + private function push_to_docker_registry() + { + $forceFail = false; + if (str($this->application->docker_registry_image_name)->isEmpty()) { + ray('empty docker_registry_image_name'); + return; + } + if ($this->restart_only) { + ray('restart_only'); + return; + } + if ($this->application->build_pack === 'dockerimage') { + ray('dockerimage'); + return; + } + if ($this->use_build_server) { + ray('use_build_server'); + $forceFail = true; + } + if ($this->server->isSwarm() && $this->build_pack !== 'dockerimage') { + ray('isSwarm'); + $forceFail = true; + } + if ($this->application->additional_servers->count() > 0) { + ray('additional_servers'); + $forceFail = true; + } + if ($this->application->additional_servers()->wherePivot('server_id', $this->server->id)->count() > 0) { + ray('this is an additional_servers, no pushy pushy'); + return; + } + ray('push_to_docker_registry noww: ' . $this->production_image_name); + try { + instant_remote_process(["docker images --format '{{json .}}' {$this->production_image_name}"], $this->server); + $this->application_deployment_queue->addLogEntry("----------------------------------------"); + $this->application_deployment_queue->addLogEntry("Pushing image to docker registry ({$this->production_image_name})."); + $this->execute_remote_command( + [ + executeInDocker($this->deployment_uuid, "docker push {$this->production_image_name}"), 'hidden' => true + ], + ); + if ($this->application->docker_registry_image_tag) { + // Tag image with latest + $this->application_deployment_queue->addLogEntry("Tagging and pushing image with latest tag."); + $this->execute_remote_command( + [ + executeInDocker($this->deployment_uuid, "docker tag {$this->production_image_name} {$this->application->docker_registry_image_name}:{$this->application->docker_registry_image_tag}"), 'ignore_errors' => true, 'hidden' => true + ], + [ + executeInDocker($this->deployment_uuid, "docker push {$this->application->docker_registry_image_name}:{$this->application->docker_registry_image_tag}"), 'ignore_errors' => true, 'hidden' => true + ], + ); + } + $this->application_deployment_queue->addLogEntry("Image pushed to docker registry."); + } catch (Exception $e) { + $this->application_deployment_queue->addLogEntry("Failed to push image to docker registry. Please check debug logs for more information."); + if ($forceFail) { + throw new RuntimeException($e->getMessage(), 69420); + } + ray($e); + } + } + private function generate_image_names() + { + if ($this->application->dockerfile) { + if ($this->application->docker_registry_image_name) { + $this->build_image_name = Str::lower("{$this->application->docker_registry_image_name}:build"); + $this->production_image_name = Str::lower("{$this->application->docker_registry_image_name}:latest"); + } else { + $this->build_image_name = Str::lower("{$this->application->uuid}:build"); + $this->production_image_name = Str::lower("{$this->application->uuid}:latest"); + } + } else if ($this->application->build_pack === 'dockerimage') { + $this->production_image_name = Str::lower("{$this->dockerImage}:{$this->dockerImageTag}"); + } else if ($this->pull_request_id !== 0) { + if ($this->application->docker_registry_image_name) { + $this->build_image_name = Str::lower("{$this->application->docker_registry_image_name}:pr-{$this->pull_request_id}-build"); + $this->production_image_name = Str::lower("{$this->application->docker_registry_image_name}:pr-{$this->pull_request_id}"); + } else { + $this->build_image_name = Str::lower("{$this->application->uuid}:pr-{$this->pull_request_id}-build"); + $this->production_image_name = Str::lower("{$this->application->uuid}:pr-{$this->pull_request_id}"); + } + } else { + $this->dockerImageTag = str($this->commit)->substr(0, 128); + if ($this->application->docker_registry_image_name) { + $this->build_image_name = Str::lower("{$this->application->docker_registry_image_name}:{$this->dockerImageTag}-build"); + $this->production_image_name = Str::lower("{$this->application->docker_registry_image_name}:{$this->dockerImageTag}"); + } else { + $this->build_image_name = Str::lower("{$this->application->uuid}:{$this->dockerImageTag}-build"); + $this->production_image_name = Str::lower("{$this->application->uuid}:{$this->dockerImageTag}"); + } + } + } + private function just_restart() + { + $this->application_deployment_queue->addLogEntry("Starting deployment of {$this->customRepository}:{$this->application->git_branch} to {$this->server->name}."); + $this->prepare_builder_image(); + $this->check_git_if_build_needed(); + $this->set_base_dir(); + $this->generate_image_names(); + $this->check_image_locally_or_remotely(); + if (str($this->saved_outputs->get('local_image_found'))->isNotEmpty()) { + $this->application_deployment_queue->addLogEntry("Image found ({$this->production_image_name}) with the same Git Commit SHA. Restarting container."); + $this->create_workdir(); + $this->generate_compose_file(); + $this->rolling_update(); + return; + } + throw new RuntimeException('Cannot find image anywhere. Please redeploy the application.'); + } + private function check_image_locally_or_remotely() + { + $this->execute_remote_command([ + "docker images -q {$this->production_image_name} 2>/dev/null", "hidden" => true, "save" => "local_image_found" + ]); + if (str($this->saved_outputs->get('local_image_found'))->isEmpty() && $this->application->docker_registry_image_name) { + $this->execute_remote_command([ + "docker pull {$this->production_image_name} 2>/dev/null", "ignore_errors" => true, "hidden" => true + ]); + $this->execute_remote_command([ + "docker images -q {$this->production_image_name} 2>/dev/null", "hidden" => true, "save" => "local_image_found" + ]); + } + } + private function save_environment_variables() + { + $envs = collect([]); + if ($this->pull_request_id !== 0) { + foreach ($this->application->environment_variables_preview as $env) { + $envs->push($env->key . '=' . $env->real_value); + } + } else { + foreach ($this->application->environment_variables as $env) { + $envs->push($env->key . '=' . $env->real_value); + } + } + $envs_base64 = base64_encode($envs->implode("\n")); + $this->execute_remote_command( + [ + executeInDocker($this->deployment_uuid, "echo '$envs_base64' | base64 -d > $this->workdir/.env") + ], + ); + } + + private function framework_based_notification() { // Laravel old env variables @@ -664,9 +695,6 @@ private function framework_based_notification() private function rolling_update() { if ($this->server->isSwarm()) { - if ($this->build_pack !== 'dockerimage') { - $this->push_to_docker_registry(forceFail: true); - } $this->application_deployment_queue->addLogEntry("Rolling update started."); $this->execute_remote_command( [ @@ -676,7 +704,6 @@ private function rolling_update() $this->application_deployment_queue->addLogEntry("Rolling update completed."); } else { if ($this->use_build_server) { - $this->push_to_docker_registry(forceFail: true); $this->write_deployment_configurations(); $this->server = $this->original_server; } diff --git a/app/Jobs/ContainerStatusJob.php b/app/Jobs/ContainerStatusJob.php index 14a90604f..1612e2191 100644 --- a/app/Jobs/ContainerStatusJob.php +++ b/app/Jobs/ContainerStatusJob.php @@ -5,6 +5,7 @@ use App\Actions\Database\StartDatabaseProxy; use App\Actions\Proxy\CheckProxy; use App\Actions\Proxy\StartProxy; +use App\Actions\Shared\ComplexStatusCheck; use App\Models\ApplicationPreview; use App\Models\Server; use App\Notifications\Container\ContainerRestarted; @@ -42,6 +43,19 @@ public function uniqueId(): int public function handle() { + $applications = $this->server->applications(); + foreach ($applications as $application) { + if ($application->additional_servers->count() > 0) { + $is_main_server = $application->destination->server->id === $this->server->id; + if ($is_main_server) { + ComplexStatusCheck::run($application); + $applications = $applications->filter(function ($value, $key) use ($application) { + return $value->id !== $application->id; + }); + } + } + } + if (!$this->server->isFunctional()) { return 'Server is not ready.'; }; @@ -83,7 +97,6 @@ public function handle() }); } } - $applications = $this->server->applications(); $databases = $this->server->databases(); $services = $this->server->services()->get(); $previews = $this->server->previews(); @@ -126,16 +139,7 @@ public function handle() $foundApplications[] = $application->id; $statusFromDb = $application->status; if ($statusFromDb !== $containerStatus) { - // if ($application->additional_networks->count() > 0) { - // } - // if (!str($containerStatus)->contains('running')) { - // $application->update(['status' => 'degraded']); - // } else { - // $application->update(['status' => $containerStatus]); - // } - // } else { $application->update(['status' => $containerStatus]); - // } } } else { //Notify user that this container should not be there. @@ -217,7 +221,7 @@ public function handle() } $exitedServices = $exitedServices->unique('id'); foreach ($exitedServices as $exitedService) { - if ($exitedService->status === 'exited') { + if (str($exitedService->status)->startsWith('exited')) { continue; } $name = data_get($exitedService, 'name'); @@ -239,7 +243,7 @@ public function handle() $notRunningApplications = $applications->pluck('id')->diff($foundApplications); foreach ($notRunningApplications as $applicationId) { $application = $applications->where('id', $applicationId)->first(); - if ($application->status === 'exited') { + if (str($application->status)->startsWith('exited')) { continue; } $application->update(['status' => 'exited']); @@ -264,7 +268,7 @@ public function handle() $notRunningApplicationPreviews = $previews->pluck('id')->diff($foundApplicationPreviews); foreach ($notRunningApplicationPreviews as $previewId) { $preview = $previews->where('id', $previewId)->first(); - if ($preview->status === 'exited') { + if (str($preview->status)->startsWith('exited')) { continue; } $preview->update(['status' => 'exited']); @@ -289,7 +293,7 @@ public function handle() $notRunningDatabases = $databases->pluck('id')->diff($foundDatabases); foreach ($notRunningDatabases as $database) { $database = $databases->where('id', $database)->first(); - if ($database->status === 'exited') { + if (str($database->status)->startsWith('exited')) { continue; } $database->update(['status' => 'exited']); diff --git a/app/Jobs/ServerStatusJob.php b/app/Jobs/ServerStatusJob.php index 0afcb4bb3..b327c3a7c 100644 --- a/app/Jobs/ServerStatusJob.php +++ b/app/Jobs/ServerStatusJob.php @@ -41,6 +41,15 @@ public function handle() throw new \RuntimeException('Server is not ready.'); }; try { + // $this->server->validateConnection(); + // $this->server->validateOS(); + // $docker_installed = $this->server->validateDockerEngine(); + // if (!$docker_installed) { + // $this->server->installDocker(); + // $this->server->validateDockerEngine(); + // } + + // $this->server->validateDockerEngineVersion(); if ($this->server->isFunctional()) { $this->cleanup(notify: false); } diff --git a/app/Livewire/Project/Application/DeploymentNavbar.php b/app/Livewire/Project/Application/DeploymentNavbar.php index 9ba0e7a27..7a397f277 100644 --- a/app/Livewire/Project/Application/DeploymentNavbar.php +++ b/app/Livewire/Project/Application/DeploymentNavbar.php @@ -48,6 +48,8 @@ public function cancel() { try { $kill_command = "docker rm -f {$this->application_deployment_queue->deployment_uuid}"; + $server_id = $this->application_deployment_queue->server_id ?? $this->application->destination->server_id; + $server = Server::find($server_id); if ($this->application_deployment_queue->logs) { $previous_logs = json_decode($this->application_deployment_queue->logs, associative: true, flags: JSON_THROW_ON_ERROR); @@ -63,8 +65,8 @@ public function cancel() $this->application_deployment_queue->update([ 'logs' => json_encode($previous_logs, flags: JSON_THROW_ON_ERROR), ]); - instant_remote_process([$kill_command], $this->server); } + instant_remote_process([$kill_command], $server); } catch (\Throwable $e) { ray($e); return handleError($e, $this); diff --git a/app/Livewire/Project/Application/Heading.php b/app/Livewire/Project/Application/Heading.php index 04320efda..1b19c445a 100644 --- a/app/Livewire/Project/Application/Heading.php +++ b/app/Livewire/Project/Application/Heading.php @@ -3,6 +3,8 @@ namespace App\Livewire\Project\Application; use App\Actions\Application\StopApplication; +use App\Events\ApplicationStatusChanged; +use App\Jobs\ComplexContainerStatusJob; use App\Jobs\ContainerStatusJob; use App\Jobs\ServerStatusJob; use App\Models\Application; @@ -29,20 +31,16 @@ public function mount() public function check_status($showNotification = false) { - $all_servers = collect([]); - $all_servers = $all_servers->push($this->application->destination->server); - $all_servers = $all_servers->merge($this->application->additional_servers); - foreach ($all_servers as $server) { - if ($server->isFunctional()) { - dispatch(new ContainerStatusJob($server)); - $this->application->refresh(); - $this->application->previews->each(function ($preview) { - $preview->refresh(); - }); - } else { - dispatch(new ServerStatusJob($this->application->destination->server)); - } + if ($this->application->destination->server->isFunctional()) { + dispatch(new ContainerStatusJob($this->application->destination->server)); + // $this->application->refresh(); + // $this->application->previews->each(function ($preview) { + // $preview->refresh(); + // }); + } else { + dispatch(new ServerStatusJob($this->application->destination->server)); } + if ($showNotification) $this->dispatch('success', "Application status updated."); } @@ -54,15 +52,19 @@ public function force_deploy_without_cache() public function deploy(bool $force_rebuild = false) { if ($this->application->build_pack === 'dockercompose' && is_null($this->application->docker_compose_raw)) { - $this->dispatch('error', 'Please load a Compose file first.'); + $this->dispatch('error', 'Failed to deploy', 'Please load a Compose file first.'); return; } - if ($this->application->destination->server->isSwarm() && is_null($this->application->docker_registry_image_name)) { - $this->dispatch('error', 'To deploy to a Swarm cluster you must set a Docker image name first.'); + if ($this->application->destination->server->isSwarm() && str($this->application->docker_registry_image_name)->isEmpty()) { + $this->dispatch('error', 'Failed to deploy', 'To deploy to a Swarm cluster you must set a Docker image name first.'); return; } - if (data_get($this->application, 'settings.is_build_server_enabled') && is_null($this->application->docker_registry_image_name)) { - $this->dispatch('error', 'To use a build server you must set a Docker image name first.
More information here: documentation'); + if (data_get($this->application, 'settings.is_build_server_enabled') && str($this->application->docker_registry_image_name)->isEmpty()) { + $this->dispatch('error', 'Failed to deploy', 'To use a build server, you must first set a Docker image.
More information here: documentation'); + return; + } + if ($this->application->additional_servers->count() > 0 && str($this->application->docker_registry_image_name)->isEmpty()) { + $this->dispatch('error', 'Failed to deploy', 'To deploy to more than one server, you must first set a Docker image.
More information here: documentation'); return; } $this->setDeploymentUuid(); @@ -90,10 +92,20 @@ public function stop() StopApplication::run($this->application); $this->application->status = 'exited'; $this->application->save(); - $this->application->refresh(); + if ($this->application->additional_servers->count() > 0) { + $this->application->additional_servers->each(function ($server) { + $server->pivot->status = "exited:unhealthy"; + $server->pivot->save(); + }); + } + ApplicationStatusChanged::dispatch(data_get($this->application, 'environment.project.team.id')); } public function restart() { + if ($this->application->additional_servers->count() > 0 && str($this->application->docker_registry_image_name)->isEmpty()) { + $this->dispatch('error', 'Failed to deploy', 'To deploy to more than one server, you must first set a Docker image.
More information here: documentation'); + return; + } $this->setDeploymentUuid(); queue_application_deployment( application: $this->application, diff --git a/app/Livewire/Project/Shared/Destination.php b/app/Livewire/Project/Shared/Destination.php index 5cc906667..cf5e2632f 100644 --- a/app/Livewire/Project/Shared/Destination.php +++ b/app/Livewire/Project/Shared/Destination.php @@ -2,15 +2,25 @@ namespace App\Livewire\Project\Shared; +use App\Actions\Application\StopApplicationOneServer; +use App\Events\ApplicationStatusChanged; use App\Models\Server; +use App\Models\StandaloneDocker; use Livewire\Component; +use Visus\Cuid2\Cuid2; class Destination extends Component { public $resource; - public $servers = []; public $networks = []; + public function getListeners() + { + $teamId = auth()->user()->currentTeam()->id; + return [ + "echo-private:team.{$teamId},ApplicationStatusChanged" => 'loadData', + ]; + } public function mount() { $this->loadData(); @@ -27,18 +37,45 @@ public function loadData() $this->networks = $this->networks->reject(function ($network) use ($all_networks) { return $all_networks->pluck('id')->contains($network->id); }); + + } + public function redeploy(int $network_id, int $server_id) + { + $deployment_uuid = new Cuid2(7); + $server = Server::find($server_id); + $destination = StandaloneDocker::find($network_id); + queue_application_deployment( + deployment_uuid: $deployment_uuid, + application: $this->resource, + server: $server, + destination: $destination, + no_questions_asked: true, + ); + return redirect()->route('project.application.deployment.show', [ + 'project_uuid' => data_get($this->resource, 'environment.project.uuid'), + 'application_uuid' => data_get($this->resource, 'uuid'), + 'deployment_uuid' => $deployment_uuid, + 'environment_name' => data_get($this->resource, 'environment.name'), + ]); } public function addServer(int $network_id, int $server_id) { $this->resource->additional_networks()->attach($network_id, ['server_id' => $server_id]); $this->resource->load(['additional_networks']); + ApplicationStatusChanged::dispatch(data_get($this->resource, 'environment.project.team.id')); $this->loadData(); - } public function removeServer(int $network_id, int $server_id) { + if ($this->resource->destination->server->id == $server_id) { + $this->dispatch('error', 'You cannot remove this destination server.', 'You are trying to remove the main server.'); + return; + } + $server = Server::find($server_id); + StopApplicationOneServer::run($this->resource, $server); $this->resource->additional_networks()->detach($network_id, ['server_id' => $server_id]); $this->resource->load(['additional_networks']); + ApplicationStatusChanged::dispatch(data_get($this->resource, 'environment.project.team.id')); $this->loadData(); } } diff --git a/app/Models/Application.php b/app/Models/Application.php index 2fdc88d5f..959d06d7f 100644 --- a/app/Models/Application.php +++ b/app/Models/Application.php @@ -15,9 +15,6 @@ class Application extends BaseModel { use SoftDeletes; protected $guarded = []; - // protected $casts = [ - // 'complex_status' => 'json', - // ]; protected static function booted() { static::saving(function ($application) { @@ -58,12 +55,12 @@ protected static function booted() public function additional_servers() { return $this->belongsToMany(Server::class, 'additional_destinations') - ->withPivot('standalone_docker_id'); + ->withPivot('standalone_docker_id', 'status'); } public function additional_networks() { return $this->belongsToMany(StandaloneDocker::class, 'additional_destinations') - ->withPivot('server_id'); + ->withPivot('server_id', 'status'); } public function is_github_based(): bool { @@ -215,22 +212,75 @@ public function portsMappingsArray(): Attribute ); } + public function realStatus() + { + return $this->getRawOriginal('status'); + } public function status(): Attribute { return Attribute::make( set: function ($value) { - if ($this->additional_networks->count() === 0) { - return $value; + if ($this->additional_servers->count() === 0) { + if (str($value)->contains('(')) { + $status = str($value)->before('(')->trim()->value(); + $health = str($value)->after('(')->before(')')->trim()->value() ?? 'unhealthy'; + } else if (str($value)->contains(':')) { + $status = str($value)->before(':')->trim()->value(); + $health = str($value)->after(':')->trim()->value() ?? 'unhealthy'; + } else { + $status = $value; + $health = 'unhealthy'; + } + return "$status:$health"; } else { - return 'complex'; + if (str($value)->contains('(')) { + $status = str($value)->before('(')->trim()->value(); + $health = str($value)->after('(')->before(')')->trim()->value() ?? 'unhealthy'; + } else if (str($value)->contains(':')) { + $status = str($value)->before(':')->trim()->value(); + $health = str($value)->after(':')->trim()->value() ?? 'unhealthy'; + } else { + $status = $value; + $health = 'unhealthy'; + } + return "$status:$health"; } - }, get: function ($value) { - if ($this->additional_networks->count() === 0) { - return $value; + if ($this->additional_servers->count() === 0) { + //running (healthy) + if (str($value)->contains('(')) { + $status = str($value)->before('(')->trim()->value(); + $health = str($value)->after('(')->before(')')->trim()->value() ?? 'unhealthy'; + } else if (str($value)->contains(':')) { + $status = str($value)->before(':')->trim()->value(); + $health = str($value)->after(':')->trim()->value() ?? 'unhealthy'; + } else { + $status = $value; + $health = 'unhealthy'; + } + return "$status:$health"; } else { - return 'complex'; + $complex_status = null; + $complex_health = null; + $complex_status = $main_server_status = str($value)->before(':')->value(); + $complex_health = $main_server_health = str($value)->after(':')->value() ?? 'unhealthy'; + $additional_servers_status = $this->additional_servers->pluck('pivot.status'); + foreach ($additional_servers_status as $status) { + $server_status = str($status)->before(':')->value(); + $server_health = str($status)->after(':')->value() ?? 'unhealthy'; + if ($server_status !== 'running') { + if ($main_server_status !== $server_status) { + $complex_status = 'degraded'; + } + } + if ($server_health !== 'healthy') { + if ($main_server_health !== $server_health) { + $complex_health = 'unhealthy'; + } + } + } + return "$complex_status:$complex_health"; } }, ); diff --git a/app/Models/StandaloneMariadb.php b/app/Models/StandaloneMariadb.php index 18c36203e..1143b018e 100644 --- a/app/Models/StandaloneMariadb.php +++ b/app/Models/StandaloneMariadb.php @@ -43,7 +43,41 @@ protected static function booted() $database->tags()->detach(); }); } - + public function realStatus() + { + return $this->getRawOriginal('status'); + } + public function status(): Attribute + { + return Attribute::make( + set: function ($value) { + if (str($value)->contains('(')) { + $status = str($value)->before('(')->trim()->value(); + $health = str($value)->after('(')->before(')')->trim()->value() ?? 'unhealthy'; + } else if (str($value)->contains(':')) { + $status = str($value)->before(':')->trim()->value(); + $health = str($value)->after(':')->trim()->value() ?? 'unhealthy'; + } else { + $status = $value; + $health = 'unhealthy'; + } + return "$status:$health"; + }, + get: function ($value) { + if (str($value)->contains('(')) { + $status = str($value)->before('(')->trim()->value(); + $health = str($value)->after('(')->before(')')->trim()->value() ?? 'unhealthy'; + } else if (str($value)->contains(':')) { + $status = str($value)->before(':')->trim()->value(); + $health = str($value)->after(':')->trim()->value() ?? 'unhealthy'; + } else { + $status = $value; + $health = 'unhealthy'; + } + return "$status:$health"; + }, + ); + } public function tags() { return $this->morphToMany(Tag::class, 'taggable'); diff --git a/app/Models/StandaloneMongodb.php b/app/Models/StandaloneMongodb.php index 939af0974..610323f74 100644 --- a/app/Models/StandaloneMongodb.php +++ b/app/Models/StandaloneMongodb.php @@ -46,7 +46,41 @@ protected static function booted() $database->tags()->detach(); }); } - + public function realStatus() + { + return $this->getRawOriginal('status'); + } + public function status(): Attribute + { + return Attribute::make( + set: function ($value) { + if (str($value)->contains('(')) { + $status = str($value)->before('(')->trim()->value(); + $health = str($value)->after('(')->before(')')->trim()->value() ?? 'unhealthy'; + } else if (str($value)->contains(':')) { + $status = str($value)->before(':')->trim()->value(); + $health = str($value)->after(':')->trim()->value() ?? 'unhealthy'; + } else { + $status = $value; + $health = 'unhealthy'; + } + return "$status:$health"; + }, + get: function ($value) { + if (str($value)->contains('(')) { + $status = str($value)->before('(')->trim()->value(); + $health = str($value)->after('(')->before(')')->trim()->value() ?? 'unhealthy'; + } else if (str($value)->contains(':')) { + $status = str($value)->before(':')->trim()->value(); + $health = str($value)->after(':')->trim()->value() ?? 'unhealthy'; + } else { + $status = $value; + $health = 'unhealthy'; + } + return "$status:$health"; + }, + ); + } public function tags() { return $this->morphToMany(Tag::class, 'taggable'); diff --git a/app/Models/StandaloneMysql.php b/app/Models/StandaloneMysql.php index 8bcc0d9fe..fa6bbe28f 100644 --- a/app/Models/StandaloneMysql.php +++ b/app/Models/StandaloneMysql.php @@ -43,7 +43,41 @@ protected static function booted() $database->tags()->detach(); }); } - + public function realStatus() + { + return $this->getRawOriginal('status'); + } + public function status(): Attribute + { + return Attribute::make( + set: function ($value) { + if (str($value)->contains('(')) { + $status = str($value)->before('(')->trim()->value(); + $health = str($value)->after('(')->before(')')->trim()->value() ?? 'unhealthy'; + } else if (str($value)->contains(':')) { + $status = str($value)->before(':')->trim()->value(); + $health = str($value)->after(':')->trim()->value() ?? 'unhealthy'; + } else { + $status = $value; + $health = 'unhealthy'; + } + return "$status:$health"; + }, + get: function ($value) { + if (str($value)->contains('(')) { + $status = str($value)->before('(')->trim()->value(); + $health = str($value)->after('(')->before(')')->trim()->value() ?? 'unhealthy'; + } else if (str($value)->contains(':')) { + $status = str($value)->before(':')->trim()->value(); + $health = str($value)->after(':')->trim()->value() ?? 'unhealthy'; + } else { + $status = $value; + $health = 'unhealthy'; + } + return "$status:$health"; + }, + ); + } public function tags() { return $this->morphToMany(Tag::class, 'taggable'); diff --git a/app/Models/StandalonePostgresql.php b/app/Models/StandalonePostgresql.php index fb6ad944d..bcc43843b 100644 --- a/app/Models/StandalonePostgresql.php +++ b/app/Models/StandalonePostgresql.php @@ -43,7 +43,41 @@ protected static function booted() $database->tags()->detach(); }); } - + public function realStatus() + { + return $this->getRawOriginal('status'); + } + public function status(): Attribute + { + return Attribute::make( + set: function ($value) { + if (str($value)->contains('(')) { + $status = str($value)->before('(')->trim()->value(); + $health = str($value)->after('(')->before(')')->trim()->value() ?? 'unhealthy'; + } else if (str($value)->contains(':')) { + $status = str($value)->before(':')->trim()->value(); + $health = str($value)->after(':')->trim()->value() ?? 'unhealthy'; + } else { + $status = $value; + $health = 'unhealthy'; + } + return "$status:$health"; + }, + get: function ($value) { + if (str($value)->contains('(')) { + $status = str($value)->before('(')->trim()->value(); + $health = str($value)->after('(')->before(')')->trim()->value() ?? 'unhealthy'; + } else if (str($value)->contains(':')) { + $status = str($value)->before(':')->trim()->value(); + $health = str($value)->after(':')->trim()->value() ?? 'unhealthy'; + } else { + $status = $value; + $health = 'unhealthy'; + } + return "$status:$health"; + }, + ); + } public function tags() { return $this->morphToMany(Tag::class, 'taggable'); diff --git a/app/Models/StandaloneRedis.php b/app/Models/StandaloneRedis.php index 73fa61a6c..59c53f882 100644 --- a/app/Models/StandaloneRedis.php +++ b/app/Models/StandaloneRedis.php @@ -38,7 +38,41 @@ protected static function booted() $database->tags()->detach(); }); } - + public function realStatus() + { + return $this->getRawOriginal('status'); + } + public function status(): Attribute + { + return Attribute::make( + set: function ($value) { + if (str($value)->contains('(')) { + $status = str($value)->before('(')->trim()->value(); + $health = str($value)->after('(')->before(')')->trim()->value() ?? 'unhealthy'; + } else if (str($value)->contains(':')) { + $status = str($value)->before(':')->trim()->value(); + $health = str($value)->after(':')->trim()->value() ?? 'unhealthy'; + } else { + $status = $value; + $health = 'unhealthy'; + } + return "$status:$health"; + }, + get: function ($value) { + if (str($value)->contains('(')) { + $status = str($value)->before('(')->trim()->value(); + $health = str($value)->after('(')->before(')')->trim()->value() ?? 'unhealthy'; + } else if (str($value)->contains(':')) { + $status = str($value)->before(':')->trim()->value(); + $health = str($value)->after(':')->trim()->value() ?? 'unhealthy'; + } else { + $status = $value; + $health = 'unhealthy'; + } + return "$status:$health"; + }, + ); + } public function tags() { return $this->morphToMany(Tag::class, 'taggable'); diff --git a/database/migrations/2024_02_06_132748_add_additional_destinations.php b/database/migrations/2024_02_06_132748_add_additional_destinations.php index d751fcc40..32e7f5b18 100644 --- a/database/migrations/2024_02_06_132748_add_additional_destinations.php +++ b/database/migrations/2024_02_06_132748_add_additional_destinations.php @@ -15,6 +15,7 @@ public function up(): void $table->id(); $table->foreignId('application_id')->constrained()->onDelete('cascade'); $table->foreignId('server_id')->constrained()->onDelete('cascade'); + $table->string('status')->default('exited'); $table->foreignId('standalone_docker_id')->constrained()->onDelete('cascade'); $table->timestamps(); }); diff --git a/resources/views/components/databases/navbar.blade.php b/resources/views/components/databases/navbar.blade.php index 9f3dfad3b..962c3d22e 100644 --- a/resources/views/components/databases/navbar.blade.php +++ b/resources/views/components/databases/navbar.blade.php @@ -22,7 +22,7 @@ @endif
- @if ($database->status !== 'exited') + @if (!str($database->status)->startsWith('exited'))