From bed959f1cd921085521840d99d7ba013fc3f20c5 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Mon, 21 Aug 2023 18:00:12 +0200 Subject: [PATCH] feat: rolling update --- app/Actions/Database/StartPostgresql.php | 4 +- .../Livewire/Project/Application/General.php | 11 ++- .../Livewire/Project/Application/Heading.php | 30 ++++--- .../Livewire/Project/Application/Previews.php | 11 ++- .../Livewire/Project/Database/Heading.php | 7 +- .../Livewire/Project/New/SimpleDockerfile.php | 4 + app/Http/Livewire/Server/Proxy/Deploy.php | 2 +- app/Jobs/ApplicationContainerStatusJob.php | 53 ++++++++++++ app/Jobs/ApplicationDeploymentJob.php | 86 +++++++++++++++---- app/Jobs/ContainerStatusJob.php | 54 ------------ app/Jobs/DatabaseContainerStatusJob.php | 52 +++++++++++ app/Jobs/ProxyCheckJob.php | 2 +- app/Jobs/ProxyContainerStatusJob.php | 2 +- app/Jobs/ProxyStartJob.php | 2 +- app/Jobs/ResourceStatusJob.php | 13 ++- bootstrap/helpers/constants.php | 1 + bootstrap/helpers/docker.php | 43 ++++++++-- bootstrap/helpers/proxy.php | 2 +- routes/webhooks.php | 2 +- 19 files changed, 263 insertions(+), 118 deletions(-) create mode 100644 app/Jobs/ApplicationContainerStatusJob.php delete mode 100644 app/Jobs/ContainerStatusJob.php create mode 100644 app/Jobs/DatabaseContainerStatusJob.php diff --git a/app/Actions/Database/StartPostgresql.php b/app/Actions/Database/StartPostgresql.php index 1b5d22f44..ba427a93d 100644 --- a/app/Actions/Database/StartPostgresql.php +++ b/app/Actions/Database/StartPostgresql.php @@ -17,7 +17,7 @@ class StartPostgresql public function __invoke(Server $server, StandalonePostgresql $database) { $this->database = $database; - $container_name = generate_container_name($this->database->uuid); + $container_name = $this->database->uuid; $this->configuration_dir = database_configuration_dir() . '/' . $container_name; $this->commands = [ @@ -36,7 +36,7 @@ class StartPostgresql 'image' => $this->database->image, 'container_name' => $container_name, 'environment' => $environment_variables, - 'restart' => 'always', + 'restart' => RESTART_MODE, 'networks' => [ $this->database->destination->network, ], diff --git a/app/Http/Livewire/Project/Application/General.php b/app/Http/Livewire/Project/Application/General.php index 04be74140..ce7fa773b 100644 --- a/app/Http/Livewire/Project/Application/General.php +++ b/app/Http/Livewire/Project/Application/General.php @@ -136,15 +136,18 @@ class General extends Component public function submit() { + ray($this->application); try { $this->validate(); $domains = Str::of($this->application->fqdn)->trim()->explode(',')->map(function ($domain) { return Str::of($domain)->trim()->lower(); }); - $port = get_port_from_dockerfile($this->application->dockerfile); - if ($port) { - $this->application->ports_exposes = $port; + if ($this->application->dockerfile) { + $port = get_port_from_dockerfile($this->application->dockerfile); + if ($port) { + $this->application->ports_exposes = $port; + } } if ($this->application->base_directory && $this->application->base_directory !== '/') { $this->application->base_directory = rtrim($this->application->base_directory, '/'); @@ -152,7 +155,7 @@ class General extends Component if ($this->application->publish_directory && $this->application->publish_directory !== '/') { $this->application->publish_directory = rtrim($this->application->publish_directory, '/'); } - $this->application->fqdn = data_get($domains->implode(','), '', null); + $this->application->fqdn = $domains->implode(','); $this->application->save(); $this->emit('success', 'Application settings updated!'); } catch (\Exception $e) { diff --git a/app/Http/Livewire/Project/Application/Heading.php b/app/Http/Livewire/Project/Application/Heading.php index fdbe5f71e..e71d4765a 100644 --- a/app/Http/Livewire/Project/Application/Heading.php +++ b/app/Http/Livewire/Project/Application/Heading.php @@ -2,7 +2,7 @@ namespace App\Http\Livewire\Project\Application; -use App\Jobs\ContainerStatusJob; +use App\Jobs\ApplicationContainerStatusJob; use App\Models\Application; use App\Notifications\Application\StatusChanged; use Livewire\Component; @@ -22,9 +22,8 @@ class Heading extends Component public function check_status() { - dispatch_sync(new ContainerStatusJob( - resource: $this->application, - container_name: generate_container_name($this->application->uuid), + dispatch_sync(new ApplicationContainerStatusJob( + application: $this->application, )); $this->application->refresh(); } @@ -58,12 +57,21 @@ class Heading extends Component public function stop() { - remote_process( - ["docker rm -f {$this->application->uuid}"], - $this->application->destination->server - ); - $this->application->status = 'stopped'; - $this->application->save(); - $this->application->environment->project->team->notify(new StatusChanged($this->application)); + $containers = getCurrentApplicationContainerStatus($this->application->destination->server, $this->application->id); + if ($containers->count() === 0) { + return; + } + foreach ($containers as $container) { + $containerName = data_get($container, 'Names'); + if ($containerName) { + remote_process( + ["docker rm -f {$containerName}"], + $this->application->destination->server + ); + $this->application->status = 'stopped'; + $this->application->save(); + $this->application->environment->project->team->notify(new StatusChanged($this->application)); + } + } } } diff --git a/app/Http/Livewire/Project/Application/Previews.php b/app/Http/Livewire/Project/Application/Previews.php index b3582fbef..ad3ca5ad9 100644 --- a/app/Http/Livewire/Project/Application/Previews.php +++ b/app/Http/Livewire/Project/Application/Previews.php @@ -2,7 +2,7 @@ namespace App\Http\Livewire\Project\Application; -use App\Jobs\ContainerStatusJob; +use App\Jobs\ApplicationContainerStatusJob; use App\Models\Application; use App\Models\ApplicationPreview; use Illuminate\Support\Collection; @@ -25,10 +25,9 @@ class Previews extends Component public function loadStatus($pull_request_id) { - dispatch(new ContainerStatusJob( - resource: $this->application, - container_name: generate_container_name($this->application->uuid, $pull_request_id), - pull_request_id: $pull_request_id + dispatch(new ApplicationContainerStatusJob( + application: $this->application, + pullRequestId: $pull_request_id )); } @@ -82,7 +81,7 @@ class Previews extends Component public function stop(int $pull_request_id) { try { - $container_name = generate_container_name($this->application->uuid, $pull_request_id); + $container_name = generateApplicationContainerName($this->application->uuid, $pull_request_id); ray('Stopping container: ' . $container_name); instant_remote_process(["docker rm -f $container_name"], $this->application->destination->server, throwError: false); diff --git a/app/Http/Livewire/Project/Database/Heading.php b/app/Http/Livewire/Project/Database/Heading.php index 59f4955a6..0d90fbf96 100644 --- a/app/Http/Livewire/Project/Database/Heading.php +++ b/app/Http/Livewire/Project/Database/Heading.php @@ -3,7 +3,7 @@ namespace App\Http\Livewire\Project\Database; use App\Actions\Database\StartPostgresql; -use App\Jobs\ContainerStatusJob; +use App\Jobs\DatabaseContainerStatusJob; use App\Notifications\Application\StatusChanged; use Livewire\Component; @@ -25,9 +25,8 @@ class Heading extends Component public function check_status() { - dispatch_sync(new ContainerStatusJob( - resource: $this->database, - container_name: generate_container_name($this->database->uuid), + dispatch_sync(new DatabaseContainerStatusJob( + database: $this->database, )); $this->database->refresh(); } diff --git a/app/Http/Livewire/Project/New/SimpleDockerfile.php b/app/Http/Livewire/Project/New/SimpleDockerfile.php index bbc82fcd5..6c4731658 100644 --- a/app/Http/Livewire/Project/New/SimpleDockerfile.php +++ b/app/Http/Livewire/Project/New/SimpleDockerfile.php @@ -59,6 +59,10 @@ CMD ["nginx", "-g", "daemon off;"] 'source_id' => 0, 'source_type' => GithubApp::class ]); + $application->update([ + 'name' => 'dockerfile-' . $application->id + ]); + redirect()->route('project.application.configuration', [ 'application_uuid' => $application->uuid, 'environment_name' => $environment->name, diff --git a/app/Http/Livewire/Server/Proxy/Deploy.php b/app/Http/Livewire/Server/Proxy/Deploy.php index 9ec3898f1..f78763479 100644 --- a/app/Http/Livewire/Server/Proxy/Deploy.php +++ b/app/Http/Livewire/Server/Proxy/Deploy.php @@ -17,7 +17,7 @@ class Deploy extends Component $this->server->proxy->last_applied_settings && $this->server->proxy->last_saved_settings !== $this->server->proxy->last_applied_settings ) { - $this->emit('saveConfiguration', $server); + $this->emit('saveConfiguration', $this->server); } $activity = resolve(StartProxy::class)($this->server); $this->emit('newMonitorActivity', $activity->id); diff --git a/app/Jobs/ApplicationContainerStatusJob.php b/app/Jobs/ApplicationContainerStatusJob.php new file mode 100644 index 000000000..6930fc69a --- /dev/null +++ b/app/Jobs/ApplicationContainerStatusJob.php @@ -0,0 +1,53 @@ +containerName = generateApplicationContainerName($application->uuid, $pullRequestId); + } + + public function uniqueId(): string + { + return $this->containerName; + } + + public function handle(): void + { + try { + $status = getApplicationContainerStatus(application: $this->application); + if ($this->application->status === 'running' && $status !== 'running') { + $this->application->environment->project->team->notify(new StatusChanged($this->application)); + } + + if ($this->pullRequestId !== 0) { + $preview = ApplicationPreview::findPreviewByApplicationAndPullId($this->application->id, $this->pullRequestId); + $preview->status = $status; + $preview->save(); + } else { + $this->application->status = $status; + $this->application->save(); + } + } catch (\Exception $e) { + ray($e->getMessage()); + } + } +} diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index fc0a3a1b7..5a76b26ba 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -50,6 +50,7 @@ class ApplicationDeploymentJob implements ShouldQueue private ApplicationPreview|null $preview = null; private string $container_name; + private string|null $currently_running_container_name = null; private string $workdir; private string $configuration_dir; private string $build_workdir; @@ -86,7 +87,7 @@ class ApplicationDeploymentJob implements ShouldQueue $this->build_workdir = "{$this->workdir}" . rtrim($this->application->base_directory, '/'); $this->is_debug_enabled = $this->application->settings->is_debug_enabled; - $this->container_name = generate_container_name($this->application->uuid, $this->pull_request_id); + $this->container_name = generateApplicationContainerName($this->application->uuid, $this->pull_request_id); $this->private_key_location = save_private_key_for_server($this->server); $this->saved_outputs = collect(); @@ -113,6 +114,10 @@ class ApplicationDeploymentJob implements ShouldQueue public function handle(): void { // ray()->measure(); + $containers = getCurrentApplicationContainerStatus($this->server, $this->application->id); + if ($containers->count() > 0) { + $this->currently_running_container_name = data_get($containers[0], 'Names'); + } $this->application_deployment_queue->update([ 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value, ]); @@ -175,9 +180,9 @@ class ApplicationDeploymentJob implements ShouldQueue $this->generate_build_env_variables(); $this->add_build_env_variables_to_dockerfile(); $this->build_image(); - $this->stop_running_container(); - $this->start_by_compose_file(); + $this->rolling_update(); } + private function deploy() { $this->execute_remote_command( @@ -206,8 +211,7 @@ class ApplicationDeploymentJob implements ShouldQueue "echo 'Docker Image found locally with the same Git Commit SHA {$this->application->uuid}:{$this->commit}. Build step skipped...'" ]); $this->generate_compose_file(); - $this->stop_running_container(); - $this->start_by_compose_file(); + $this->rolling_update(); return; } } @@ -219,8 +223,54 @@ class ApplicationDeploymentJob implements ShouldQueue $this->generate_build_env_variables(); $this->add_build_env_variables_to_dockerfile(); $this->build_image(); - $this->stop_running_container(); + $this->rolling_update(); + } + + private function rolling_update() + { $this->start_by_compose_file(); + $this->health_check(); + $this->stop_running_container(); + } + private function health_check() + { + ray('New container name: ',$this->container_name); + if ($this->container_name) { + $counter = 0; + $this->execute_remote_command( + [ + "echo 'Waiting for health check to pass on the new version of your application.'" + ], + ); + while ($counter < $this->application->health_check_retries) { + $this->execute_remote_command( + [ + "echo 'Attempt {$counter} of {$this->application->health_check_retries}'" + ], + [ + "docker inspect --format='{{json .State.Health.Status}}' {$this->container_name}", + "hidden" => true, + "save" => "health_check" + ], + + ); + $this->execute_remote_command( + [ + "echo 'New application version health check status: {$this->saved_outputs->get('health_check')}'" + ], + ); + if (Str::of($this->saved_outputs->get('health_check'))->contains('healthy')) { + $this->execute_remote_command( + [ + "echo 'Rolling update completed.'" + ], + ); + break; + } + $counter++; + sleep($this->application->health_check_interval); + } + } } private function deploy_pull_request() { @@ -241,8 +291,7 @@ class ApplicationDeploymentJob implements ShouldQueue // $this->generate_build_env_variables(); // $this->add_build_env_variables_to_dockerfile(); $this->build_image(); - $this->stop_running_container(); - $this->start_by_compose_file(); + $this->rolling_update(); } private function prepare_builder_image() @@ -409,7 +458,7 @@ class ApplicationDeploymentJob implements ShouldQueue $this->container_name => [ 'image' => $this->production_image_name, 'container_name' => $this->container_name, - 'restart' => 'always', + 'restart' => RESTART_MODE, 'environment' => $environment_variables, 'labels' => $this->set_labels_for_applications(), 'expose' => $ports, @@ -539,8 +588,8 @@ class ApplicationDeploymentJob implements ShouldQueue $schema = $url->getScheme(); $slug = Str::slug($host . $path); - $http_label = "{$this->application->uuid}-{$slug}-http"; - $https_label = "{$this->application->uuid}-{$slug}-https"; + $http_label = "{$this->container_name}-{$slug}-http"; + $https_label = "{$this->container_name}-{$slug}-https"; if ($schema === 'https') { // Set labels for https @@ -647,23 +696,22 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf"); private function stop_running_container() { - $this->execute_remote_command( - ["echo -n 'Removing old running application.'"], - [$this->execute_in_builder("docker rm -f $this->container_name >/dev/null 2>&1"), "hidden" => true], - ); + if ($this->currently_running_container_name) { + $this->execute_remote_command( + ["echo -n 'Removing old application version.'"], + [$this->execute_in_builder("docker rm -f $this->currently_running_container_name >/dev/null 2>&1"), "hidden" => true], + ); + } } private function start_by_compose_file() { $this->execute_remote_command( - ["echo -n 'Starting new application... '"], + ["echo -n 'Rolling update started.'"], [$this->execute_in_builder("docker compose --project-directory {$this->workdir} up -d >/dev/null"), "hidden" => true], - ["echo 'Done. 🎉'"], ); } - - private function generate_build_env_variables() { $this->build_args = collect(["--build-arg SOURCE_COMMIT={$this->commit}"]); diff --git a/app/Jobs/ContainerStatusJob.php b/app/Jobs/ContainerStatusJob.php deleted file mode 100644 index f2be8b4d7..000000000 --- a/app/Jobs/ContainerStatusJob.php +++ /dev/null @@ -1,54 +0,0 @@ -resource = $resource; - $this->container_name = $container_name; - $this->pull_request_id = $pull_request_id; - } - - public function uniqueId(): string - { - return $this->container_name; - } - - public function handle(): void - { - try { - $status = get_container_status(server: $this->resource->destination->server, container_id: $this->container_name, throwError: false); - if ($this->resource->status === 'running' && $status !== 'running') { - $this->resource->environment->project->team->notify(new StatusChanged($this->resource)); - } - - if ($this->pull_request_id) { - $preview = ApplicationPreview::findPreviewByApplicationAndPullId($this->resource->id, $this->pull_request_id); - $preview->status = $status; - $preview->save(); - } else { - $this->resource->status = $status; - $this->resource->save(); - } - } catch (\Exception $e) { - ray($e->getMessage()); - } - } -} diff --git a/app/Jobs/DatabaseContainerStatusJob.php b/app/Jobs/DatabaseContainerStatusJob.php new file mode 100644 index 000000000..f2fc7053c --- /dev/null +++ b/app/Jobs/DatabaseContainerStatusJob.php @@ -0,0 +1,52 @@ +containerName = $database->uuid; + } + + public function uniqueId(): string + { + return $this->containerName; + } + + public function handle(): void + { + try { + $status = getContainerStatus( + server: $this->database->destination->server, + container_id: $this->containerName, + throwError: false + ); + + if ($this->database->status === 'running' && $status !== 'running') { + $this->database->environment->project->team->notify(new StatusChanged($this->database)); + } + if ($this->database->status !== $status) { + $this->database->status = $status; + $this->database->save(); + } + } catch (\Exception $e) { + ray($e->getMessage()); + } + } +} diff --git a/app/Jobs/ProxyCheckJob.php b/app/Jobs/ProxyCheckJob.php index 7c498107d..bcaf8c8bd 100755 --- a/app/Jobs/ProxyCheckJob.php +++ b/app/Jobs/ProxyCheckJob.php @@ -24,7 +24,7 @@ class ProxyCheckJob implements ShouldQueue $container_name = 'coolify-proxy'; $servers = Server::isUsable()->whereNotNull('proxy')->get(); foreach ($servers as $server) { - $status = get_container_status(server: $server, container_id: $container_name); + $status = getContainerStatus(server: $server, container_id: $container_name); if ($status === 'running') { continue; } diff --git a/app/Jobs/ProxyContainerStatusJob.php b/app/Jobs/ProxyContainerStatusJob.php index 68372895a..3f8c21a3e 100644 --- a/app/Jobs/ProxyContainerStatusJob.php +++ b/app/Jobs/ProxyContainerStatusJob.php @@ -39,7 +39,7 @@ class ProxyContainerStatusJob implements ShouldQueue, ShouldBeUnique public function handle(): void { try { - $container = get_container_status(server: $this->server, all_data: true, container_id: 'coolify-proxy', throwError: true); + $container = getContainerStatus(server: $this->server, all_data: true, container_id: 'coolify-proxy', throwError: true); $status = $container['State']['Status']; if ($this->server->proxy->status !== $status) { $this->server->proxy->status = $status; diff --git a/app/Jobs/ProxyStartJob.php b/app/Jobs/ProxyStartJob.php index 5119210fc..90811d836 100755 --- a/app/Jobs/ProxyStartJob.php +++ b/app/Jobs/ProxyStartJob.php @@ -23,7 +23,7 @@ class ProxyStartJob implements ShouldQueue try { $container_name = 'coolify-proxy'; ray('Starting proxy for server: ' . $this->server->name); - $status = get_container_status(server: $this->server, container_id: $container_name); + $status = getContainerStatus(server: $this->server, container_id: $container_name); if ($status === 'running') { return; } diff --git a/app/Jobs/ResourceStatusJob.php b/app/Jobs/ResourceStatusJob.php index 5ac162d3f..122423f0a 100644 --- a/app/Jobs/ResourceStatusJob.php +++ b/app/Jobs/ResourceStatusJob.php @@ -3,6 +3,7 @@ namespace App\Jobs; use App\Models\Application; +use App\Models\StandalonePostgresql; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldBeUnique; use Illuminate\Contracts\Queue\ShouldQueue; @@ -15,19 +16,25 @@ class ResourceStatusJob implements ShouldQueue, ShouldBeUnique use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; public $applications; + public $postgresqls; public function __construct() { $this->applications = Application::all(); + $this->postgresqls = StandalonePostgresql::all(); } public function handle(): void { try { foreach ($this->applications as $application) { - dispatch(new ContainerStatusJob( - resource: $application, - container_name: generate_container_name($application->uuid), + dispatch(new ApplicationContainerStatusJob( + application: $application, + )); + } + foreach ($this->postgresqls as $postgresql) { + dispatch(new DatabaseContainerStatusJob( + database: $postgresql, )); } } catch (\Exception $e) { diff --git a/bootstrap/helpers/constants.php b/bootstrap/helpers/constants.php index 0e42b311e..cea780c24 100644 --- a/bootstrap/helpers/constants.php +++ b/bootstrap/helpers/constants.php @@ -9,3 +9,4 @@ const VALID_CRON_STRINGS = [ 'monthly' => '0 0 1 * *', 'yearly' => '0 0 1 1 *', ]; +const RESTART_MODE = 'unless-stopped'; diff --git a/bootstrap/helpers/docker.php b/bootstrap/helpers/docker.php index 7b78abe8a..1d4004d07 100644 --- a/bootstrap/helpers/docker.php +++ b/bootstrap/helpers/docker.php @@ -1,14 +1,27 @@ reject(fn ($line) => empty($line)) ->map(fn ($outputLine) => json_decode($outputLine, true, flags: JSON_THROW_ON_ERROR)); } @@ -44,26 +57,38 @@ function format_docker_envs_to_json($rawOutput) } } -function get_container_status(Server $server, string $container_id, bool $all_data = false, bool $throwError = false) +function getApplicationContainerStatus(Application $application) { + $server = $application->destination->server; + $id = $application->id; + + $containers = getCurrentApplicationContainerStatus($server, $id); + if ($containers->count() > 0) { + $status = data_get($containers[0], 'State', 'exited'); + return $status; + } + return 'exited'; +} +function getContainerStatus(Server $server, string $container_id, bool $all_data = false, bool $throwError = false) { - check_server_connection($server); + // check_server_connection($server); $container = instant_remote_process(["docker inspect --format '{{json .}}' {$container_id}"], $server, $throwError); if (!$container) { return 'exited'; } $container = format_docker_command_output_to_json($container); if ($all_data) { - return $container[0]; + return $container; } return data_get($container[0], 'State.Status', 'exited'); } -function generate_container_name(string $uuid, int $pull_request_id = 0) +function generateApplicationContainerName(string $uuid, int $pull_request_id = 0) { - if ($pull_request_id !== 0) { - return $uuid . '-pr-' . $pull_request_id; + $now = now()->format('YmdHis'); + if ($pull_request_id !== 0 && $pull_request_id !== null) { + return $uuid . '-pr-' . $pull_request_id . '-' . $now; } else { - return $uuid; + return $uuid . '-' . $now; } } function get_port_from_dockerfile($dockerfile): int diff --git a/bootstrap/helpers/proxy.php b/bootstrap/helpers/proxy.php index 86c9ca528..37049aaa3 100644 --- a/bootstrap/helpers/proxy.php +++ b/bootstrap/helpers/proxy.php @@ -32,7 +32,7 @@ function generate_default_proxy_configuration(Server $server) "traefik" => [ "container_name" => "coolify-proxy", "image" => "traefik:v2.10", - "restart" => "always", + "restart" => RESTART_MODE, "extra_hosts" => [ "host.docker.internal:host-gateway", ], diff --git a/routes/webhooks.php b/routes/webhooks.php index 77ef23c5d..1e196181b 100644 --- a/routes/webhooks.php +++ b/routes/webhooks.php @@ -161,7 +161,7 @@ Route::post('/source/github/events', function () { $found = ApplicationPreview::where('application_id', $application->id)->where('pull_request_id', $pull_request_id)->first(); if ($found) { $found->delete(); - $container_name = generate_container_name($application->uuid, $pull_request_id); + $container_name = generateApplicationContainerName($application->uuid, $pull_request_id); ray('Stopping container: ' . $container_name); remote_process(["docker rm -f $container_name"], $application->destination->server); return response('Preview Deployment closed.');