From 80035395ff58f15097f63f8b03c433ee8211d9fa Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 14 Feb 2024 15:31:43 +0100 Subject: [PATCH 1/6] Update version numbers + do not cleanup queue on cloud --- app/Console/Commands/Init.php | 3 +++ config/sentry.php | 2 +- config/version.php | 2 +- versions.json | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/app/Console/Commands/Init.php b/app/Console/Commands/Init.php index 315f256c0..c69d411dd 100644 --- a/app/Console/Commands/Init.php +++ b/app/Console/Commands/Init.php @@ -125,6 +125,9 @@ class Init extends Command // Cleanup any failed deployments try { + if (isCloud()) { + return; + } $queued_inprogress_deployments = ApplicationDeploymentQueue::whereIn('status', [ApplicationDeploymentStatus::IN_PROGRESS->value, ApplicationDeploymentStatus::QUEUED->value])->get(); foreach ($queued_inprogress_deployments as $deployment) { ray($deployment->id, $deployment->status); diff --git a/config/sentry.php b/config/sentry.php index 071a24929..a67eea44e 100644 --- a/config/sentry.php +++ b/config/sentry.php @@ -7,7 +7,7 @@ return [ // The release version of your application // Example with dynamic git hash: trim(exec('git --git-dir ' . base_path('.git') . ' log --pretty="%h" -n1 HEAD')) - 'release' => '4.0.0-beta.216', + 'release' => '4.0.0-beta.217', // When left empty or `null` the Laravel environment will be used 'environment' => config('app.env'), diff --git a/config/version.php b/config/version.php index e1bf71945..d13c0ce6f 100644 --- a/config/version.php +++ b/config/version.php @@ -1,3 +1,3 @@ Date: Thu, 15 Feb 2024 11:55:43 +0100 Subject: [PATCH 2/6] feat: add metabase feat: consistent container names fix: for services, you only need to add basicauth label, others are added by coolify fix: label uuids are not randomly generated all the time fix: changing force https will change the labels --- app/Jobs/ApplicationDeploymentJob.php | 35 ++++++++---- app/Livewire/Project/Application/Advanced.php | 10 +++- app/Livewire/Project/Application/General.php | 13 +---- app/Models/Application.php | 2 +- bootstrap/helpers/docker.php | 53 ++++++++++++++++--- bootstrap/helpers/shared.php | 4 +- ..._consistent_application_container_name.php | 28 ++++++++++ .../project/application/advanced.blade.php | 6 ++- templates/compose/metabase.yaml | 35 ++++++++++++ templates/service-templates.json | 11 ++++ 10 files changed, 162 insertions(+), 35 deletions(-) create mode 100644 database/migrations/2024_02_15_101921_add_consistent_application_container_name.php create mode 100644 templates/compose/metabase.yaml diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index ccf1bba3d..aa68c20ad 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -133,6 +133,8 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted $this->is_debug_enabled = $this->application->settings->is_debug_enabled; $this->container_name = generateApplicationContainerName($this->application, $this->pull_request_id); + ray('New container name: ', $this->container_name); + savePrivateKeyToFs($this->server); $this->saved_outputs = collect(); @@ -711,9 +713,14 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted $this->write_deployment_configurations(); $this->server = $this->original_server; } - if (count($this->application->ports_mappings_array) > 0) { + if (count($this->application->ports_mappings_array) > 0 || (bool) $this->application->settings->is_consistent_container_name_enabled) { $this->application_deployment_queue->addLogEntry("----------------------------------------"); - $this->application_deployment_queue->addLogEntry("Application has ports mapped to the host system, rolling update is not supported."); + if (count($this->application->ports_mappings_array) > 0) { + $this->application_deployment_queue->addLogEntry("Application has ports mapped to the host system, rolling update is not supported."); + } + if ((bool) $this->application->settings->is_consistent_container_name_enabled) { + $this->application_deployment_queue->addLogEntry("Consistent container name feature enabled, rolling update is not supported."); + } $this->stop_running_container(force: true); $this->start_by_compose_file(); } else { @@ -1199,13 +1206,18 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted // ]; // } - $docker_compose['services'][$this->application->uuid] = $docker_compose['services'][$this->container_name]; - - data_forget($docker_compose, 'services.' . $this->container_name); - - $custom_compose = convert_docker_run_to_compose($this->application->custom_docker_run_options); - if (count($custom_compose) > 0) { - $docker_compose['services'][$this->application->uuid] = array_merge_recursive($docker_compose['services'][$this->application->uuid], $custom_compose); + if ((bool)$this->application->settings->is_consistent_container_name_enabled) { + $custom_compose = convert_docker_run_to_compose($this->application->custom_docker_run_options); + if (count($custom_compose) > 0) { + $docker_compose['services'][$this->container_name] = array_merge_recursive($docker_compose['services'][$this->container_name], $custom_compose); + } + } else { + $docker_compose['services'][$this->application->uuid] = $docker_compose['services'][$this->container_name]; + data_forget($docker_compose, 'services.' . $this->container_name); + $custom_compose = convert_docker_run_to_compose($this->application->custom_docker_run_options); + if (count($custom_compose) > 0) { + $docker_compose['services'][$this->application->uuid] = array_merge_recursive($docker_compose['services'][$this->application->uuid], $custom_compose); + } } $this->docker_compose = Yaml::dump($docker_compose, 10); @@ -1490,6 +1502,11 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf"); [executeInDocker($this->deployment_uuid, "docker rm -f $containerName >/dev/null 2>&1"), "hidden" => true, "ignore_errors" => true], ); }); + if ($this->application->settings->is_consistent_container_name_enabled) { + $this->execute_remote_command( + [executeInDocker($this->deployment_uuid, "docker rm -f $this->container_name >/dev/null 2>&1"), "hidden" => true, "ignore_errors" => true], + ); + } } else { $this->application_deployment_queue->addLogEntry("New container is not healthy, rolling back to the old container."); $this->application_deployment_queue->update([ diff --git a/app/Livewire/Project/Application/Advanced.php b/app/Livewire/Project/Application/Advanced.php index a2b7afa6a..08b4f9523 100644 --- a/app/Livewire/Project/Application/Advanced.php +++ b/app/Livewire/Project/Application/Advanced.php @@ -8,20 +8,25 @@ use Livewire\Component; class Advanced extends Component { public Application $application; + public bool $is_force_https_enabled; protected $rules = [ 'application.settings.is_git_submodules_enabled' => 'boolean|required', 'application.settings.is_git_lfs_enabled' => 'boolean|required', 'application.settings.is_preview_deployments_enabled' => 'boolean|required', 'application.settings.is_auto_deploy_enabled' => 'boolean|required', - 'application.settings.is_force_https_enabled' => 'boolean|required', + 'is_force_https_enabled' => 'boolean|required', 'application.settings.is_log_drain_enabled' => 'boolean|required', 'application.settings.is_gpu_enabled' => 'boolean|required', 'application.settings.is_build_server_enabled' => 'boolean|required', + 'application.settings.is_consistent_container_name_enabled' => 'boolean|required', 'application.settings.gpu_driver' => 'string|required', 'application.settings.gpu_count' => 'string|required', 'application.settings.gpu_device_ids' => 'string|required', 'application.settings.gpu_options' => 'string|required', ]; + public function mount() { + $this->is_force_https_enabled = $this->application->settings->is_force_https_enabled; + } public function instantSave() { if ($this->application->isLogDrainEnabled()) { @@ -31,7 +36,8 @@ class Advanced extends Component return; } } - if ($this->application->settings->is_force_https_enabled) { + if ($this->application->settings->is_force_https_enabled !== $this->is_force_https_enabled) { + $this->application->settings->is_force_https_enabled = $this->is_force_https_enabled; $this->dispatch('resetDefaultLabels', false); } $this->application->settings->save(); diff --git a/app/Livewire/Project/Application/General.php b/app/Livewire/Project/Application/General.php index c11dbfe4b..673d2473f 100644 --- a/app/Livewire/Project/Application/General.php +++ b/app/Livewire/Project/Application/General.php @@ -126,7 +126,6 @@ class General extends Component $this->application->save(); } $this->initialDockerComposeLocation = $this->application->docker_compose_location; - $this->checkLabelUpdates(); } public function instantSave() { @@ -184,15 +183,6 @@ class General extends Component $this->submit(); $this->dispatch('build_pack_updated'); } - public function checkLabelUpdates() - { - if (md5($this->application->custom_labels) !== md5(implode("|", generateLabelsApplication($this->application)))) { - $this->labelsChanged = true; - } else { - $this->labelsChanged = false; - } - } - public function getWildcardDomain() { $server = data_get($this->application, 'destination.server'); @@ -246,7 +236,7 @@ class General extends Component if ($this->application->additional_servers->count() === 0) { foreach ($domains as $domain) { if (!validate_dns_entry($domain, $this->application->destination->server)) { - $showToaster && $this->dispatch('error', "Validating DNS ($domain) failed.","Make sure you have added the DNS records correctly.

Check this documentation for further help."); + $showToaster && $this->dispatch('error', "Validating DNS ($domain) failed.", "Make sure you have added the DNS records correctly.

Check this documentation for further help."); } } } @@ -279,7 +269,6 @@ class General extends Component } catch (\Throwable $e) { return handleError($e, $this); } finally { - $this->checkLabelUpdates(); $this->isConfigurationChanged = $this->application->isConfigurationChanged(); } } diff --git a/app/Models/Application.php b/app/Models/Application.php index b37476565..df1fb8038 100644 --- a/app/Models/Application.php +++ b/app/Models/Application.php @@ -470,7 +470,7 @@ class Application extends BaseModel { return data_get($this, 'settings.is_log_drain_enabled', false); } - public function isConfigurationChanged($save = false) + public function isConfigurationChanged(bool $save = false) { $newConfigHash = $this->fqdn . $this->git_repository . $this->git_branch . $this->git_commit_sha . $this->build_pack . $this->static_image . $this->install_command . $this->build_command . $this->start_command . $this->port_exposes . $this->port_mappings . $this->base_directory . $this->publish_directory . $this->dockerfile . $this->dockerfile_location . $this->custom_labels; if ($this->pull_request_id === 0 || $this->pull_request_id === null) { diff --git a/bootstrap/helpers/docker.php b/bootstrap/helpers/docker.php index 5308b3fa8..ae8f259fc 100644 --- a/bootstrap/helpers/docker.php +++ b/bootstrap/helpers/docker.php @@ -123,10 +123,14 @@ function getContainerStatus(Server $server, string $container_id, bool $all_data function generateApplicationContainerName(Application $application, $pull_request_id = 0) { + $consistent_container_name = $application->settings->is_consistent_container_name_enabled; $now = now()->format('Hisu'); if ($pull_request_id !== 0 && $pull_request_id !== null) { return $application->uuid . '-pr-' . $pull_request_id; } else { + if ($consistent_container_name) { + return $application->uuid; + } return $application->uuid . '-' . $now; } } @@ -209,15 +213,34 @@ function generateServiceSpecificFqdns(ServiceApplication|Application $resource, } return $payload; } -function fqdnLabelsForTraefik(string $uuid, Collection $domains, bool $is_force_https_enabled = false, $onlyPort = null) +function fqdnLabelsForTraefik(string $uuid, Collection $domains, bool $is_force_https_enabled = false, $onlyPort = null, ?Collection $serviceLabels = null) { $labels = collect([]); $labels->push('traefik.enable=true'); $labels->push("traefik.http.middlewares.gzip.compress=true"); $labels->push("traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https"); + + $basic_auth = false; + $basic_auth_middleware = null; + + if ($serviceLabels) { + $basic_auth = $serviceLabels->contains(function ($value) { + return str_contains($value, 'basicauth'); + }); + if ($basic_auth) { + $basic_auth_middleware = $serviceLabels + ->map(function ($item) { + if (preg_match('/traefik\.http\.middlewares\.(.*?)\.basicauth\.users/', $item, $matches)) { + return $matches[1]; + } + }) + ->filter() + ->first(); + } + } foreach ($domains as $loop => $domain) { try { - $uuid = new Cuid2(7); + // $uuid = new Cuid2(7); $url = Url::fromString($domain); $host = $url->getHost(); $path = $url->getPath(); @@ -239,11 +262,18 @@ function fqdnLabelsForTraefik(string $uuid, Collection $domains, bool $is_force_ } if ($path !== '/') { $labels->push("traefik.http.middlewares.{$https_label}-stripprefix.stripprefix.prefixes={$path}"); - $labels->push("traefik.http.routers.{$https_label}.middlewares={$https_label}-stripprefix,gzip"); + $middlewares = "gzip,{$https_label}-stripprefix"; + if ($basic_auth && $basic_auth_middleware) { + $middlewares = $middlewares . ',' . $basic_auth_middleware; + } + $labels->push("traefik.http.routers.{$https_label}.middlewares={$middlewares}"); } else { - $labels->push("traefik.http.routers.{$https_label}.middlewares=gzip"); + $middlewares = "gzip"; + if ($basic_auth && $basic_auth_middleware) { + $middlewares = $middlewares . ',' . $basic_auth_middleware; + } + $labels->push("traefik.http.routers.{$https_label}.middlewares={$middlewares}"); } - $labels->push("traefik.http.routers.{$https_label}.tls=true"); $labels->push("traefik.http.routers.{$https_label}.tls.certresolver=letsencrypt"); @@ -267,16 +297,23 @@ function fqdnLabelsForTraefik(string $uuid, Collection $domains, bool $is_force_ } if ($path !== '/') { $labels->push("traefik.http.middlewares.{$http_label}-stripprefix.stripprefix.prefixes={$path}"); - $labels->push("traefik.http.routers.{$http_label}.middlewares={$http_label}-stripprefix,gzip"); + $middlewares = "gzip,{$http_label}-stripprefix"; + if ($basic_auth && $basic_auth_middleware) { + $middlewares = $middlewares . ',' . $basic_auth_middleware; + } + $labels->push("traefik.http.routers.{$http_label}.middlewares={$middlewares}"); } else { - $labels->push("traefik.http.routers.{$http_label}.middlewares=gzip"); + $middlewares = "gzip"; + if ($basic_auth && $basic_auth_middleware) { + $middlewares = $middlewares . ',' . $basic_auth_middleware; + } + $labels->push("traefik.http.routers.{$http_label}.middlewares={$middlewares}"); } } } catch (\Throwable $e) { continue; } } - return $labels->sort(); } function generateLabelsApplication(Application $application, ?ApplicationPreview $preview = null): array diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php index 9ab26e59a..bc33388eb 100644 --- a/bootstrap/helpers/shared.php +++ b/bootstrap/helpers/shared.php @@ -1039,7 +1039,7 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal $serviceLabels = $serviceLabels->merge($defaultLabels); if (!$isDatabase && $fqdns->count() > 0) { if ($fqdns) { - $serviceLabels = $serviceLabels->merge(fqdnLabelsForTraefik($resource->uuid, $fqdns, true)); + $serviceLabels = $serviceLabels->merge(fqdnLabelsForTraefik($resource->uuid, $fqdns, true, serviceLabels: $serviceLabels)); } } if ($resource->server->isLogDrainEnabled() && $savedService->isLogDrainEnabled()) { @@ -1480,7 +1480,7 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal return $preview_fqdn; }); } - $serviceLabels = $serviceLabels->merge(fqdnLabelsForTraefik($uuid, $fqdns)); + $serviceLabels = $serviceLabels->merge(fqdnLabelsForTraefik($uuid, $fqdns,serviceLabels: $serviceLabels)); } } } diff --git a/database/migrations/2024_02_15_101921_add_consistent_application_container_name.php b/database/migrations/2024_02_15_101921_add_consistent_application_container_name.php new file mode 100644 index 000000000..827b4c426 --- /dev/null +++ b/database/migrations/2024_02_15_101921_add_consistent_application_container_name.php @@ -0,0 +1,28 @@ +boolean('is_consistent_container_name_enabled')->default(false); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('application_settings', function (Blueprint $table) { + $table->dropColumn('is_consistent_container_name_enabled'); + }); + } +}; diff --git a/resources/views/livewire/project/application/advanced.blade.php b/resources/views/livewire/project/application/advanced.blade.php index 49196b208..53d984698 100644 --- a/resources/views/livewire/project/application/advanced.blade.php +++ b/resources/views/livewire/project/application/advanced.blade.php @@ -15,7 +15,11 @@ @endif + instantSave id="is_force_https_enabled" label="Force Https" /> +

Logs

@if (!$application->settings->is_raw_compose_deployment_enabled) Date: Thu, 15 Feb 2024 12:01:59 +0100 Subject: [PATCH 3/6] Refactor application FQDN handling --- app/Livewire/Project/Application/General.php | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/app/Livewire/Project/Application/General.php b/app/Livewire/Project/Application/General.php index 673d2473f..732be62d0 100644 --- a/app/Livewire/Project/Application/General.php +++ b/app/Livewire/Project/Application/General.php @@ -163,6 +163,7 @@ class General extends Component } return $domain; } + public function updatedApplicationBuildPack() { if ($this->application->build_pack !== 'nixpacks') { @@ -202,6 +203,13 @@ class General extends Component public function updatedApplicationFqdn() { + $this->application->fqdn = str($this->application->fqdn)->replaceEnd(',', '')->trim(); + $this->application->fqdn = str($this->application->fqdn)->replaceStart(',', '')->trim(); + $this->application->fqdn = str($this->application->fqdn)->trim()->explode(',')->map(function ($domain) { + return str($domain)->trim()->lower(); + }); + $this->application->fqdn = $this->application->fqdn->unique()->implode(','); + $this->application->save(); $this->resetDefaultLabels(false); // $this->dispatch('success', 'Labels reset to default!'); } @@ -228,11 +236,7 @@ class General extends Component ]); } if (data_get($this->application, 'fqdn')) { - $this->application->fqdn = str($this->application->fqdn)->replaceEnd(',', '')->trim(); - $domains = str($this->application->fqdn)->trim()->explode(',')->map(function ($domain) { - return str($domain)->trim()->lower(); - }); - $domains = $domains->unique(); + $domains = str($this->application->fqdn)->trim()->explode(','); if ($this->application->additional_servers->count() === 0) { foreach ($domains as $domain) { if (!validate_dns_entry($domain, $this->application->destination->server)) { @@ -243,7 +247,6 @@ class General extends Component check_fqdn_usage($this->application); $this->application->fqdn = $domains->implode(','); } - if (data_get($this->application, 'custom_docker_run_options')) { $this->application->custom_docker_run_options = str($this->application->custom_docker_run_options)->trim(); } From c770c8d988b526f42bf97cafe2cfe04c753feb44 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 15 Feb 2024 12:04:52 +0100 Subject: [PATCH 4/6] Add warning icon for configuration not applied --- .../views/livewire/project/application/general.blade.php | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/resources/views/livewire/project/application/general.blade.php b/resources/views/livewire/project/application/general.blade.php index 8f13ce177..f33a3c1a0 100644 --- a/resources/views/livewire/project/application/general.blade.php +++ b/resources/views/livewire/project/application/general.blade.php @@ -6,8 +6,11 @@ Save @if ($isConfigurationChanged && !is_null($application->config_hash)) -
Configuration not applied to the running application. You need to - redeploy.
+
+ + + +
@endif
General configuration for your application.
From dfba593072dcfa415794776a049c75f04113e5a6 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 15 Feb 2024 12:08:48 +0100 Subject: [PATCH 5/6] feat: magic for traefik redirectregex in services --- bootstrap/helpers/docker.php | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/bootstrap/helpers/docker.php b/bootstrap/helpers/docker.php index ae8f259fc..f01eb972f 100644 --- a/bootstrap/helpers/docker.php +++ b/bootstrap/helpers/docker.php @@ -222,7 +222,8 @@ function fqdnLabelsForTraefik(string $uuid, Collection $domains, bool $is_force_ $basic_auth = false; $basic_auth_middleware = null; - + $redirect = false; + $redirect_middleware = null; if ($serviceLabels) { $basic_auth = $serviceLabels->contains(function ($value) { return str_contains($value, 'basicauth'); @@ -237,6 +238,19 @@ function fqdnLabelsForTraefik(string $uuid, Collection $domains, bool $is_force_ ->filter() ->first(); } + $redirect = $serviceLabels->contains(function ($value) { + return str_contains($value, 'redirectregex'); + }); + if ($redirect) { + $redirect_middleware = $serviceLabels + ->map(function ($item) { + if (preg_match('/traefik\.http\.middlewares\.(.*?)\.redirectregex\.regex/', $item, $matches)) { + return $matches[1]; + } + }) + ->filter() + ->first(); + } } foreach ($domains as $loop => $domain) { try { @@ -266,12 +280,18 @@ function fqdnLabelsForTraefik(string $uuid, Collection $domains, bool $is_force_ if ($basic_auth && $basic_auth_middleware) { $middlewares = $middlewares . ',' . $basic_auth_middleware; } + if ($redirect && $redirect_middleware) { + $middlewares = $middlewares . ',' . $redirect_middleware; + } $labels->push("traefik.http.routers.{$https_label}.middlewares={$middlewares}"); } else { $middlewares = "gzip"; if ($basic_auth && $basic_auth_middleware) { $middlewares = $middlewares . ',' . $basic_auth_middleware; } + if ($redirect && $redirect_middleware) { + $middlewares = $middlewares . ',' . $redirect_middleware; + } $labels->push("traefik.http.routers.{$https_label}.middlewares={$middlewares}"); } $labels->push("traefik.http.routers.{$https_label}.tls=true"); @@ -301,12 +321,18 @@ function fqdnLabelsForTraefik(string $uuid, Collection $domains, bool $is_force_ if ($basic_auth && $basic_auth_middleware) { $middlewares = $middlewares . ',' . $basic_auth_middleware; } + if ($redirect && $redirect_middleware) { + $middlewares = $middlewares . ',' . $redirect_middleware; + } $labels->push("traefik.http.routers.{$http_label}.middlewares={$middlewares}"); } else { $middlewares = "gzip"; if ($basic_auth && $basic_auth_middleware) { $middlewares = $middlewares . ',' . $basic_auth_middleware; } + if ($redirect && $redirect_middleware) { + $middlewares = $middlewares . ',' . $redirect_middleware; + } $labels->push("traefik.http.routers.{$http_label}.middlewares={$middlewares}"); } } From 00feef40a3c22b1dc5a2855b04069edc527ff1ef Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 15 Feb 2024 12:29:30 +0100 Subject: [PATCH 6/6] Fix image tag in docker-compose.prod.yml --- docker-compose.prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index b4b81530c..407be245b 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -1,7 +1,7 @@ version: '3.8' services: coolify: - image: "ghcr.io/coollabsio/coolify:${LATEST_IMAGE:latest}" + image: "ghcr.io/coollabsio/coolify:${LATEST_IMAGE:-latest}" volumes: - type: bind source: /data/coolify/source/.env