From 3dd36a22717b954c30b4ad339783cd26ef5b229a Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 22 Nov 2023 15:18:37 +0100 Subject: [PATCH 01/46] Fix container status handling and notifications --- app/Jobs/ContainerStatusJob.php | 18 +++++++----------- bootstrap/helpers/shared.php | 2 +- config/sentry.php | 2 +- config/version.php | 2 +- versions.json | 2 +- 5 files changed, 11 insertions(+), 15 deletions(-) diff --git a/app/Jobs/ContainerStatusJob.php b/app/Jobs/ContainerStatusJob.php index a45bebf8e..79265f7a1 100644 --- a/app/Jobs/ContainerStatusJob.php +++ b/app/Jobs/ContainerStatusJob.php @@ -35,17 +35,13 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted return $this->server->id; } - public function handle(): void + public function handle() { // ray("checking container statuses for {$this->server->id}"); try { if (!$this->server->isServerReady()) { return; }; - $containers = instant_remote_process(["docker container ls -q"], $this->server); - if (!$containers) { - return; - } $containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this->server); $containers = format_docker_command_output_to_json($containers); $applications = $this->server->applications(); @@ -167,7 +163,7 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted } else { $url = null; } - $this->server->team->notify(new ContainerStopped($containerName, $this->server, $url)); + $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); $exitedService->update(['status' => 'exited']); } @@ -194,7 +190,7 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted $url = null; } - $this->server->team->notify(new ContainerStopped($containerName, $this->server, $url)); + $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); } $notRunningApplicationPreviews = $previews->pluck('id')->diff($foundApplicationPreviews); foreach ($notRunningApplicationPreviews as $previewId) { @@ -219,7 +215,7 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted $url = null; } - $this->server->team->notify(new ContainerStopped($containerName, $this->server, $url)); + $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); } $notRunningDatabases = $databases->pluck('id')->diff($foundDatabases); foreach ($notRunningDatabases as $database) { @@ -243,7 +239,7 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted } else { $url = null; } - $this->server->team->notify(new ContainerStopped($containerName, $this->server, $url)); + $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); } // Check if proxy is running @@ -256,7 +252,7 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted $shouldStart = CheckProxy::run($this->server); if ($shouldStart) { StartProxy::run($this->server, false); - $this->server->team->notify(new ContainerRestarted('coolify-proxy', $this->server)); + $this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server)); } else { ray('Proxy could not be started.'); } @@ -272,7 +268,7 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted } catch (\Throwable $e) { send_internal_notification("ContainerStatusJob failed on ({$this->server->id}) with: " . $e->getMessage()); ray($e->getMessage()); - handleError($e); + return handleError($e); } } } diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php index 071c252ff..839f86835 100644 --- a/bootstrap/helpers/shared.php +++ b/bootstrap/helpers/shared.php @@ -281,7 +281,7 @@ function send_internal_notification(string $message): void try { $baseUrl = config('app.name'); $team = Team::find(0); - $team->notify(new GeneralNotification("👀 {$baseUrl}: " . $message)); + $team?->notify(new GeneralNotification("👀 {$baseUrl}: " . $message)); ray("👀 {$baseUrl}: " . $message); } catch (\Throwable $e) { ray($e->getMessage()); diff --git a/config/sentry.php b/config/sentry.php index efd043adf..c07fda49b 100644 --- a/config/sentry.php +++ b/config/sentry.php @@ -7,7 +7,7 @@ return [ // The release version of your application // Example with dynamic git hash: trim(exec('git --git-dir ' . base_path('.git') . ' log --pretty="%h" -n1 HEAD')) - 'release' => '4.0.0-beta.146', + 'release' => '4.0.0-beta.147', // When left empty or `null` the Laravel environment will be used 'environment' => config('app.env'), diff --git a/config/version.php b/config/version.php index 8c257643a..cf8ab7438 100644 --- a/config/version.php +++ b/config/version.php @@ -1,3 +1,3 @@ Date: Wed, 22 Nov 2023 15:18:49 +0100 Subject: [PATCH 02/46] wip --- .../ApplicationDeploySimpleDockerfileJob.php | 19 +++++- app/Models/Server.php | 62 +++++++++++++++++++ 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/app/Jobs/ApplicationDeploySimpleDockerfileJob.php b/app/Jobs/ApplicationDeploySimpleDockerfileJob.php index a9e17bc80..0cf9f61c4 100644 --- a/app/Jobs/ApplicationDeploySimpleDockerfileJob.php +++ b/app/Jobs/ApplicationDeploySimpleDockerfileJob.php @@ -2,6 +2,8 @@ namespace App\Jobs; +use App\Models\Application; +use App\Models\ApplicationDeploymentQueue; use App\Traits\ExecuteRemoteCommand; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldBeEncrypted; @@ -23,7 +25,22 @@ class ApplicationDeploySimpleDockerfileJob implements ShouldQueue, ShouldBeEncry { $this->applicationDeploymentQueueId = $applicationDeploymentQueueId; } - public function handle() { + public function handle() + { ray('Deploying Simple Dockerfile'); + $applicationDeploymentQueue = ApplicationDeploymentQueue::find($this->applicationDeploymentQueueId); + $application = Application::find($applicationDeploymentQueue->application_id); + $destination = $application->destination->getMorphClass()::where('id', $application->destination->id)->first(); + $server = data_get($destination, 'server'); + $commands = collect([]); + $commands->push( + [ + 'command' => 'echo "Starting deployment of simple dockerfile."', + ], + [ + 'command' => 'ls -la', + ] + ); + $server->executeRemoteCommand(commands: $commands, logModel: $applicationDeploymentQueue); } } diff --git a/app/Models/Server.php b/app/Models/Server.php index 30e07c1d8..7d161d5f1 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -4,12 +4,16 @@ namespace App\Models; use App\Actions\Server\InstallLogDrain; use App\Actions\Server\InstallNewRelic; +use App\Enums\ApplicationDeploymentStatus; use App\Enums\ProxyStatus; use App\Enums\ProxyTypes; use App\Notifications\Server\Revived; use App\Notifications\Server\Unreachable; use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Casts\Attribute; +use Illuminate\Support\Carbon; +use Illuminate\Support\Collection; +use Illuminate\Support\Facades\Process; use Illuminate\Support\Sleep; use Spatie\SchemalessAttributes\Casts\SchemalessAttributes; use Spatie\SchemalessAttributes\SchemalessAttributesTrait; @@ -18,6 +22,7 @@ use Illuminate\Support\Str; class Server extends BaseModel { use SchemalessAttributesTrait; + public static $batch_counter = 0; protected static function booted() { @@ -387,4 +392,61 @@ class Server extends BaseModel { return instant_remote_process(["docker network create coolify --attachable >/dev/null 2>&1 || true"], $this, false); } + public function executeRemoteCommand(Collection $commands, ApplicationDeploymentQueue $logModel) + { + static::$batch_counter++; + foreach ($commands as $command) { + $command = data_get($command, 'command') ?? $command[0] ?? null; + if (is_null($command)) { + continue; + } + $hidden = data_get($command, 'hidden', false); + $ignoreErrors = data_get($command, 'ignoreErrors', false); + $customOutputType = data_get($command, 'customOutputType'); + $saveOutput = data_get($command, 'saveOutput'); + $remoteCommand = generateSshCommand($this, $command); + + $process = Process::timeout(3600)->idleTimeout(3600)->start($remoteCommand, function (string $type, string $output) use ($command, $hidden, $customOutputType, $logModel, $saveOutput) { + $output = str($output)->trim(); + if ($output->startsWith('╔')) { + $output = "\n" . $output; + } + $newLogEntry = [ + 'command' => remove_iip($command), + 'output' => remove_iip($output), + 'type' => $customOutputType ?? $type === 'err' ? 'stderr' : 'stdout', + 'timestamp' => Carbon::now('UTC'), + 'hidden' => $hidden, + 'batch' => static::$batch_counter, + ]; + if (!$logModel->logs) { + $new_log_entry['order'] = 1; + } else { + $previous_logs = json_decode($this->log_model->logs, associative: true, flags: JSON_THROW_ON_ERROR); + $new_log_entry['order'] = count($previous_logs) + 1; + } + $previousLogs = json_decode($logModel->logs, associative: true, flags: JSON_THROW_ON_ERROR); + $newLogEntry['order'] = count($previousLogs) + 1; + $previousLogs[] = $newLogEntry; + $logModel->logs = json_encode($previousLogs, flags: JSON_THROW_ON_ERROR); + $logModel->save(); + if ($saveOutput) { + $this->remoteCommandOutputs[$saveOutput] = str($output)->trim(); + } + }); + $logModel->update([ + 'current_process_id' => $process->id(), + ]); + + $processResult = $process->wait(); + if ($processResult->exitCode() !== 0) { + if (!$ignoreErrors) { + $status = ApplicationDeploymentStatus::FAILED->value; + $logModel->status = $status; + $logModel->save(); + throw new \RuntimeException($processResult->errorOutput()); + } + } + } + } } From 407eba8b765719c81d02d360fa5bea46ad6a9657 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 22 Nov 2023 16:39:16 +0100 Subject: [PATCH 03/46] Fix DockerCleanupJob exception message --- app/Jobs/DockerCleanupJob.php | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/Jobs/DockerCleanupJob.php b/app/Jobs/DockerCleanupJob.php index 14ca11b22..a48ddd248 100644 --- a/app/Jobs/DockerCleanupJob.php +++ b/app/Jobs/DockerCleanupJob.php @@ -3,8 +3,6 @@ namespace App\Jobs; use App\Models\Server; -use App\Notifications\Server\HighDiskUsage; -use Exception; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldQueue; @@ -13,6 +11,7 @@ use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\Middleware\WithoutOverlapping; use Illuminate\Queue\SerializesModels; use Illuminate\Support\Facades\Log; +use RuntimeException; class DockerCleanupJob implements ShouldQueue, ShouldBeEncrypted { @@ -35,7 +34,7 @@ class DockerCleanupJob implements ShouldQueue, ShouldBeEncrypted } }); if ($isInprogress) { - throw new Exception('DockerCleanupJob: ApplicationDeploymentQueue is not empty, skipping...'); + throw new RuntimeException('DockerCleanupJob: ApplicationDeploymentQueue is not empty, skipping...'); } if (!$this->server->isFunctional()) { return; From 9012f6b953e6c34a085161331b01be0c883f29d0 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 22 Nov 2023 16:40:49 +0100 Subject: [PATCH 04/46] Fix GitHub App retrieval in webhooks.php --- routes/webhooks.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/routes/webhooks.php b/routes/webhooks.php index a2dbf5a1a..f59f8f681 100644 --- a/routes/webhooks.php +++ b/routes/webhooks.php @@ -317,7 +317,10 @@ Route::post('/source/github/events', function () { // Installation handled by setup redirect url. Repositories queried on-demand. return response('cool'); } - $github_app = GithubApp::where('app_id', $x_github_hook_installation_target_id)->firstOrFail(); + $github_app = GithubApp::where('app_id', $x_github_hook_installation_target_id)->first(); + if (is_null($github_app)) { + return response('Nothing to do. No GitHub App found.'); + } $webhook_secret = data_get($github_app, 'webhook_secret'); $hmac = hash_hmac('sha256', request()->getContent(), $webhook_secret); From 7fb9e672cf41678f62a10adaa6b13535a0bf9b9f Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 22 Nov 2023 20:56:25 +0100 Subject: [PATCH 05/46] Fix server execution method parameter name --- app/Jobs/ApplicationDeployDockerImageJob.php | 40 ++++++++++---------- app/Models/Server.php | 18 ++++----- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/app/Jobs/ApplicationDeployDockerImageJob.php b/app/Jobs/ApplicationDeployDockerImageJob.php index dda5629c8..24526937a 100644 --- a/app/Jobs/ApplicationDeployDockerImageJob.php +++ b/app/Jobs/ApplicationDeployDockerImageJob.php @@ -33,17 +33,18 @@ class ApplicationDeployDockerImageJob implements ShouldQueue, ShouldBeEncrypted ray('Deploying Docker Image'); try { $applicationDeploymentQueue = ApplicationDeploymentQueue::find($this->applicationDeploymentQueueId); - $application = Application::find($applicationDeploymentQueue->application_id); $deploymentUuid = data_get($applicationDeploymentQueue, 'deployment_uuid'); - $dockerImage = data_get($application, 'docker_registry_image_name'); - $dockerImageTag = data_get($application, 'docker_registry_image_tag'); - $productionImageName = str("{$dockerImage}:{$dockerImageTag}"); - $destination = $application->destination->getMorphClass()::where('id', $application->destination->id)->first(); $pullRequestId = data_get($applicationDeploymentQueue, 'pull_request_id'); - $server = data_get($destination, 'server'); - $network = data_get($destination, 'network'); + $application = Application::find($applicationDeploymentQueue->application_id)->firstOrFail(); + $server = data_get($application->destination, 'server'); + $network = data_get($application->destination, 'network'); + + $dockerImage = data_get($application, 'docker_registry_image_name'); + $dockerImageTag = data_get($application, 'docker_registry_image_tag'); + + $productionImageName = str("{$dockerImage}:{$dockerImageTag}"); $containerName = generateApplicationContainerName($application, $pullRequestId); savePrivateKeyToFs($server); @@ -53,15 +54,11 @@ class ApplicationDeployDockerImageJob implements ShouldQueue, ShouldBeEncrypted $applicationDeploymentQueue->update([ 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value, ]); - $this->executeRemoteCommand( - server: $server, - logModel: $applicationDeploymentQueue, - commands: prepareHelperContainer($server, $network, $deploymentUuid) + $server->executeRemoteCommand( + commands: prepareHelperContainer($server, $network, $deploymentUuid), + loggingModel: $applicationDeploymentQueue ); - - $this->executeRemoteCommand( - server: $server, - logModel: $applicationDeploymentQueue, + $server->executeRemoteCommand( commands: generateComposeFile( deploymentUuid: $deploymentUuid, server: $server, @@ -70,13 +67,16 @@ class ApplicationDeployDockerImageJob implements ShouldQueue, ShouldBeEncrypted containerName: $containerName, imageName: $productionImageName, pullRequestId: $pullRequestId - ) + ), + loggingModel: $applicationDeploymentQueue ); - $this->executeRemoteCommand( - server: $server, - logModel: $applicationDeploymentQueue, - commands: rollingUpdate(application: $application, deploymentUuid: $deploymentUuid) + $server->executeRemoteCommand( + commands: rollingUpdate(application: $application, deploymentUuid: $deploymentUuid), + loggingModel: $applicationDeploymentQueue ); + $applicationDeploymentQueue->update([ + 'status' => ApplicationDeploymentStatus::FINISHED->value, + ]); } catch (Throwable $e) { $this->executeRemoteCommand( server: $server, diff --git a/app/Models/Server.php b/app/Models/Server.php index 7d161d5f1..50e69f3b3 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -392,7 +392,7 @@ class Server extends BaseModel { return instant_remote_process(["docker network create coolify --attachable >/dev/null 2>&1 || true"], $this, false); } - public function executeRemoteCommand(Collection $commands, ApplicationDeploymentQueue $logModel) + public function executeRemoteCommand(Collection $commands, ApplicationDeploymentQueue $loggingModel) { static::$batch_counter++; foreach ($commands as $command) { @@ -406,7 +406,7 @@ class Server extends BaseModel $saveOutput = data_get($command, 'saveOutput'); $remoteCommand = generateSshCommand($this, $command); - $process = Process::timeout(3600)->idleTimeout(3600)->start($remoteCommand, function (string $type, string $output) use ($command, $hidden, $customOutputType, $logModel, $saveOutput) { + $process = Process::timeout(3600)->idleTimeout(3600)->start($remoteCommand, function (string $type, string $output) use ($command, $hidden, $customOutputType, $loggingModel, $saveOutput) { $output = str($output)->trim(); if ($output->startsWith('╔')) { $output = "\n" . $output; @@ -419,22 +419,22 @@ class Server extends BaseModel 'hidden' => $hidden, 'batch' => static::$batch_counter, ]; - if (!$logModel->logs) { + if (!$loggingModel->logs) { $new_log_entry['order'] = 1; } else { $previous_logs = json_decode($this->log_model->logs, associative: true, flags: JSON_THROW_ON_ERROR); $new_log_entry['order'] = count($previous_logs) + 1; } - $previousLogs = json_decode($logModel->logs, associative: true, flags: JSON_THROW_ON_ERROR); + $previousLogs = json_decode($loggingModel->logs, associative: true, flags: JSON_THROW_ON_ERROR); $newLogEntry['order'] = count($previousLogs) + 1; $previousLogs[] = $newLogEntry; - $logModel->logs = json_encode($previousLogs, flags: JSON_THROW_ON_ERROR); - $logModel->save(); + $loggingModel->logs = json_encode($previousLogs, flags: JSON_THROW_ON_ERROR); + $loggingModel->save(); if ($saveOutput) { $this->remoteCommandOutputs[$saveOutput] = str($output)->trim(); } }); - $logModel->update([ + $loggingModel->update([ 'current_process_id' => $process->id(), ]); @@ -442,8 +442,8 @@ class Server extends BaseModel if ($processResult->exitCode() !== 0) { if (!$ignoreErrors) { $status = ApplicationDeploymentStatus::FAILED->value; - $logModel->status = $status; - $logModel->save(); + $loggingModel->status = $status; + $loggingModel->save(); throw new \RuntimeException($processResult->errorOutput()); } } From 2a7a63a672c06cb5fc46418afba84bcca718c494 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 23 Nov 2023 09:05:22 +0100 Subject: [PATCH 06/46] Add trigger.dev service --- .../compose/trigger-without-database.yaml | 20 +++++++++ templates/compose/trigger.yaml | 41 +++++++++++++++++++ templates/service-templates.json | 28 +++++++++++++ 3 files changed, 89 insertions(+) create mode 100644 templates/compose/trigger-without-database.yaml create mode 100644 templates/compose/trigger.yaml diff --git a/templates/compose/trigger-without-database.yaml b/templates/compose/trigger-without-database.yaml new file mode 100644 index 000000000..4fc1f8992 --- /dev/null +++ b/templates/compose/trigger-without-database.yaml @@ -0,0 +1,20 @@ +# documentation: https://trigger.dev/docs/documentation/guides/self-hosting +# slogan: The open source Background Jobs framework for TypeScript +# tags: trigger.dev, background jobs, typescript, trigger, jobs, cron, scheduler + +services: + trigger: + image: ghcr.io/triggerdotdev/trigger.dev:latest + environment: + - SERVICE_FQDN_TRIGGER + - LOGIN_ORIGIN=$SERVICE_FQDN_TRIGGER + - APP_ORIGIN=$SERVICE_FQDN_TRIGGER + - MAGIC_LINK_SECRET=$SERVICE_PASSWORD_64_MAGIC + - ENCRYPTION_KEY=$SERVICE_PASSWORD_64_ENCRYPTION + - SESSION_SECRET=$SERVICE_PASSWORD_64_SESSION + - DATABASE_URL=${DATABASE_URL} + - DIRECT_URL=${DATABASE_URL} + - RUNTIME_PLATFORM=docker-compose + - NODE_ENV=production + healthcheck: + test: ["NONE"] diff --git a/templates/compose/trigger.yaml b/templates/compose/trigger.yaml new file mode 100644 index 000000000..b297fea63 --- /dev/null +++ b/templates/compose/trigger.yaml @@ -0,0 +1,41 @@ +# documentation: https://trigger.dev/docs/documentation/guides/self-hosting +# slogan: The open source Background Jobs framework for TypeScript +# tags: trigger.dev, background jobs, typescript, trigger, jobs, cron, scheduler + +services: + trigger: + image: ghcr.io/triggerdotdev/trigger.dev:latest + environment: + - SERVICE_FQDN_TRIGGER + - LOGIN_ORIGIN=$SERVICE_FQDN_TRIGGER + - APP_ORIGIN=$SERVICE_FQDN_TRIGGER + - MAGIC_LINK_SECRET=$SERVICE_PASSWORD_64_MAGIC + - ENCRYPTION_KEY=$SERVICE_PASSWORD_64_ENCRYPTION + - SESSION_SECRET=$SERVICE_PASSWORD_64_SESSION + - POSTGRES_USER=$SERVICE_USER_POSTGRES + - POSTGRES_PASSWORD=$SERVICE_PASSWORD_POSTGRES + - POSTGRES_DB=${POSTGRES_DB:-trigger} + - POSTGRES_HOST=postgres + - DATABASE_URL=postgres://$SERVICE_USER_POSTGRES:$SERVICE_PASSWORD_POSTGRES@postgresql:5432/$POSTGRES_DB + - DIRECT_URL=postgres://$SERVICE_USER_POSTGRES:$SERVICE_PASSWORD_POSTGRES@postgresql:5432/$POSTGRES_DB + - RUNTIME_PLATFORM=docker-compose + - NODE_ENV=production + depends_on: + postgresql: + condition: service_healthy + healthcheck: + test: ["NONE"] + postgresql: + image: postgres:15-alpine + volumes: + - postgresql-data:/var/lib/postgresql/data + environment: + - POSTGRES_USER=$SERVICE_USER_POSTGRES + - POSTGRES_PASSWORD=$SERVICE_PASSWORD_POSTGRES + - POSTGRES_DB=${POSTGRES_DB:-trigger} + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] + interval: 5s + timeout: 20s + retries: 10 + diff --git a/templates/service-templates.json b/templates/service-templates.json index 9b6b4c2fe..36ee6895f 100644 --- a/templates/service-templates.json +++ b/templates/service-templates.json @@ -441,6 +441,34 @@ "internet" ] }, + "trigger-without-database": { + "documentation": "https:\/\/trigger.dev\/docs\/documentation\/guides\/self-hosting", + "slogan": "The open source Background Jobs framework for TypeScript", + "compose": "c2VydmljZXM6CiAgdHJpZ2dlcjoKICAgIGltYWdlOiAnZ2hjci5pby90cmlnZ2VyZG90ZGV2L3RyaWdnZXIuZGV2OmxhdGVzdCcKICAgIGVudmlyb25tZW50OgogICAgICAtIFNFUlZJQ0VfRlFETl9UUklHR0VSCiAgICAgIC0gTE9HSU5fT1JJR0lOPSRTRVJWSUNFX0ZRRE5fVFJJR0dFUgogICAgICAtIEFQUF9PUklHSU49JFNFUlZJQ0VfRlFETl9UUklHR0VSCiAgICAgIC0gTUFHSUNfTElOS19TRUNSRVQ9JFNFUlZJQ0VfUEFTU1dPUkRfNjRfTUFHSUMKICAgICAgLSBFTkNSWVBUSU9OX0tFWT0kU0VSVklDRV9QQVNTV09SRF82NF9FTkNSWVBUSU9OCiAgICAgIC0gU0VTU0lPTl9TRUNSRVQ9JFNFUlZJQ0VfUEFTU1dPUkRfNjRfU0VTU0lPTgogICAgICAtICdEQVRBQkFTRV9VUkw9JHtEQVRBQkFTRV9VUkx9JwogICAgICAtICdESVJFQ1RfVVJMPSR7REFUQUJBU0VfVVJMfScKICAgICAgLSBSVU5USU1FX1BMQVRGT1JNPWRvY2tlci1jb21wb3NlCiAgICAgIC0gTk9ERV9FTlY9cHJvZHVjdGlvbgogICAgaGVhbHRoY2hlY2s6CiAgICAgIHRlc3Q6CiAgICAgICAgLSBOT05FCg==", + "tags": [ + "trigger.dev", + "background jobs", + "typescript", + "trigger", + "jobs", + "cron", + "scheduler" + ] + }, + "trigger": { + "documentation": "https:\/\/trigger.dev\/docs\/documentation\/guides\/self-hosting", + "slogan": "The open source Background Jobs framework for TypeScript", + "compose": "c2VydmljZXM6CiAgdHJpZ2dlcjoKICAgIGltYWdlOiAnZ2hjci5pby90cmlnZ2VyZG90ZGV2L3RyaWdnZXIuZGV2OmxhdGVzdCcKICAgIGVudmlyb25tZW50OgogICAgICAtIFNFUlZJQ0VfRlFETl9UUklHR0VSCiAgICAgIC0gTE9HSU5fT1JJR0lOPSRTRVJWSUNFX0ZRRE5fVFJJR0dFUgogICAgICAtIEFQUF9PUklHSU49JFNFUlZJQ0VfRlFETl9UUklHR0VSCiAgICAgIC0gTUFHSUNfTElOS19TRUNSRVQ9JFNFUlZJQ0VfUEFTU1dPUkRfNjRfTUFHSUMKICAgICAgLSBFTkNSWVBUSU9OX0tFWT0kU0VSVklDRV9QQVNTV09SRF82NF9FTkNSWVBUSU9OCiAgICAgIC0gU0VTU0lPTl9TRUNSRVQ9JFNFUlZJQ0VfUEFTU1dPUkRfNjRfU0VTU0lPTgogICAgICAtIFBPU1RHUkVTX1VTRVI9JFNFUlZJQ0VfVVNFUl9QT1NUR1JFUwogICAgICAtIFBPU1RHUkVTX1BBU1NXT1JEPSRTRVJWSUNFX1BBU1NXT1JEX1BPU1RHUkVTCiAgICAgIC0gJ1BPU1RHUkVTX0RCPSR7UE9TVEdSRVNfREI6LXRyaWdnZXJ9JwogICAgICAtIFBPU1RHUkVTX0hPU1Q9cG9zdGdyZXMKICAgICAgLSAnREFUQUJBU0VfVVJMPXBvc3RncmVzOi8vJFNFUlZJQ0VfVVNFUl9QT1NUR1JFUzokU0VSVklDRV9QQVNTV09SRF9QT1NUR1JFU0Bwb3N0Z3Jlc3FsOjU0MzIvJFBPU1RHUkVTX0RCJwogICAgICAtICdESVJFQ1RfVVJMPXBvc3RncmVzOi8vJFNFUlZJQ0VfVVNFUl9QT1NUR1JFUzokU0VSVklDRV9QQVNTV09SRF9QT1NUR1JFU0Bwb3N0Z3Jlc3FsOjU0MzIvJFBPU1RHUkVTX0RCJwogICAgICAtIFJVTlRJTUVfUExBVEZPUk09ZG9ja2VyLWNvbXBvc2UKICAgICAgLSBOT0RFX0VOVj1wcm9kdWN0aW9uCiAgICBkZXBlbmRzX29uOgogICAgICBwb3N0Z3Jlc3FsOgogICAgICAgIGNvbmRpdGlvbjogc2VydmljZV9oZWFsdGh5CiAgICBoZWFsdGhjaGVjazoKICAgICAgdGVzdDoKICAgICAgICAtIE5PTkUKICBwb3N0Z3Jlc3FsOgogICAgaW1hZ2U6ICdwb3N0Z3JlczoxNS1hbHBpbmUnCiAgICB2b2x1bWVzOgogICAgICAtICdwb3N0Z3Jlc3FsLWRhdGE6L3Zhci9saWIvcG9zdGdyZXNxbC9kYXRhJwogICAgZW52aXJvbm1lbnQ6CiAgICAgIC0gUE9TVEdSRVNfVVNFUj0kU0VSVklDRV9VU0VSX1BPU1RHUkVTCiAgICAgIC0gUE9TVEdSRVNfUEFTU1dPUkQ9JFNFUlZJQ0VfUEFTU1dPUkRfUE9TVEdSRVMKICAgICAgLSAnUE9TVEdSRVNfREI9JHtQT1NUR1JFU19EQjotdHJpZ2dlcn0nCiAgICBoZWFsdGhjaGVjazoKICAgICAgdGVzdDoKICAgICAgICAtIENNRC1TSEVMTAogICAgICAgIC0gJ3BnX2lzcmVhZHkgLVUgJCR7UE9TVEdSRVNfVVNFUn0gLWQgJCR7UE9TVEdSRVNfREJ9JwogICAgICBpbnRlcnZhbDogNXMKICAgICAgdGltZW91dDogMjBzCiAgICAgIHJldHJpZXM6IDEwCg==", + "tags": [ + "trigger.dev", + "background jobs", + "typescript", + "trigger", + "jobs", + "cron", + "scheduler" + ] + }, "umami": { "documentation": "https:\/\/umami.is\/docs\/getting-started", "slogan": "Umami is a lightweight, self-hosted web analytics platform designed to provide website owners with insights into visitor behavior without compromising user privacy.", From 3cf41e1e231dc0893834d5dc4046164cf6701c7d Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 23 Nov 2023 10:47:25 +0100 Subject: [PATCH 07/46] Update server basic value --- config/constants.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/constants.php b/config/constants.php index d125ce1cc..1ccae7116 100644 --- a/config/constants.php +++ b/config/constants.php @@ -26,7 +26,7 @@ return [ 'server' => [ 'zero' => 0, 'self-hosted' => 999999999999, - 'basic' => 1, + 'basic' => 2, 'pro' => 10, 'ultimate' => 25, ], From 72cfa3e7b0aae4681e997d227d4d2080ff8bdb81 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 23 Nov 2023 10:51:57 +0100 Subject: [PATCH 08/46] Update server limits using environment variables --- config/constants.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/constants.php b/config/constants.php index 1ccae7116..4febbf043 100644 --- a/config/constants.php +++ b/config/constants.php @@ -26,9 +26,9 @@ return [ 'server' => [ 'zero' => 0, 'self-hosted' => 999999999999, - 'basic' => 2, - 'pro' => 10, - 'ultimate' => 25, + 'basic' => env('LIMIT_SERVER_BASIC', 2), + 'pro' => env('LIMIT_SERVER_PRO', 10), + 'ultimate' => env('LIMIT_SERVER_ULTIMATE', 25), ], 'email' => [ 'zero' => true, From 97fd56b9e49fc47326fdfa9a67de94ee32b5d0a5 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 23 Nov 2023 10:57:11 +0100 Subject: [PATCH 09/46] Update number of servers in pricing plans --- resources/views/components/pricing-plans.blade.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/views/components/pricing-plans.blade.php b/resources/views/components/pricing-plans.blade.php index 4078b7976..aabf7bde6 100644 --- a/resources/views/components/pricing-plans.blade.php +++ b/resources/views/components/pricing-plans.blade.php @@ -81,7 +81,7 @@ d="M10 18a8 8 0 100-16 8 8 0 000 16zm3.857-9.809a.75.75 0 00-1.214-.882l-3.483 4.79-1.88-1.88a.75.75 0 10-1.06 1.061l2.5 2.5a.75.75 0 001.137-.089l4-5.5z" clip-rule="evenodd" /> - 1 server + 2 servers
  • Date: Thu, 23 Nov 2023 11:35:19 +0100 Subject: [PATCH 10/46] Update Weblate configuration --- templates/compose/weblate.yaml | 12 +++++++++--- templates/service-templates.json | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/templates/compose/weblate.yaml b/templates/compose/weblate.yaml index c09e8622c..524896e71 100644 --- a/templates/compose/weblate.yaml +++ b/templates/compose/weblate.yaml @@ -18,15 +18,18 @@ services: - POSTGRES_HOST=postgresql - POSTGRES_PORT=5432 - REDIS_HOST=redis + - REDIS_PORT=6379 + - REDIS_PASSWORD=$SERVICE_PASSWORD_REDIS volumes: - weblate-data:/app/data + - weblate-cache:/app/cache healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8080"] interval: 2s timeout: 10s - retries: 15 + retries: 30 postgresql: - image: postgres:15-alpine + image: postgres:16-alpine volumes: - postgresql-data:/var/lib/postgresql/data environment: @@ -40,7 +43,10 @@ services: retries: 10 redis: image: redis:7-alpine - command: redis-server --appendonly yes + command: > + --appendonly yes --requirepass ${SERVICE_PASSWORD_REDIS} + environment: + - REDIS_PASSWORD=$SERVICE_PASSWORD_REDIS volumes: - weblate-redis-data:/data healthcheck: diff --git a/templates/service-templates.json b/templates/service-templates.json index 36ee6895f..f7800141f 100644 --- a/templates/service-templates.json +++ b/templates/service-templates.json @@ -505,7 +505,7 @@ "weblate": { "documentation": "https:\/\/docs.weblate.org\/en\/latest\/admin\/install\/docker.html", "slogan": "Weblate is a libre software web-based continuous localization system.", - "compose": "c2VydmljZXM6CiAgd2VibGF0ZToKICAgIGltYWdlOiAnd2VibGF0ZS93ZWJsYXRlOmxhdGVzdCcKICAgIGVudmlyb25tZW50OgogICAgICAtIFNFUlZJQ0VfRlFETl9XRUJMQVRFCiAgICAgIC0gV0VCTEFURV9TSVRFX0RPTUFJTj0kU0VSVklDRV9VUkxfV0VCTEFURQogICAgICAtICdXRUJMQVRFX0FETUlOX05BTUU9JHtXRUJMQVRFX0FETUlOX05BTUU6LUFkbWlufScKICAgICAgLSAnV0VCTEFURV9BRE1JTl9FTUFJTD0ke1dFQkxBVEVfQURNSU5fRU1BSUw6LWFkbWluQGV4YW1wbGUuY29tfScKICAgICAgLSBXRUJMQVRFX0FETUlOX1BBU1NXT1JEPSRTRVJWSUNFX1BBU1NXT1JEX1dFQkxBVEUKICAgICAgLSAnREVGQVVMVF9GUk9NX0VNQUlMPSR7V0VCTEFURV9BRE1JTl9FTUFJTDotYWRtaW5AZXhhbXBsZS5jb219JwogICAgICAtIFBPU1RHUkVTX1BBU1NXT1JEPSRTRVJWSUNFX1BBU1NXT1JEX1BPU1RHUkVTCiAgICAgIC0gUE9TVEdSRVNfVVNFUj0kU0VSVklDRV9VU0VSX1BPU1RHUkVTCiAgICAgIC0gJ1BPU1RHUkVTX0RBVEFCQVNFPSR7UE9TVEdSRVNfREI6LXdlYmxhdGV9JwogICAgICAtIFBPU1RHUkVTX0hPU1Q9cG9zdGdyZXNxbAogICAgICAtIFBPU1RHUkVTX1BPUlQ9NTQzMgogICAgICAtIFJFRElTX0hPU1Q9cmVkaXMKICAgIHZvbHVtZXM6CiAgICAgIC0gJ3dlYmxhdGUtZGF0YTovYXBwL2RhdGEnCiAgICBoZWFsdGhjaGVjazoKICAgICAgdGVzdDoKICAgICAgICAtIENNRAogICAgICAgIC0gY3VybAogICAgICAgIC0gJy1mJwogICAgICAgIC0gJ2h0dHA6Ly9sb2NhbGhvc3Q6ODA4MCcKICAgICAgaW50ZXJ2YWw6IDJzCiAgICAgIHRpbWVvdXQ6IDEwcwogICAgICByZXRyaWVzOiAxNQogIHBvc3RncmVzcWw6CiAgICBpbWFnZTogJ3Bvc3RncmVzOjE1LWFscGluZScKICAgIHZvbHVtZXM6CiAgICAgIC0gJ3Bvc3RncmVzcWwtZGF0YTovdmFyL2xpYi9wb3N0Z3Jlc3FsL2RhdGEnCiAgICBlbnZpcm9ubWVudDoKICAgICAgLSBQT1NUR1JFU19VU0VSPSRTRVJWSUNFX1VTRVJfUE9TVEdSRVMKICAgICAgLSBQT1NUR1JFU19QQVNTV09SRD0kU0VSVklDRV9QQVNTV09SRF9QT1NUR1JFUwogICAgICAtICdQT1NUR1JFU19EQj0ke1BPU1RHUkVTX0RCOi13ZWJsYXRlfScKICAgIGhlYWx0aGNoZWNrOgogICAgICB0ZXN0OgogICAgICAgIC0gQ01ELVNIRUxMCiAgICAgICAgLSAncGdfaXNyZWFkeSAtVSAkJHtQT1NUR1JFU19VU0VSfSAtZCAkJHtQT1NUR1JFU19EQn0nCiAgICAgIGludGVydmFsOiA1cwogICAgICB0aW1lb3V0OiAyMHMKICAgICAgcmV0cmllczogMTAKICByZWRpczoKICAgIGltYWdlOiAncmVkaXM6Ny1hbHBpbmUnCiAgICBjb21tYW5kOiAncmVkaXMtc2VydmVyIC0tYXBwZW5kb25seSB5ZXMnCiAgICB2b2x1bWVzOgogICAgICAtICd3ZWJsYXRlLXJlZGlzLWRhdGE6L2RhdGEnCiAgICBoZWFsdGhjaGVjazoKICAgICAgdGVzdDoKICAgICAgICAtIENNRAogICAgICAgIC0gcmVkaXMtY2xpCiAgICAgICAgLSBwaW5nCiAgICAgIGludGVydmFsOiA1cwogICAgICB0aW1lb3V0OiAyMHMKICAgICAgcmV0cmllczogMTAK", + "compose": "c2VydmljZXM6CiAgd2VibGF0ZToKICAgIGltYWdlOiAnd2VibGF0ZS93ZWJsYXRlOmxhdGVzdCcKICAgIGVudmlyb25tZW50OgogICAgICAtIFNFUlZJQ0VfRlFETl9XRUJMQVRFCiAgICAgIC0gV0VCTEFURV9TSVRFX0RPTUFJTj0kU0VSVklDRV9VUkxfV0VCTEFURQogICAgICAtICdXRUJMQVRFX0FETUlOX05BTUU9JHtXRUJMQVRFX0FETUlOX05BTUU6LUFkbWlufScKICAgICAgLSAnV0VCTEFURV9BRE1JTl9FTUFJTD0ke1dFQkxBVEVfQURNSU5fRU1BSUw6LWFkbWluQGV4YW1wbGUuY29tfScKICAgICAgLSBXRUJMQVRFX0FETUlOX1BBU1NXT1JEPSRTRVJWSUNFX1BBU1NXT1JEX1dFQkxBVEUKICAgICAgLSAnREVGQVVMVF9GUk9NX0VNQUlMPSR7V0VCTEFURV9BRE1JTl9FTUFJTDotYWRtaW5AZXhhbXBsZS5jb219JwogICAgICAtIFBPU1RHUkVTX1BBU1NXT1JEPSRTRVJWSUNFX1BBU1NXT1JEX1BPU1RHUkVTCiAgICAgIC0gUE9TVEdSRVNfVVNFUj0kU0VSVklDRV9VU0VSX1BPU1RHUkVTCiAgICAgIC0gJ1BPU1RHUkVTX0RBVEFCQVNFPSR7UE9TVEdSRVNfREI6LXdlYmxhdGV9JwogICAgICAtIFBPU1RHUkVTX0hPU1Q9cG9zdGdyZXNxbAogICAgICAtIFBPU1RHUkVTX1BPUlQ9NTQzMgogICAgICAtIFJFRElTX0hPU1Q9cmVkaXMKICAgICAgLSBSRURJU19QT1JUPTYzNzkKICAgICAgLSBSRURJU19QQVNTV09SRD0kU0VSVklDRV9QQVNTV09SRF9SRURJUwogICAgdm9sdW1lczoKICAgICAgLSAnd2VibGF0ZS1kYXRhOi9hcHAvZGF0YScKICAgICAgLSAnd2VibGF0ZS1jYWNoZTovYXBwL2NhY2hlJwogICAgaGVhbHRoY2hlY2s6CiAgICAgIHRlc3Q6CiAgICAgICAgLSBDTUQKICAgICAgICAtIGN1cmwKICAgICAgICAtICctZicKICAgICAgICAtICdodHRwOi8vbG9jYWxob3N0OjgwODAnCiAgICAgIGludGVydmFsOiAycwogICAgICB0aW1lb3V0OiAxMHMKICAgICAgcmV0cmllczogMzAKICBwb3N0Z3Jlc3FsOgogICAgaW1hZ2U6ICdwb3N0Z3JlczoxNi1hbHBpbmUnCiAgICB2b2x1bWVzOgogICAgICAtICdwb3N0Z3Jlc3FsLWRhdGE6L3Zhci9saWIvcG9zdGdyZXNxbC9kYXRhJwogICAgZW52aXJvbm1lbnQ6CiAgICAgIC0gUE9TVEdSRVNfVVNFUj0kU0VSVklDRV9VU0VSX1BPU1RHUkVTCiAgICAgIC0gUE9TVEdSRVNfUEFTU1dPUkQ9JFNFUlZJQ0VfUEFTU1dPUkRfUE9TVEdSRVMKICAgICAgLSAnUE9TVEdSRVNfREI9JHtQT1NUR1JFU19EQjotd2VibGF0ZX0nCiAgICBoZWFsdGhjaGVjazoKICAgICAgdGVzdDoKICAgICAgICAtIENNRC1TSEVMTAogICAgICAgIC0gJ3BnX2lzcmVhZHkgLVUgJCR7UE9TVEdSRVNfVVNFUn0gLWQgJCR7UE9TVEdSRVNfREJ9JwogICAgICBpbnRlcnZhbDogNXMKICAgICAgdGltZW91dDogMjBzCiAgICAgIHJldHJpZXM6IDEwCiAgcmVkaXM6CiAgICBpbWFnZTogJ3JlZGlzOjctYWxwaW5lJwogICAgY29tbWFuZDogIi0tYXBwZW5kb25seSB5ZXMgLS1yZXF1aXJlcGFzcyAke1NFUlZJQ0VfUEFTU1dPUkRfUkVESVN9XG4iCiAgICBlbnZpcm9ubWVudDoKICAgICAgLSBSRURJU19QQVNTV09SRD0kU0VSVklDRV9QQVNTV09SRF9SRURJUwogICAgdm9sdW1lczoKICAgICAgLSAnd2VibGF0ZS1yZWRpcy1kYXRhOi9kYXRhJwogICAgaGVhbHRoY2hlY2s6CiAgICAgIHRlc3Q6CiAgICAgICAgLSBDTUQKICAgICAgICAtIHJlZGlzLWNsaQogICAgICAgIC0gcGluZwogICAgICBpbnRlcnZhbDogNXMKICAgICAgdGltZW91dDogMjBzCiAgICAgIHJldHJpZXM6IDEwCg==", "tags": [ "localization", "translation", From 16261fc36e7c8a47a54cd962dc365bb7d4428e96 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 23 Nov 2023 11:40:29 +0100 Subject: [PATCH 11/46] Remove unnecessary code and update services list loading --- app/Http/Livewire/Project/New/Select.php | 1 - resources/views/livewire/project/new/select.blade.php | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/app/Http/Livewire/Project/New/Select.php b/app/Http/Livewire/Project/New/Select.php index a4b80ce69..934afd9df 100644 --- a/app/Http/Livewire/Project/New/Select.php +++ b/app/Http/Livewire/Project/New/Select.php @@ -47,7 +47,6 @@ class Select extends Component } public function render() { - $this->loadServices(); return view('livewire.project.new.select'); } diff --git a/resources/views/livewire/project/new/select.blade.php b/resources/views/livewire/project/new/select.blade.php index 05b1aa6db..9ffa76e74 100644 --- a/resources/views/livewire/project/new/select.blade.php +++ b/resources/views/livewire/project/new/select.blade.php @@ -145,7 +145,7 @@ --}} -
    +

    Services

    Reload Services List Date: Thu, 23 Nov 2023 11:49:49 +0100 Subject: [PATCH 12/46] Fix service loading issue in project select page --- app/Http/Livewire/Project/New/Select.php | 5 +++-- resources/views/livewire/project/new/select.blade.php | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/app/Http/Livewire/Project/New/Select.php b/app/Http/Livewire/Project/New/Select.php index 934afd9df..13fbb8886 100644 --- a/app/Http/Livewire/Project/New/Select.php +++ b/app/Http/Livewire/Project/New/Select.php @@ -47,6 +47,7 @@ class Select extends Component } public function render() { + if ($this->search) $this->loadServices(); return view('livewire.project.new.select'); } @@ -68,10 +69,10 @@ class Select extends Component // } // } - public function loadServices(bool $force = false) + public function loadServices() { try { - if (count($this->allServices) > 0 && !$force) { + if (count($this->allServices) > 0) { if (!$this->search) { $this->services = $this->allServices; return; diff --git a/resources/views/livewire/project/new/select.blade.php b/resources/views/livewire/project/new/select.blade.php index 9ffa76e74..60da34cd1 100644 --- a/resources/views/livewire/project/new/select.blade.php +++ b/resources/views/livewire/project/new/select.blade.php @@ -145,9 +145,9 @@
    --}} -
    +

    Services

    - Reload Services List + Reload Services List From 96327af83814d4b3b7d0fc87096181a68c0e7210 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 23 Nov 2023 12:44:08 +0100 Subject: [PATCH 13/46] Update log-drains.blade.php and add trigger-with-external-database.yaml and service-templates.json --- .../livewire/server/log-drains.blade.php | 37 ++++++++++--------- ...ml => trigger-with-external-database.yaml} | 0 templates/service-templates.json | 2 +- 3 files changed, 20 insertions(+), 19 deletions(-) rename templates/compose/{trigger-without-database.yaml => trigger-with-external-database.yaml} (100%) diff --git a/resources/views/livewire/server/log-drains.blade.php b/resources/views/livewire/server/log-drains.blade.php index 2e1109dc7..465702053 100644 --- a/resources/views/livewire/server/log-drains.blade.php +++ b/resources/views/livewire/server/log-drains.blade.php @@ -24,24 +24,7 @@
    - {{--

    Highlight.io

    -
    - -
    -
    -
    -
    - -
    -
    -
    - - Save - -
    -
    --}} +

    Axiom

    + {{--

    Highlight.io

    +
    + +
    +
    +
    +
    + +
    +
    +
    + + Save + +
    +
    --}}
    diff --git a/templates/compose/trigger-without-database.yaml b/templates/compose/trigger-with-external-database.yaml similarity index 100% rename from templates/compose/trigger-without-database.yaml rename to templates/compose/trigger-with-external-database.yaml diff --git a/templates/service-templates.json b/templates/service-templates.json index f7800141f..8c2c55a93 100644 --- a/templates/service-templates.json +++ b/templates/service-templates.json @@ -441,7 +441,7 @@ "internet" ] }, - "trigger-without-database": { + "trigger-with-external-database": { "documentation": "https:\/\/trigger.dev\/docs\/documentation\/guides\/self-hosting", "slogan": "The open source Background Jobs framework for TypeScript", "compose": "c2VydmljZXM6CiAgdHJpZ2dlcjoKICAgIGltYWdlOiAnZ2hjci5pby90cmlnZ2VyZG90ZGV2L3RyaWdnZXIuZGV2OmxhdGVzdCcKICAgIGVudmlyb25tZW50OgogICAgICAtIFNFUlZJQ0VfRlFETl9UUklHR0VSCiAgICAgIC0gTE9HSU5fT1JJR0lOPSRTRVJWSUNFX0ZRRE5fVFJJR0dFUgogICAgICAtIEFQUF9PUklHSU49JFNFUlZJQ0VfRlFETl9UUklHR0VSCiAgICAgIC0gTUFHSUNfTElOS19TRUNSRVQ9JFNFUlZJQ0VfUEFTU1dPUkRfNjRfTUFHSUMKICAgICAgLSBFTkNSWVBUSU9OX0tFWT0kU0VSVklDRV9QQVNTV09SRF82NF9FTkNSWVBUSU9OCiAgICAgIC0gU0VTU0lPTl9TRUNSRVQ9JFNFUlZJQ0VfUEFTU1dPUkRfNjRfU0VTU0lPTgogICAgICAtICdEQVRBQkFTRV9VUkw9JHtEQVRBQkFTRV9VUkx9JwogICAgICAtICdESVJFQ1RfVVJMPSR7REFUQUJBU0VfVVJMfScKICAgICAgLSBSVU5USU1FX1BMQVRGT1JNPWRvY2tlci1jb21wb3NlCiAgICAgIC0gTk9ERV9FTlY9cHJvZHVjdGlvbgogICAgaGVhbHRoY2hlY2s6CiAgICAgIHRlc3Q6CiAgICAgICAgLSBOT05FCg==", From d1e10dacc05e952a5a447fefd403f428d73de8ad Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 23 Nov 2023 21:02:30 +0100 Subject: [PATCH 14/46] wip --- app/Jobs/ApplicationDeployDockerImageJob.php | 122 ++++++++++++------ app/Models/Application.php | 16 +++ app/Models/ApplicationDeploymentQueue.php | 35 +++++ app/Models/Server.php | 50 ++++--- bootstrap/helpers/applications.php | 84 ++++++------ bootstrap/helpers/remoteProcess.php | 4 +- .../application/deployment-logs.blade.php | 2 +- 7 files changed, 214 insertions(+), 99 deletions(-) diff --git a/app/Jobs/ApplicationDeployDockerImageJob.php b/app/Jobs/ApplicationDeployDockerImageJob.php index 24526937a..c21ad5680 100644 --- a/app/Jobs/ApplicationDeployDockerImageJob.php +++ b/app/Jobs/ApplicationDeployDockerImageJob.php @@ -5,8 +5,8 @@ namespace App\Jobs; use App\Enums\ApplicationDeploymentStatus; use App\Models\Application; use App\Models\ApplicationDeploymentQueue; +use App\Models\Server; use App\Traits\ExecuteRemoteCommandNew; -use Exception; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldQueue; @@ -21,77 +21,117 @@ class ApplicationDeployDockerImageJob implements ShouldQueue, ShouldBeEncrypted public $timeout = 3600; public $tries = 1; - public string $applicationDeploymentQueueId; + public $remoteCommandOutputs = []; + public Server $server; + public string $containerName; - public function __construct(string $applicationDeploymentQueueId) + public function __construct(public ApplicationDeploymentQueue $deploymentQueueEntry, public Application $application) { - $this->applicationDeploymentQueueId = $applicationDeploymentQueueId; } public function handle() { ray()->clearAll(); ray('Deploying Docker Image'); + static::$batch_counter = 0; try { - $applicationDeploymentQueue = ApplicationDeploymentQueue::find($this->applicationDeploymentQueueId); + $deploymentUuid = data_get($this->deploymentQueueEntry, 'deployment_uuid'); + $pullRequestId = data_get($this->deploymentQueueEntry, 'pull_request_id'); - $deploymentUuid = data_get($applicationDeploymentQueue, 'deployment_uuid'); - $pullRequestId = data_get($applicationDeploymentQueue, 'pull_request_id'); + $this->server = data_get($this->application->destination, 'server'); + $network = data_get($this->application->destination, 'network'); - $application = Application::find($applicationDeploymentQueue->application_id)->firstOrFail(); - $server = data_get($application->destination, 'server'); - $network = data_get($application->destination, 'network'); - - $dockerImage = data_get($application, 'docker_registry_image_name'); - $dockerImageTag = data_get($application, 'docker_registry_image_tag'); + $dockerImage = data_get($this->application, 'docker_registry_image_name'); + $dockerImageTag = data_get($this->application, 'docker_registry_image_tag'); $productionImageName = str("{$dockerImage}:{$dockerImageTag}"); - - $containerName = generateApplicationContainerName($application, $pullRequestId); - savePrivateKeyToFs($server); + $this->containerName = generateApplicationContainerName($this->application, $pullRequestId); + savePrivateKeyToFs($this->server); ray("echo 'Starting deployment of {$productionImageName}.'"); - $applicationDeploymentQueue->update([ + $this->deploymentQueueEntry->update([ 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value, ]); - $server->executeRemoteCommand( - commands: prepareHelperContainer($server, $network, $deploymentUuid), - loggingModel: $applicationDeploymentQueue + + $this->deploymentQueueEntry->addLogEntry('Starting deployment of ' . $productionImageName); + + $this->server->executeRemoteCommand( + commands: collect( + [ + [ + "name" => "ls", + "command" => 'ls -la', + "hidden" => true, + ], + [ + "name" => "pwd", + "command" => 'pwd', + "hidden" => true, + ] + ], + ), + loggingModel: $this->deploymentQueueEntry ); - $server->executeRemoteCommand( + $this->server->executeRemoteCommand( + commands: prepareHelperContainer($this->server, $network, $deploymentUuid), + loggingModel: $this->deploymentQueueEntry + ); + $this->server->executeRemoteCommand( commands: generateComposeFile( deploymentUuid: $deploymentUuid, - server: $server, + server: $this->server, network: $network, - application: $application, - containerName: $containerName, + application: $this->application, + containerName: $this->containerName, imageName: $productionImageName, pullRequestId: $pullRequestId ), - loggingModel: $applicationDeploymentQueue + loggingModel: $this->deploymentQueueEntry ); - $server->executeRemoteCommand( - commands: rollingUpdate(application: $application, deploymentUuid: $deploymentUuid), - loggingModel: $applicationDeploymentQueue - ); - $applicationDeploymentQueue->update([ + $this->deploymentQueueEntry->addLogEntry('----------------------------------------'); + + // Rolling update not possible + if (count($this->application->ports_mappings_array) > 0) { + $this->deploymentQueueEntry->addLogEntry('Application has ports mapped to the host system, rolling update is not supported.'); + $this->deploymentQueueEntry->addLogEntry('Stopping running container.'); + $this->server->stopApplicationRelatedRunningContainers($this->application->id, $this->containerName); + } else { + $this->deploymentQueueEntry->addLogEntry('Rolling update started.'); + // TODO + $this->server->executeRemoteCommand( + commands: startNewApplication(application: $this->application, deploymentUuid: $deploymentUuid, loggingModel: $this->deploymentQueueEntry), + loggingModel: $this->deploymentQueueEntry + ); + // $this->server->executeRemoteCommand( + // commands: healthCheckContainer(application: $this->application, containerName: $this->containerName , loggingModel: $this->deploymentQueueEntry), + // loggingModel: $this->deploymentQueueEntry + // ); + + } + + ray($this->remoteCommandOutputs); + $this->deploymentQueueEntry->update([ 'status' => ApplicationDeploymentStatus::FINISHED->value, ]); } catch (Throwable $e) { - $this->executeRemoteCommand( - server: $server, - logModel: $applicationDeploymentQueue, - commands: [ - "echo 'Oops something is not okay, are you okay? 😢'", - "echo '{$e->getMessage()}'", - "echo -n 'Deployment failed. Removing the new version of your application.'", - executeInDocker($deploymentUuid, "docker rm -f $containerName >/dev/null 2>&1"), - ] - ); - // $this->next(ApplicationDeploymentStatus::FAILED->value); + $this->fail($e); throw $e; } } + public function failed(Throwable $exception): void + { + $this->deploymentQueueEntry->addLogEntry('Oops something is not okay, are you okay? 😢', 'error'); + $this->deploymentQueueEntry->addLogEntry($exception->getMessage(), 'error'); + $this->deploymentQueueEntry->addLogEntry('Deployment failed. Removing the new version of your application.'); + + $this->server->executeRemoteCommand( + commands: removeOldDeployment($this->containerName), + loggingModel: $this->deploymentQueueEntry + ); + $this->deploymentQueueEntry->update([ + 'status' => ApplicationDeploymentStatus::FAILED->value, + ]); + } // private function next(string $status) // { // // If the deployment is cancelled by the user, don't update the status diff --git a/app/Models/Application.php b/app/Models/Application.php index 785ef3040..815376d18 100644 --- a/app/Models/Application.php +++ b/app/Models/Application.php @@ -342,4 +342,20 @@ class Application extends BaseModel } return false; } + public function healthCheckUrl() { + if ($this->dockerfile || $this->build_pack === 'dockerfile' || $this->build_pack === 'dockerimage') { + return null; + } + if (!$this->health_check_port) { + $health_check_port = $this->ports_exposes_array[0]; + } else { + $health_check_port = $this->health_check_port; + } + if ($this->health_check_path) { + $full_healthcheck_url = "{$this->health_check_scheme}://{$this->health_check_host}:{$health_check_port}{$this->health_check_path}"; + } else { + $full_healthcheck_url = "{$this->health_check_scheme}://{$this->health_check_host}:{$health_check_port}/"; + } + return $full_healthcheck_url; + } } diff --git a/app/Models/ApplicationDeploymentQueue.php b/app/Models/ApplicationDeploymentQueue.php index 1b7ae4781..bc9b9471a 100644 --- a/app/Models/ApplicationDeploymentQueue.php +++ b/app/Models/ApplicationDeploymentQueue.php @@ -3,8 +3,43 @@ namespace App\Models; use Illuminate\Database\Eloquent\Model; +use Illuminate\Support\Carbon; class ApplicationDeploymentQueue extends Model { protected $guarded = []; + + public function getOutput($name) { + if (!$this->logs) { + return null; + } + return collect(json_decode($this->logs))->where('name', $name)->first()?->output ?? null; + } + + public function addLogEntry(string $message, string $type = 'stdout', bool $hidden = false) + { + if ($type === 'error') { + $type = 'stderr'; + } + $newLogEntry = [ + 'command' => null, + 'output' => $message, + 'type' => $type, + 'timestamp' => Carbon::now('UTC'), + 'hidden' => $hidden, + 'batch' => 1, + ]; + if ($this->logs) { + $previousLogs = json_decode($this->logs, associative: true, flags: JSON_THROW_ON_ERROR); + $newLogEntry['order'] = count($previousLogs) + 1; + $previousLogs[] = $newLogEntry; + $this->update([ + 'logs' => json_encode($previousLogs, flags: JSON_THROW_ON_ERROR), + ]); + } else { + $this->update([ + 'logs' => json_encode([$newLogEntry], flags: JSON_THROW_ON_ERROR), + ]); + } + } } diff --git a/app/Models/Server.php b/app/Models/Server.php index 50e69f3b3..7811b1aaa 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -396,23 +396,23 @@ class Server extends BaseModel { static::$batch_counter++; foreach ($commands as $command) { - $command = data_get($command, 'command') ?? $command[0] ?? null; - if (is_null($command)) { - continue; + $realCommand = data_get($command, 'command'); + if (is_null($realCommand)) { + throw new \RuntimeException('Command is not set'); } $hidden = data_get($command, 'hidden', false); $ignoreErrors = data_get($command, 'ignoreErrors', false); $customOutputType = data_get($command, 'customOutputType'); - $saveOutput = data_get($command, 'saveOutput'); - $remoteCommand = generateSshCommand($this, $command); + $name = data_get($command, 'name'); + $remoteCommand = generateSshCommand($this, $realCommand); - $process = Process::timeout(3600)->idleTimeout(3600)->start($remoteCommand, function (string $type, string $output) use ($command, $hidden, $customOutputType, $loggingModel, $saveOutput) { + $process = Process::timeout(3600)->idleTimeout(3600)->start($remoteCommand, function (string $type, string $output) use ($realCommand, $hidden, $customOutputType, $loggingModel, $name) { $output = str($output)->trim(); if ($output->startsWith('╔')) { $output = "\n" . $output; } $newLogEntry = [ - 'command' => remove_iip($command), + 'command' => remove_iip($realCommand), 'output' => remove_iip($output), 'type' => $customOutputType ?? $type === 'err' ? 'stderr' : 'stdout', 'timestamp' => Carbon::now('UTC'), @@ -420,19 +420,21 @@ class Server extends BaseModel 'batch' => static::$batch_counter, ]; if (!$loggingModel->logs) { - $new_log_entry['order'] = 1; + $newLogEntry['order'] = 1; } else { - $previous_logs = json_decode($this->log_model->logs, associative: true, flags: JSON_THROW_ON_ERROR); - $new_log_entry['order'] = count($previous_logs) + 1; + $previousLogs = json_decode($loggingModel->logs, associative: true, flags: JSON_THROW_ON_ERROR); + $newLogEntry['order'] = count($previousLogs) + 1; } - $previousLogs = json_decode($loggingModel->logs, associative: true, flags: JSON_THROW_ON_ERROR); - $newLogEntry['order'] = count($previousLogs) + 1; + if ($name) { + $newLogEntry['name'] = $name; + } + $previousLogs[] = $newLogEntry; $loggingModel->logs = json_encode($previousLogs, flags: JSON_THROW_ON_ERROR); $loggingModel->save(); - if ($saveOutput) { - $this->remoteCommandOutputs[$saveOutput] = str($output)->trim(); - } + // if ($name) { + // $loggingModel['savedOutputs'][$name] = str($output)->trim(); + // } }); $loggingModel->update([ 'current_process_id' => $process->id(), @@ -449,4 +451,22 @@ class Server extends BaseModel } } } + public function stopApplicationRelatedRunningContainers(string $applicationId, string $containerName) + { + $containers = getCurrentApplicationContainerStatus($this, $applicationId, 0); + $containers = $containers->filter(function ($container) use ($containerName) { + return data_get($container, 'Names') !== $containerName; + }); + $containers->each(function ($container) { + $removableContainer = data_get($container, 'Names'); + $this->server->executeRemoteCommand( + commands: collect([ + 'command' => "docker rm -f $removableContainer >/dev/null 2>&1", + 'hidden' => true, + 'ignoreErrors' => true + ]), + loggingModel: $this->deploymentQueueEntry + ); + }); + } } diff --git a/bootstrap/helpers/applications.php b/bootstrap/helpers/applications.php index 1cead8a54..17c3aa486 100644 --- a/bootstrap/helpers/applications.php +++ b/bootstrap/helpers/applications.php @@ -37,7 +37,9 @@ function queue_application_deployment(int $application_id, string $deployment_uu return; } // New deployment - // dispatchDeploymentJob($deployment->id); + // dispatchDeploymentJob($deployment); + + // Old deployment dispatch(new ApplicationDeploymentJob( application_deployment_queue_id: $deployment->id, ))->onConnection('long-running')->onQueue('long-running'); @@ -48,29 +50,34 @@ function queue_next_deployment(Application $application) { $next_found = ApplicationDeploymentQueue::where('application_id', $application->id)->where('status', 'queued')->first(); if ($next_found) { - // New deployment + // New deployment // dispatchDeploymentJob($next_found->id); + + // Old deployment dispatch(new ApplicationDeploymentJob( application_deployment_queue_id: $next_found->id, ))->onConnection('long-running')->onQueue('long-running'); - } } -function dispatchDeploymentJob($id) +function dispatchDeploymentJob(ApplicationDeploymentQueue $deploymentQueueEntry) { - $applicationQueue = ApplicationDeploymentQueue::find($id); - $application = Application::find($applicationQueue->application_id); + $application = Application::find($deploymentQueueEntry->application_id); - $isRestartOnly = data_get($applicationQueue, 'restart_only'); + $isRestartOnly = data_get($deploymentQueueEntry, 'restart_only'); $isSimpleDockerFile = data_get($application, 'dockerfile'); $isDockerImage = data_get($application, 'build_pack') === 'dockerimage'; - if ($isRestartOnly) { - ApplicationRestartJob::dispatch(applicationDeploymentQueueId: $id)->onConnection('long-running')->onQueue('long-running'); - } else if ($isSimpleDockerFile) { - ApplicationDeploySimpleDockerfileJob::dispatch(applicationDeploymentQueueId: $id)->onConnection('long-running')->onQueue('long-running'); - } else if ($isDockerImage) { - ApplicationDeployDockerImageJob::dispatch(applicationDeploymentQueueId: $id)->onConnection('long-running')->onQueue('long-running'); + // if ($isRestartOnly) { + // ApplicationRestartJob::dispatch(queue: $deploymentQueueEntry, application: $application)->onConnection('long-running')->onQueue('long-running'); + // } else if ($isSimpleDockerFile) { + // ApplicationDeploySimpleDockerfileJob::dispatch(applicationDeploymentQueueId: $id)->onConnection('long-running')->onQueue('long-running'); + // } else + + if ($isDockerImage) { + ApplicationDeployDockerImageJob::dispatch( + deploymentQueueEntry: $deploymentQueueEntry, + application: $application + )->onConnection('long-running')->onQueue('long-running'); } else { throw new Exception('Unknown build pack'); } @@ -201,7 +208,6 @@ function generateComposeFile(string $deploymentUuid, Server $server, string $net ]; } if ($application->settings->is_gpu_enabled) { - ray('asd'); $docker_compose['services'][$containerName]['deploy']['resources']['reservations']['devices'] = [ [ 'driver' => data_get($application, 'settings.gpu_driver', 'nvidia'), @@ -302,39 +308,37 @@ function generateEnvironmentVariables(Application $application, $ports, int $pul return $environment_variables->all(); } -function rollingUpdate(Application $application, string $deploymentUuid) +function startNewApplication(Application $application, string $deploymentUuid, ApplicationDeploymentQueue $loggingModel) { $commands = collect([]); $workDir = generateWorkdir($deploymentUuid, $application); - if (count($application->ports_mappings_array) > 0) { - // $this->execute_remote_command( - // [ - // "echo '\n----------------------------------------'", - // ], - // ["echo -n 'Application has ports mapped to the host system, rolling update is not supported.'"], - // ); - // $this->stop_running_container(force: true); - // $this->start_by_compose_file(); + if ($application->build_pack === 'dockerimage') { + $loggingModel->addLogEntry('Pulling latest images from the registry.'); + $commands->push( + [ + "command" => executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} pull"), + "hidden" => true + ], + [ + "command" => executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} up --build -d"), + "hidden" => true + ], + ); } else { $commands->push( [ - "command" => "echo '\n----------------------------------------'" + "command" => executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} up --build -d"), + "hidden" => true ], - [ - "command" => "echo -n 'Rolling update started.'" - ] ); - if ($application->build_pack === 'dockerimage') { - $commands->push( - ["echo -n 'Pulling latest images from the registry.'"], - [executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} pull"), "hidden" => true], - [executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} up --build -d"), "hidden" => true], - ); - } else { - $commands->push( - [executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} up --build -d"), "hidden" => true], - ); - } - return $commands; } + return $commands; +} +function removeOldDeployment(string $containerName) +{ + $commands = collect([]); + $commands->push( + ["docker rm -f $containerName >/dev/null 2>&1"], + ); + return $commands; } diff --git a/bootstrap/helpers/remoteProcess.php b/bootstrap/helpers/remoteProcess.php index 948e47329..4f755c89e 100644 --- a/bootstrap/helpers/remoteProcess.php +++ b/bootstrap/helpers/remoteProcess.php @@ -165,9 +165,9 @@ function decode_remote_command_output(?ApplicationDeploymentQueue $application_d $formatted = $formatted->filter(fn ($i) => $i['hidden'] === false ?? false); } $formatted = $formatted - ->sortBy(fn ($i) => $i['order']) + ->sortBy(fn ($i) => data_get($i, 'order')) ->map(function ($i) { - $i['timestamp'] = Carbon::parse($i['timestamp'])->format('Y-M-d H:i:s.u'); + data_set($i, 'timestamp', Carbon::parse(data_get($i, 'timestamp'))->format('Y-M-d H:i:s.u')); return $i; }); return $formatted; diff --git a/resources/views/livewire/project/application/deployment-logs.blade.php b/resources/views/livewire/project/application/deployment-logs.blade.php index 09cb8232e..6afabbe13 100644 --- a/resources/views/livewire/project/application/deployment-logs.blade.php +++ b/resources/views/livewire/project/application/deployment-logs.blade.php @@ -49,7 +49,7 @@
    $line['hidden'], - 'text-error' => $line['type'] == 'stderr', + 'text-red-500' => $line['type'] == 'stderr', ])>[{{ $line['timestamp'] }}] @if ($line['hidden'])
    COMMAND:
    {{ $line['command'] }}

    OUTPUT: @endif{{ $line['output'] }}@if ($line['hidden']) From c5a932ab8803d4fb6cd9ac41208d1f00c2bdf70c Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Fri, 24 Nov 2023 08:38:49 +0100 Subject: [PATCH 15/46] Add environment variables for GitHub authentication and email configuration --- templates/compose/trigger-with-external-database.yaml | 5 +++++ templates/compose/trigger.yaml | 5 +++++ templates/service-templates.json | 4 ++-- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/templates/compose/trigger-with-external-database.yaml b/templates/compose/trigger-with-external-database.yaml index 4fc1f8992..53fa7ffd0 100644 --- a/templates/compose/trigger-with-external-database.yaml +++ b/templates/compose/trigger-with-external-database.yaml @@ -16,5 +16,10 @@ services: - DIRECT_URL=${DATABASE_URL} - RUNTIME_PLATFORM=docker-compose - NODE_ENV=production + - AUTH_GITHUB_CLIENT_ID=${AUTH_GITHUB_CLIENT_ID} + - AUTH_GITHUB_CLIENT_SECRET=${AUTH_GITHUB_CLIENT_SECRET} + - RESEND_API_KEY=${RESEND_API_KEY} + - FROM_EMAIL=${FROM_EMAIL} + - REPLY_TO_EMAIL=${REPLY_TO_EMAIL} healthcheck: test: ["NONE"] diff --git a/templates/compose/trigger.yaml b/templates/compose/trigger.yaml index b297fea63..7f2bd8167 100644 --- a/templates/compose/trigger.yaml +++ b/templates/compose/trigger.yaml @@ -20,6 +20,11 @@ services: - DIRECT_URL=postgres://$SERVICE_USER_POSTGRES:$SERVICE_PASSWORD_POSTGRES@postgresql:5432/$POSTGRES_DB - RUNTIME_PLATFORM=docker-compose - NODE_ENV=production + - AUTH_GITHUB_CLIENT_ID=${AUTH_GITHUB_CLIENT_ID} + - AUTH_GITHUB_CLIENT_SECRET=${AUTH_GITHUB_CLIENT_SECRET} + - RESEND_API_KEY=${RESEND_API_KEY} + - FROM_EMAIL=${FROM_EMAIL} + - REPLY_TO_EMAIL=${REPLY_TO_EMAIL} depends_on: postgresql: condition: service_healthy diff --git a/templates/service-templates.json b/templates/service-templates.json index 8c2c55a93..3b5b7c4e7 100644 --- a/templates/service-templates.json +++ b/templates/service-templates.json @@ -444,7 +444,7 @@ "trigger-with-external-database": { "documentation": "https:\/\/trigger.dev\/docs\/documentation\/guides\/self-hosting", "slogan": "The open source Background Jobs framework for TypeScript", - "compose": "c2VydmljZXM6CiAgdHJpZ2dlcjoKICAgIGltYWdlOiAnZ2hjci5pby90cmlnZ2VyZG90ZGV2L3RyaWdnZXIuZGV2OmxhdGVzdCcKICAgIGVudmlyb25tZW50OgogICAgICAtIFNFUlZJQ0VfRlFETl9UUklHR0VSCiAgICAgIC0gTE9HSU5fT1JJR0lOPSRTRVJWSUNFX0ZRRE5fVFJJR0dFUgogICAgICAtIEFQUF9PUklHSU49JFNFUlZJQ0VfRlFETl9UUklHR0VSCiAgICAgIC0gTUFHSUNfTElOS19TRUNSRVQ9JFNFUlZJQ0VfUEFTU1dPUkRfNjRfTUFHSUMKICAgICAgLSBFTkNSWVBUSU9OX0tFWT0kU0VSVklDRV9QQVNTV09SRF82NF9FTkNSWVBUSU9OCiAgICAgIC0gU0VTU0lPTl9TRUNSRVQ9JFNFUlZJQ0VfUEFTU1dPUkRfNjRfU0VTU0lPTgogICAgICAtICdEQVRBQkFTRV9VUkw9JHtEQVRBQkFTRV9VUkx9JwogICAgICAtICdESVJFQ1RfVVJMPSR7REFUQUJBU0VfVVJMfScKICAgICAgLSBSVU5USU1FX1BMQVRGT1JNPWRvY2tlci1jb21wb3NlCiAgICAgIC0gTk9ERV9FTlY9cHJvZHVjdGlvbgogICAgaGVhbHRoY2hlY2s6CiAgICAgIHRlc3Q6CiAgICAgICAgLSBOT05FCg==", + "compose": "c2VydmljZXM6CiAgdHJpZ2dlcjoKICAgIGltYWdlOiAnZ2hjci5pby90cmlnZ2VyZG90ZGV2L3RyaWdnZXIuZGV2OmxhdGVzdCcKICAgIGVudmlyb25tZW50OgogICAgICAtIFNFUlZJQ0VfRlFETl9UUklHR0VSCiAgICAgIC0gTE9HSU5fT1JJR0lOPSRTRVJWSUNFX0ZRRE5fVFJJR0dFUgogICAgICAtIEFQUF9PUklHSU49JFNFUlZJQ0VfRlFETl9UUklHR0VSCiAgICAgIC0gTUFHSUNfTElOS19TRUNSRVQ9JFNFUlZJQ0VfUEFTU1dPUkRfNjRfTUFHSUMKICAgICAgLSBFTkNSWVBUSU9OX0tFWT0kU0VSVklDRV9QQVNTV09SRF82NF9FTkNSWVBUSU9OCiAgICAgIC0gU0VTU0lPTl9TRUNSRVQ9JFNFUlZJQ0VfUEFTU1dPUkRfNjRfU0VTU0lPTgogICAgICAtICdEQVRBQkFTRV9VUkw9JHtEQVRBQkFTRV9VUkx9JwogICAgICAtICdESVJFQ1RfVVJMPSR7REFUQUJBU0VfVVJMfScKICAgICAgLSBSVU5USU1FX1BMQVRGT1JNPWRvY2tlci1jb21wb3NlCiAgICAgIC0gTk9ERV9FTlY9cHJvZHVjdGlvbgogICAgICAtICdBVVRIX0dJVEhVQl9DTElFTlRfSUQ9JHtBVVRIX0dJVEhVQl9DTElFTlRfSUR9JwogICAgICAtICdBVVRIX0dJVEhVQl9DTElFTlRfU0VDUkVUPSR7QVVUSF9HSVRIVUJfQ0xJRU5UX1NFQ1JFVH0nCiAgICAgIC0gJ1JFU0VORF9BUElfS0VZPSR7UkVTRU5EX0FQSV9LRVl9JwogICAgICAtICdGUk9NX0VNQUlMPSR7RlJPTV9FTUFJTH0nCiAgICAgIC0gJ1JFUExZX1RPX0VNQUlMPSR7UkVQTFlfVE9fRU1BSUx9JwogICAgaGVhbHRoY2hlY2s6CiAgICAgIHRlc3Q6CiAgICAgICAgLSBOT05FCg==", "tags": [ "trigger.dev", "background jobs", @@ -458,7 +458,7 @@ "trigger": { "documentation": "https:\/\/trigger.dev\/docs\/documentation\/guides\/self-hosting", "slogan": "The open source Background Jobs framework for TypeScript", - "compose": "c2VydmljZXM6CiAgdHJpZ2dlcjoKICAgIGltYWdlOiAnZ2hjci5pby90cmlnZ2VyZG90ZGV2L3RyaWdnZXIuZGV2OmxhdGVzdCcKICAgIGVudmlyb25tZW50OgogICAgICAtIFNFUlZJQ0VfRlFETl9UUklHR0VSCiAgICAgIC0gTE9HSU5fT1JJR0lOPSRTRVJWSUNFX0ZRRE5fVFJJR0dFUgogICAgICAtIEFQUF9PUklHSU49JFNFUlZJQ0VfRlFETl9UUklHR0VSCiAgICAgIC0gTUFHSUNfTElOS19TRUNSRVQ9JFNFUlZJQ0VfUEFTU1dPUkRfNjRfTUFHSUMKICAgICAgLSBFTkNSWVBUSU9OX0tFWT0kU0VSVklDRV9QQVNTV09SRF82NF9FTkNSWVBUSU9OCiAgICAgIC0gU0VTU0lPTl9TRUNSRVQ9JFNFUlZJQ0VfUEFTU1dPUkRfNjRfU0VTU0lPTgogICAgICAtIFBPU1RHUkVTX1VTRVI9JFNFUlZJQ0VfVVNFUl9QT1NUR1JFUwogICAgICAtIFBPU1RHUkVTX1BBU1NXT1JEPSRTRVJWSUNFX1BBU1NXT1JEX1BPU1RHUkVTCiAgICAgIC0gJ1BPU1RHUkVTX0RCPSR7UE9TVEdSRVNfREI6LXRyaWdnZXJ9JwogICAgICAtIFBPU1RHUkVTX0hPU1Q9cG9zdGdyZXMKICAgICAgLSAnREFUQUJBU0VfVVJMPXBvc3RncmVzOi8vJFNFUlZJQ0VfVVNFUl9QT1NUR1JFUzokU0VSVklDRV9QQVNTV09SRF9QT1NUR1JFU0Bwb3N0Z3Jlc3FsOjU0MzIvJFBPU1RHUkVTX0RCJwogICAgICAtICdESVJFQ1RfVVJMPXBvc3RncmVzOi8vJFNFUlZJQ0VfVVNFUl9QT1NUR1JFUzokU0VSVklDRV9QQVNTV09SRF9QT1NUR1JFU0Bwb3N0Z3Jlc3FsOjU0MzIvJFBPU1RHUkVTX0RCJwogICAgICAtIFJVTlRJTUVfUExBVEZPUk09ZG9ja2VyLWNvbXBvc2UKICAgICAgLSBOT0RFX0VOVj1wcm9kdWN0aW9uCiAgICBkZXBlbmRzX29uOgogICAgICBwb3N0Z3Jlc3FsOgogICAgICAgIGNvbmRpdGlvbjogc2VydmljZV9oZWFsdGh5CiAgICBoZWFsdGhjaGVjazoKICAgICAgdGVzdDoKICAgICAgICAtIE5PTkUKICBwb3N0Z3Jlc3FsOgogICAgaW1hZ2U6ICdwb3N0Z3JlczoxNS1hbHBpbmUnCiAgICB2b2x1bWVzOgogICAgICAtICdwb3N0Z3Jlc3FsLWRhdGE6L3Zhci9saWIvcG9zdGdyZXNxbC9kYXRhJwogICAgZW52aXJvbm1lbnQ6CiAgICAgIC0gUE9TVEdSRVNfVVNFUj0kU0VSVklDRV9VU0VSX1BPU1RHUkVTCiAgICAgIC0gUE9TVEdSRVNfUEFTU1dPUkQ9JFNFUlZJQ0VfUEFTU1dPUkRfUE9TVEdSRVMKICAgICAgLSAnUE9TVEdSRVNfREI9JHtQT1NUR1JFU19EQjotdHJpZ2dlcn0nCiAgICBoZWFsdGhjaGVjazoKICAgICAgdGVzdDoKICAgICAgICAtIENNRC1TSEVMTAogICAgICAgIC0gJ3BnX2lzcmVhZHkgLVUgJCR7UE9TVEdSRVNfVVNFUn0gLWQgJCR7UE9TVEdSRVNfREJ9JwogICAgICBpbnRlcnZhbDogNXMKICAgICAgdGltZW91dDogMjBzCiAgICAgIHJldHJpZXM6IDEwCg==", + "compose": "c2VydmljZXM6CiAgdHJpZ2dlcjoKICAgIGltYWdlOiAnZ2hjci5pby90cmlnZ2VyZG90ZGV2L3RyaWdnZXIuZGV2OmxhdGVzdCcKICAgIGVudmlyb25tZW50OgogICAgICAtIFNFUlZJQ0VfRlFETl9UUklHR0VSCiAgICAgIC0gTE9HSU5fT1JJR0lOPSRTRVJWSUNFX0ZRRE5fVFJJR0dFUgogICAgICAtIEFQUF9PUklHSU49JFNFUlZJQ0VfRlFETl9UUklHR0VSCiAgICAgIC0gTUFHSUNfTElOS19TRUNSRVQ9JFNFUlZJQ0VfUEFTU1dPUkRfNjRfTUFHSUMKICAgICAgLSBFTkNSWVBUSU9OX0tFWT0kU0VSVklDRV9QQVNTV09SRF82NF9FTkNSWVBUSU9OCiAgICAgIC0gU0VTU0lPTl9TRUNSRVQ9JFNFUlZJQ0VfUEFTU1dPUkRfNjRfU0VTU0lPTgogICAgICAtIFBPU1RHUkVTX1VTRVI9JFNFUlZJQ0VfVVNFUl9QT1NUR1JFUwogICAgICAtIFBPU1RHUkVTX1BBU1NXT1JEPSRTRVJWSUNFX1BBU1NXT1JEX1BPU1RHUkVTCiAgICAgIC0gJ1BPU1RHUkVTX0RCPSR7UE9TVEdSRVNfREI6LXRyaWdnZXJ9JwogICAgICAtIFBPU1RHUkVTX0hPU1Q9cG9zdGdyZXMKICAgICAgLSAnREFUQUJBU0VfVVJMPXBvc3RncmVzOi8vJFNFUlZJQ0VfVVNFUl9QT1NUR1JFUzokU0VSVklDRV9QQVNTV09SRF9QT1NUR1JFU0Bwb3N0Z3Jlc3FsOjU0MzIvJFBPU1RHUkVTX0RCJwogICAgICAtICdESVJFQ1RfVVJMPXBvc3RncmVzOi8vJFNFUlZJQ0VfVVNFUl9QT1NUR1JFUzokU0VSVklDRV9QQVNTV09SRF9QT1NUR1JFU0Bwb3N0Z3Jlc3FsOjU0MzIvJFBPU1RHUkVTX0RCJwogICAgICAtIFJVTlRJTUVfUExBVEZPUk09ZG9ja2VyLWNvbXBvc2UKICAgICAgLSBOT0RFX0VOVj1wcm9kdWN0aW9uCiAgICAgIC0gJ0FVVEhfR0lUSFVCX0NMSUVOVF9JRD0ke0FVVEhfR0lUSFVCX0NMSUVOVF9JRH0nCiAgICAgIC0gJ0FVVEhfR0lUSFVCX0NMSUVOVF9TRUNSRVQ9JHtBVVRIX0dJVEhVQl9DTElFTlRfU0VDUkVUfScKICAgICAgLSAnUkVTRU5EX0FQSV9LRVk9JHtSRVNFTkRfQVBJX0tFWX0nCiAgICAgIC0gJ0ZST01fRU1BSUw9JHtGUk9NX0VNQUlMfScKICAgICAgLSAnUkVQTFlfVE9fRU1BSUw9JHtSRVBMWV9UT19FTUFJTH0nCiAgICBkZXBlbmRzX29uOgogICAgICBwb3N0Z3Jlc3FsOgogICAgICAgIGNvbmRpdGlvbjogc2VydmljZV9oZWFsdGh5CiAgICBoZWFsdGhjaGVjazoKICAgICAgdGVzdDoKICAgICAgICAtIE5PTkUKICBwb3N0Z3Jlc3FsOgogICAgaW1hZ2U6ICdwb3N0Z3JlczoxNS1hbHBpbmUnCiAgICB2b2x1bWVzOgogICAgICAtICdwb3N0Z3Jlc3FsLWRhdGE6L3Zhci9saWIvcG9zdGdyZXNxbC9kYXRhJwogICAgZW52aXJvbm1lbnQ6CiAgICAgIC0gUE9TVEdSRVNfVVNFUj0kU0VSVklDRV9VU0VSX1BPU1RHUkVTCiAgICAgIC0gUE9TVEdSRVNfUEFTU1dPUkQ9JFNFUlZJQ0VfUEFTU1dPUkRfUE9TVEdSRVMKICAgICAgLSAnUE9TVEdSRVNfREI9JHtQT1NUR1JFU19EQjotdHJpZ2dlcn0nCiAgICBoZWFsdGhjaGVjazoKICAgICAgdGVzdDoKICAgICAgICAtIENNRC1TSEVMTAogICAgICAgIC0gJ3BnX2lzcmVhZHkgLVUgJCR7UE9TVEdSRVNfVVNFUn0gLWQgJCR7UE9TVEdSRVNfREJ9JwogICAgICBpbnRlcnZhbDogNXMKICAgICAgdGltZW91dDogMjBzCiAgICAgIHJldHJpZXM6IDEwCg==", "tags": [ "trigger.dev", "background jobs", From 65a196172222d30760917884d1d8a165acdcb964 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Fri, 24 Nov 2023 10:12:37 +0100 Subject: [PATCH 16/46] Add environment variables for Horizon balance --- docker-compose.prod.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index be972721f..ca6b42348 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -28,6 +28,8 @@ services: - REDIS_HOST - REDIS_PASSWORD - HORIZON_MAX_PROCESSES + - HORIZON_BALANCE_MAX_SHIFT + - HORIZON_BALANCE_COOLDOWN - SSL_MODE=off - PHP_PM_CONTROL=dynamic - PHP_PM_START_SERVERS=1 From f96a91eb313d590aa7aebdf0e7f62249e2abad41 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Fri, 24 Nov 2023 15:48:23 +0100 Subject: [PATCH 17/46] wip: compose based apps --- app/Console/Commands/SyncBunny.php | 9 + .../Livewire/Project/Application/General.php | 52 ++ app/Jobs/ApplicationDeploymentJob.php | 179 ++-- app/Models/Application.php | 231 ++++- app/Models/Server.php | 48 +- app/Models/Service.php | 517 +---------- app/Traits/ExecuteRemoteCommand.php | 2 +- bootstrap/helpers/docker.php | 29 +- bootstrap/helpers/proxy.php | 1 + bootstrap/helpers/shared.php | 858 +++++++++++++++++- ..._24_080341_add_docker_compose_location.php | 35 + .../components/applications/navbar.blade.php | 25 +- .../project/application/general.blade.php | 104 ++- .../livewire/project/new/select.blade.php | 4 +- 14 files changed, 1389 insertions(+), 705 deletions(-) create mode 100644 database/migrations/2023_11_24_080341_add_docker_compose_location.php diff --git a/app/Console/Commands/SyncBunny.php b/app/Console/Commands/SyncBunny.php index f3ddaaffe..a59d13ab8 100644 --- a/app/Console/Commands/SyncBunny.php +++ b/app/Console/Commands/SyncBunny.php @@ -71,6 +71,15 @@ class SyncBunny extends Command ]); }); try { + if (!$only_template && !$only_version) { + $this->info('About to sync files (docker-compose.prod.yaml, upgrade.sh, install.sh, etc) to BunnyCDN.'); + } + if ($only_template) { + $this->info('About to sync service-templates.json to BunnyCDN.'); + } + if ($only_version) { + $this->info('About to sync versions.json to BunnyCDN.'); + } $confirmed = confirm('Are you sure you want to sync?'); if (!$confirmed) { return; diff --git a/app/Http/Livewire/Project/Application/General.php b/app/Http/Livewire/Project/Application/General.php index 27c5023c3..a90fadb53 100644 --- a/app/Http/Livewire/Project/Application/General.php +++ b/app/Http/Livewire/Project/Application/General.php @@ -6,6 +6,7 @@ use App\Models\Application; use Illuminate\Support\Collection; use Illuminate\Support\Str; use Livewire\Component; +use Visus\Cuid2\Cuid2; class General extends Component { @@ -27,6 +28,9 @@ class General extends Component public bool $is_static; + public $parsedServices = []; + public $parsedServiceDomains = []; + protected $listeners = [ 'resetDefaultLabels' ]; @@ -50,6 +54,9 @@ class General extends Component 'application.docker_registry_image_name' => 'nullable', 'application.docker_registry_image_tag' => 'nullable', 'application.dockerfile_location' => 'nullable', + 'application.docker_compose_location' => 'nullable', + 'application.docker_compose' => 'nullable', + 'application.docker_compose_raw' => 'nullable', 'application.custom_labels' => 'nullable', 'application.dockerfile_target_build' => 'nullable', 'application.settings.is_static' => 'boolean|required', @@ -74,6 +81,9 @@ class General extends Component 'application.docker_registry_image_name' => 'Docker registry image name', 'application.docker_registry_image_tag' => 'Docker registry image tag', 'application.dockerfile_location' => 'Dockerfile location', + 'application.docker_compose_location' => 'Docker compose location', + 'application.docker_compose' => 'Docker compose', + 'application.docker_compose_raw' => 'Docker compose raw', 'application.custom_labels' => 'Custom labels', 'application.dockerfile_target_build' => 'Dockerfile target build', 'application.settings.is_static' => 'Is static', @@ -81,6 +91,14 @@ class General extends Component public function mount() { + try { + $this->parsedServices = $this->application->parseCompose(); + ray($this->parsedServices); + } catch (\Throwable $e) { + $this->emit('error', $e->getMessage()); + } + $this->parsedServiceDomains = $this->application->docker_compose_domains ? json_decode($this->application->docker_compose_domains, true) : []; + $this->ports_exposes = $this->application->ports_exposes; if (str($this->application->status)->startsWith('running') && is_null($this->application->config_hash)) { $this->application->isConfigurationChanged(true); @@ -98,6 +116,38 @@ class General extends Component $this->application->settings->save(); $this->emit('success', 'Settings saved.'); } + public function loadComposeFile($isInit = false) + { + if ($isInit && $this->application->docker_compose_raw) { + return; + } + $uuid = new Cuid2(); + ['commands' => $cloneCommand] = $this->application->generateGitImportCommands(deployment_uuid: $uuid, only_checkout: true, exec_in_docker: false, custom_base_dir: '.'); + $workdir = rtrim($this->application->base_directory, '/'); + $composeFile = $this->application->docker_compose_location; + $commands = collect([ + "mkdir -p /tmp/{$uuid} && cd /tmp/{$uuid}", + $cloneCommand, + "git sparse-checkout init --cone", + "git sparse-checkout set .$workdir$composeFile", + "git read-tree -mu HEAD", + "cat .$workdir$composeFile", + ]); + $composeFileContent = instant_remote_process($commands, $this->application->destination->server, false); + if (!$composeFileContent) { + $this->emit('error', "Could not load compose file from $workdir$composeFile"); + return; + } else { + $this->application->docker_compose_raw = $composeFileContent; + $this->application->save(); + } + $commands = collect([ + "rm -rf /tmp/{$uuid}", + ]); + instant_remote_process($commands, $this->application->destination->server, false); + $this->parsedServices = $this->application->parseCompose(); + $this->emit('success', 'Compose file loaded.'); + } public function updatedApplicationBuildPack() { if ($this->application->build_pack !== 'nixpacks') { @@ -172,8 +222,10 @@ class General extends Component $this->customLabels = str($this->customLabels)->replace(',', "\n"); } $this->application->custom_labels = $this->customLabels->explode("\n")->implode(','); + $this->application->docker_compose_domains = json_encode($this->parsedServiceDomains); $this->application->save(); $showToaster && $this->emit('success', 'Application settings updated!'); + $this->parsedServices = $this->application->parseCompose(); } catch (\Throwable $e) { return handleError($e, $this); } finally { diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index 1dcd51b22..ac7d4dde7 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -73,6 +73,7 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted private $docker_compose; private $docker_compose_base64; private string $dockerfile_location = '/Dockerfile'; + private string $docker_compose_location = '/docker-compose.yml'; private ?string $addHosts = null; private ?string $buildTarget = null; private $log_model; @@ -114,7 +115,7 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted $this->destination = $this->application->destination->getMorphClass()::where('id', $this->application->destination->id)->first(); $this->server = $this->mainServer = $this->destination->server; $this->serverUser = $this->server->user; - $this->basedir = "/artifacts/{$this->deployment_uuid}"; + $this->basedir = $this->application->generateBaseDir($this->deployment_uuid); $this->workdir = "{$this->basedir}" . rtrim($this->application->base_directory, '/'); $this->configuration_dir = application_configuration_dir() . "/{$this->application->uuid}"; $this->is_debug_enabled = $this->application->settings->is_debug_enabled; @@ -183,16 +184,10 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted } // Check custom port - preg_match('/(?<=:)\d+(?=\/)/', $this->application->git_repository, $matches); - if (count($matches) === 1) { - $this->customPort = $matches[0]; - $gitHost = str($this->application->git_repository)->before(':'); - $gitRepo = str($this->application->git_repository)->after('/'); - $this->customRepository = "$gitHost:$gitRepo"; - } else { - $this->customRepository = $this->application->git_repository; - } + ['repository' => $this->customRepository, 'port' => $this->customPort] = $this->application->customRepository(); + try { + ray($this->application->build_pack); if ($this->restart_only && $this->application->build_pack !== 'dockerimage') { $this->just_restart(); if ($this->server->isProxyShouldRun()) { @@ -203,6 +198,8 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted return; } else if ($this->application->dockerfile) { $this->deploy_simple_dockerfile(); + } else if ($this->application->build_pack === 'dockercompose') { + $this->deploy_docker_compose_buildpack(); } else if ($this->application->build_pack === 'dockerimage') { $this->deploy_dockerimage_buildpack(); } else if ($this->application->build_pack === 'dockerfile') { @@ -397,19 +394,20 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted ]); } } - // private function save_environment_variables() - // { - // $envs = collect([]); - // foreach ($this->application->environment_variables as $env) { - // $envs->push($env->key . '=' . $env->value); - // } - // $envs_base64 = base64_encode($envs->implode("\n")); - // $this->execute_remote_command( - // [ - // executeInDocker($this->deployment_uuid, "echo '$envs_base64' | base64 -d > $this->workdir/.env") - // ], - // ); - // } + private function save_environment_variables() + { + $envs = collect([]); + foreach ($this->application->environment_variables as $env) { + $envs->push($env->key . '=' . $env->value); + } + $envs_base64 = base64_encode($envs->implode("\n")); + $this->execute_remote_command( + [ + executeInDocker($this->deployment_uuid, "echo '$envs_base64' | base64 -d > $this->workdir/.env") + ], + ); + } + private function deploy_simple_dockerfile() { $dockerfile_base64 = base64_encode($this->application->dockerfile); @@ -447,7 +445,36 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted $this->generate_compose_file(); $this->rolling_update(); } + private function deploy_docker_compose_buildpack() + { + if (data_get($this->application, 'docker_compose_location')) { + $this->docker_compose_location = $this->application->docker_compose_location; + } + $this->application_deployment_queue->addLogEntry("Starting deployment of {$this->application->name}."); + $this->server->executeRemoteCommand( + commands: $this->application->prepareHelperImage($this->deployment_uuid), + loggingModel: $this->application_deployment_queue + ); + $this->check_git_if_build_needed(); + $this->clone_repository(); + $this->generate_image_names(); + $this->cleanup_git(); + $composeFile = $this->application->parseCompose(); + $yaml = Yaml::dump($composeFile->toArray(), 10); + $this->docker_compose_base64 = base64_encode($yaml); + $this->execute_remote_command([ + executeInDocker($this->deployment_uuid, "echo '{$this->docker_compose_base64}' | base64 -d > {$this->workdir}/docker-compose.yaml"), "hidden" => true + ]); + $this->execute_remote_command([ + "docker network create --attachable '{$this->application->uuid}' >/dev/null || true", "hidden" => true, "ignore_errors" => true + ], [ + "docker network connect {$this->application->uuid} coolify-proxy || true", "hidden" => true, "ignore_errors" => true + ]); + $this->save_environment_variables(); + $this->stop_running_container(force: true); + $this->start_by_compose_file(); + } private function deploy_dockerfile_buildpack() { if (data_get($this->application, 'dockerfile_location')) { @@ -472,7 +499,7 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted // $this->push_to_docker_registry(); // $this->deploy_to_additional_destinations(); // } else { - $this->rolling_update(); + $this->rolling_update(); // } } private function deploy_nixpacks_buildpack() @@ -725,6 +752,7 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted private function clone_repository() { $importCommands = $this->generate_git_import_commands(); + ray($importCommands); $this->execute_remote_command( [ "echo '\n----------------------------------------'", @@ -740,90 +768,14 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted private function generate_git_import_commands() { - $this->branch = $this->application->git_branch; - $commands = collect([]); - $git_clone_command = "git clone -q -b {$this->application->git_branch}"; - if ($this->pull_request_id !== 0) { - $pr_branch_name = "pr-{$this->pull_request_id}-coolify"; - } - if ($this->application->deploymentType() === 'source') { - $source_html_url = data_get($this->application, 'source.html_url'); - $url = parse_url(filter_var($source_html_url, FILTER_SANITIZE_URL)); - $source_html_url_host = $url['host']; - $source_html_url_scheme = $url['scheme']; - - if ($this->source->getMorphClass() == 'App\Models\GithubApp') { - if ($this->source->is_public) { - $this->fullRepoUrl = "{$this->source->html_url}/{$this->customRepository}"; - $git_clone_command = "{$git_clone_command} {$this->source->html_url}/{$this->customRepository} {$this->basedir}"; - $git_clone_command = $this->set_git_import_settings($git_clone_command); - - $commands->push(executeInDocker($this->deployment_uuid, $git_clone_command)); - } else { - $github_access_token = generate_github_installation_token($this->source); - $commands->push(executeInDocker($this->deployment_uuid, "git clone -q -b {$this->application->git_branch} $source_html_url_scheme://x-access-token:$github_access_token@$source_html_url_host/{$this->customRepository}.git {$this->basedir}")); - $this->fullRepoUrl = "$source_html_url_scheme://x-access-token:$github_access_token@$source_html_url_host/{$this->customRepository}.git"; - } - if ($this->pull_request_id !== 0) { - $this->branch = "pull/{$this->pull_request_id}/head:$pr_branch_name"; - $commands->push(executeInDocker($this->deployment_uuid, "cd {$this->basedir} && git fetch origin $this->branch && git checkout $pr_branch_name")); - } - return $commands->implode(' && '); - } - } - if ($this->application->deploymentType() === 'deploy_key') { - $this->fullRepoUrl = $this->customRepository; - $private_key = data_get($this->application, 'private_key.private_key'); - if (is_null($private_key)) { - throw new RuntimeException('Private key not found. Please add a private key to the application and try again.'); - } - $private_key = base64_encode($private_key); - $git_clone_command_base = "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$this->customPort} -o Port={$this->customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa\" {$git_clone_command} {$this->customRepository} {$this->basedir}"; - $git_clone_command = $this->set_git_import_settings($git_clone_command_base); - $commands = collect([ - executeInDocker($this->deployment_uuid, "mkdir -p /root/.ssh"), - executeInDocker($this->deployment_uuid, "echo '{$private_key}' | base64 -d > /root/.ssh/id_rsa"), - executeInDocker($this->deployment_uuid, "chmod 600 /root/.ssh/id_rsa"), - ]); - if ($this->pull_request_id !== 0) { - ray($this->git_type); - if ($this->git_type === 'gitlab') { - $this->branch = "merge-requests/{$this->pull_request_id}/head:$pr_branch_name"; - $commands->push(executeInDocker($this->deployment_uuid, "echo 'Checking out $this->branch'")); - $git_clone_command = "{$git_clone_command} && cd {$this->basedir} && GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$this->customPort} -o Port={$this->customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa\" git fetch origin $this->branch && git checkout $pr_branch_name"; - } - if ($this->git_type === 'github') { - $this->branch = "pull/{$this->pull_request_id}/head:$pr_branch_name"; - $commands->push(executeInDocker($this->deployment_uuid, "echo 'Checking out $this->branch'")); - $git_clone_command = "{$git_clone_command} && cd {$this->basedir} && GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$this->customPort} -o Port={$this->customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa\" git fetch origin $this->branch && git checkout $pr_branch_name"; - } - } - - $commands->push(executeInDocker($this->deployment_uuid, $git_clone_command)); - return $commands->implode(' && '); - } - if ($this->application->deploymentType() === 'other') { - $this->fullRepoUrl = $this->customRepository; - $git_clone_command = "{$git_clone_command} {$this->customRepository} {$this->basedir}"; - $git_clone_command = $this->set_git_import_settings($git_clone_command); - $commands->push(executeInDocker($this->deployment_uuid, $git_clone_command)); - return $commands->implode(' && '); - } + ['commands' => $commands, 'branch' => $this->branch, 'fullRepoUrl' => $this->fullRepoUrl] = $this->application->generateGitImportCommands($this->deployment_uuid, $this->pull_request_id, $this->git_type); + return $commands; } private function set_git_import_settings($git_clone_command) { - if ($this->application->git_commit_sha !== 'HEAD') { - $git_clone_command = "{$git_clone_command} && cd {$this->basedir} && git -c advice.detachedHead=false checkout {$this->application->git_commit_sha} >/dev/null 2>&1"; - } - if ($this->application->settings->is_git_submodules_enabled) { - $git_clone_command = "{$git_clone_command} && cd {$this->basedir} && git submodule update --init --recursive"; - } - if ($this->application->settings->is_git_lfs_enabled) { - $git_clone_command = "{$git_clone_command} && cd {$this->basedir} && git lfs pull"; - } - return $git_clone_command; + return $this->application->setGitImportSettings($this->deployment_uuid, $git_clone_command); } private function cleanup_git() @@ -879,6 +831,26 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted $this->env_args = $this->env_args->implode(' '); } + private function modify_compose_file() + { + // ray("{$this->workdir}{$this->docker_compose_location}"); + $this->execute_remote_command([executeInDocker($this->deployment_uuid, "cat {$this->workdir}{$this->docker_compose_location}"), "hidden" => true, "save" => 'compose_file']); + if ($this->saved_outputs->get('compose_file')) { + $compose = $this->saved_outputs->get('compose_file'); + } + try { + $yaml = Yaml::parse($compose); + } catch (\Exception $e) { + throw new \Exception($e->getMessage()); + } + $services = data_get($yaml, 'services'); + $topLevelNetworks = collect(data_get($yaml, 'networks', [])); + $definedNetwork = collect([$this->application->uuid]); + + $services = collect($services)->map(function ($service, $serviceName) use ($topLevelNetworks, $definedNetwork) { + $serviceNetworks = collect(data_get($service, 'networks', [])); + }); + } private function generate_compose_file() { $ports = $this->application->settings->is_static ? [80] : $this->application->ports_exposes_array; @@ -1209,6 +1181,7 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf"); $this->execute_remote_command(["echo -n 'Removing old container.'"]); if ($this->newVersionIsHealthy || $force) { $containers = getCurrentApplicationContainerStatus($this->server, $this->application->id, $this->pull_request_id); + ray($containers); if ($this->pull_request_id !== 0) { $containers = $containers->filter(function ($container) { return data_get($container, 'Names') === $this->container_name; diff --git a/app/Models/Application.php b/app/Models/Application.php index 815376d18..ee7ccf60b 100644 --- a/app/Models/Application.php +++ b/app/Models/Application.php @@ -6,6 +6,8 @@ use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Database\Eloquent\Relations\HasMany; use Spatie\Activitylog\Models\Activity; use Illuminate\Support\Str; +use RuntimeException; +use Symfony\Component\Yaml\Yaml; class Application extends BaseModel { @@ -123,6 +125,21 @@ class Application extends BaseModel } ); } + public function dockerComposeLocation(): Attribute + { + return Attribute::make( + set: function ($value) { + if (is_null($value) || $value === '') { + return '/docker-compose.yml'; + } else { + if ($value !== '/') { + return Str::start(Str::replaceEnd('/', '', $value), '/'); + } + return Str::start($value, '/'); + } + } + ); + } public function baseDirectory(): Attribute { return Attribute::make( @@ -157,7 +174,16 @@ class Application extends BaseModel : explode(',', $this->ports_exposes) ); } - + public function serviceType() + { + $found = str(collect(SPECIFIC_SERVICES)->filter(function ($service) { + return str($this->image)->before(':')->value() === $service; + })->first()); + if ($found->isNotEmpty()) { + return $found; + } + return null; + } public function environment_variables(): HasMany { return $this->hasMany(EnvironmentVariable::class)->where('is_preview', false)->orderBy('key', 'asc'); @@ -342,7 +368,8 @@ class Application extends BaseModel } return false; } - public function healthCheckUrl() { + public function healthCheckUrl() + { if ($this->dockerfile || $this->build_pack === 'dockerfile' || $this->build_pack === 'dockerimage') { return null; } @@ -358,4 +385,204 @@ class Application extends BaseModel } return $full_healthcheck_url; } + function customRepository() + { + preg_match('/(?<=:)\d+(?=\/)/', $this->git_repository, $matches); + $port = 22; + if (count($matches) === 1) { + $port = $matches[0]; + $gitHost = str($this->git_repository)->before(':'); + $gitRepo = str($this->git_repository)->after('/'); + $repository = "$gitHost:$gitRepo"; + } else { + $repository = $this->git_repository; + } + return [ + 'repository' => $repository, + 'port' => $port + ]; + } + function generateBaseDir(string $uuid) + { + return "/artifacts/{$uuid}"; + } + function setGitImportSettings(string $deployment_uuid, string $git_clone_command) + { + $baseDir = $this->generateBaseDir($deployment_uuid); + if ($this->git_commit_sha !== 'HEAD') { + $git_clone_command = "{$git_clone_command} && cd {$baseDir} && git -c advice.detachedHead=false checkout {$this->git_commit_sha} >/dev/null 2>&1"; + } + if ($this->settings->is_git_submodules_enabled) { + $git_clone_command = "{$git_clone_command} && cd {$baseDir} && git submodule update --init --recursive"; + } + if ($this->settings->is_git_lfs_enabled) { + $git_clone_command = "{$git_clone_command} && cd {$baseDir} && git lfs pull"; + } + return $git_clone_command; + } + function generateGitImportCommands(string $deployment_uuid, int $pull_request_id = 0, ?string $git_type = null, bool $exec_in_docker = true, bool $only_checkout = false, ?string $custom_base_dir = null) + { + $branch = $this->git_branch; + ['repository' => $customRepository, 'port' => $customPort] = $this->customRepository(); + $baseDir = $custom_base_dir ?? $this->generateBaseDir($deployment_uuid); + $commands = collect([]); + $git_clone_command = "git clone -b {$this->git_branch}"; + if ($only_checkout) { + $git_clone_command = "git clone --no-checkout -b {$this->git_branch}"; + } + if ($pull_request_id !== 0) { + $pr_branch_name = "pr-{$pull_request_id}-coolify"; + } + + if ($this->deploymentType() === 'source') { + $source_html_url = data_get($this, 'source.html_url'); + $url = parse_url(filter_var($source_html_url, FILTER_SANITIZE_URL)); + $source_html_url_host = $url['host']; + $source_html_url_scheme = $url['scheme']; + + if ($this->source->getMorphClass() == 'App\Models\GithubApp') { + if ($this->source->is_public) { + $fullRepoUrl = "{$this->source->html_url}/{$customRepository}"; + $git_clone_command = "{$git_clone_command} {$this->source->html_url}/{$customRepository} {$baseDir}"; + if (!$only_checkout) { + $git_clone_command = $this->setGitImportSettings($deployment_uuid, $git_clone_command); + } + if ($exec_in_docker) { + $commands->push(executeInDocker($deployment_uuid, $git_clone_command)); + } else { + $commands->push($git_clone_command); + } + } else { + $github_access_token = generate_github_installation_token($this->source); + if ($exec_in_docker) { + $commands->push(executeInDocker($deployment_uuid, "{$git_clone_command} $source_html_url_scheme://x-access-token:$github_access_token@$source_html_url_host/{$customRepository}.git {$baseDir}")); + $fullRepoUrl = "$source_html_url_scheme://x-access-token:$github_access_token@$source_html_url_host/{$customRepository}.git"; + } else { + $commands->push("{$git_clone_command} $source_html_url_scheme://x-access-token:$github_access_token@$source_html_url_host/{$customRepository} {$baseDir}"); + $fullRepoUrl = "$source_html_url_scheme://x-access-token:$github_access_token@$source_html_url_host/{$customRepository}"; + } + } + if ($pull_request_id !== 0) { + $branch = "pull/{$pull_request_id}/head:$pr_branch_name"; + if ($exec_in_docker) { + $commands->push(executeInDocker($deployment_uuid, "cd {$baseDir} && git fetch origin {$branch} && git checkout $pr_branch_name")); + } else { + $commands->push("cd {$baseDir} && git fetch origin {$branch} && git checkout $pr_branch_name"); + } + } + return [ + 'commands' => $commands->implode(' && '), + 'branch' => $branch, + 'fullRepoUrl' => $fullRepoUrl + ]; + } + } + if ($this->deploymentType() === 'deploy_key') { + $fullRepoUrl = $customRepository; + $private_key = data_get($this, 'private_key.private_key'); + if (is_null($private_key)) { + throw new RuntimeException('Private key not found. Please add a private key to the application and try again.'); + } + $private_key = base64_encode($private_key); + $git_clone_command_base = "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$customPort} -o Port={$customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa\" {$git_clone_command} {$customRepository} {$baseDir}"; + if (!$only_checkout) { + $git_clone_command = $this->setGitImportSettings($deployment_uuid, $git_clone_command_base); + } + if ($exec_in_docker) { + $commands = collect([ + executeInDocker($deployment_uuid, "mkdir -p /root/.ssh"), + executeInDocker($deployment_uuid, "echo '{$private_key}' | base64 -d > /root/.ssh/id_rsa"), + executeInDocker($deployment_uuid, "chmod 600 /root/.ssh/id_rsa"), + ]); + } else { + $commands = collect([ + "mkdir -p /root/.ssh", + "echo '{$private_key}' | base64 -d > /root/.ssh/id_rsa", + "chmod 600 /root/.ssh/id_rsa", + ]); + } + if ($pull_request_id !== 0) { + if ($git_type === 'gitlab') { + $branch = "merge-requests/{$pull_request_id}/head:$pr_branch_name"; + if ($exec_in_docker) { + $commands->push(executeInDocker($deployment_uuid, "echo 'Checking out $branch'")); + } else { + $commands->push("echo 'Checking out $branch'"); + } + $git_clone_command = "{$git_clone_command} && cd {$baseDir} && GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$customPort} -o Port={$customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa\" git fetch origin $branch && git checkout $pr_branch_name"; + } + if ($git_type === 'github') { + $branch = "pull/{$pull_request_id}/head:$pr_branch_name"; + if ($exec_in_docker) { + $commands->push(executeInDocker($deployment_uuid, "echo 'Checking out $branch'")); + } else { + $commands->push("echo 'Checking out $branch'"); + } + $git_clone_command = "{$git_clone_command} && cd {$baseDir} && GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$customPort} -o Port={$customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa\" git fetch origin $branch && git checkout $pr_branch_name"; + } + } + + if ($exec_in_docker) { + $commands->push(executeInDocker($deployment_uuid, $git_clone_command)); + } else { + $commands->push($git_clone_command); + } + return [ + 'commands' => $commands->implode(' && '), + 'branch' => $branch, + 'fullRepoUrl' => $fullRepoUrl + ]; + } + if ($this->deploymentType() === 'other') { + $fullRepoUrl = $customRepository; + $git_clone_command = "{$git_clone_command} {$customRepository} {$baseDir}"; + $git_clone_command = $this->setGitImportSettings($deployment_uuid, $git_clone_command); + if ($exec_in_docker) { + $commands->push(executeInDocker($deployment_uuid, $git_clone_command)); + } else { + $commands->push($git_clone_command); + } + return [ + 'commands' => $commands->implode(' && '), + 'branch' => $branch, + 'fullRepoUrl' => $fullRepoUrl + ]; + } + } + public function prepareHelperImage(string $deploymentUuid) + { + $basedir = $this->generateBaseDir($deploymentUuid); + $helperImage = config('coolify.helper_image'); + $server = data_get($this, 'destination.server'); + $network = data_get($this, 'destination.network'); + + $serverUserHomeDir = instant_remote_process(["echo \$HOME"], $server); + $dockerConfigFileExists = instant_remote_process(["test -f {$serverUserHomeDir}/.docker/config.json && echo 'OK' || echo 'NOK'"], $server); + + $commands = collect([]); + if ($dockerConfigFileExists === 'OK') { + $commands->push([ + "command" => "docker run -d --network $network -v /:/host --name $deploymentUuid --rm -v {$serverUserHomeDir}/.docker/config.json:/root/.docker/config.json:ro -v /var/run/docker.sock:/var/run/docker.sock $helperImage", + "hidden" => true, + ]); + } else { + $commands->push([ + "command" => "docker run -d --network {$network} -v /:/host --name {$deploymentUuid} --rm -v /var/run/docker.sock:/var/run/docker.sock {$helperImage}", + "hidden" => true, + ]); + } + $commands->push([ + "command" => executeInDocker($deploymentUuid, "mkdir -p {$basedir}"), + "hidden" => true, + ]); + return $commands; + } + function parseCompose() + { + if ($this->docker_compose_raw) { + return parseDockerComposeFile($this); + } else { + return collect([]); + } + } } diff --git a/app/Models/Server.php b/app/Models/Server.php index 7811b1aaa..8df4c1e3f 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -392,7 +392,7 @@ class Server extends BaseModel { return instant_remote_process(["docker network create coolify --attachable >/dev/null 2>&1 || true"], $this, false); } - public function executeRemoteCommand(Collection $commands, ApplicationDeploymentQueue $loggingModel) + public function executeRemoteCommand(Collection $commands, ?ApplicationDeploymentQueue $loggingModel = null) { static::$batch_counter++; foreach ($commands as $command) { @@ -419,33 +419,35 @@ class Server extends BaseModel 'hidden' => $hidden, 'batch' => static::$batch_counter, ]; - if (!$loggingModel->logs) { - $newLogEntry['order'] = 1; - } else { - $previousLogs = json_decode($loggingModel->logs, associative: true, flags: JSON_THROW_ON_ERROR); - $newLogEntry['order'] = count($previousLogs) + 1; - } - if ($name) { - $newLogEntry['name'] = $name; - } + if ($loggingModel) { + if (!$loggingModel->logs) { + $newLogEntry['order'] = 1; + } else { + $previousLogs = json_decode($loggingModel->logs, associative: true, flags: JSON_THROW_ON_ERROR); + $newLogEntry['order'] = count($previousLogs) + 1; + } + if ($name) { + $newLogEntry['name'] = $name; + } - $previousLogs[] = $newLogEntry; - $loggingModel->logs = json_encode($previousLogs, flags: JSON_THROW_ON_ERROR); - $loggingModel->save(); - // if ($name) { - // $loggingModel['savedOutputs'][$name] = str($output)->trim(); - // } + $previousLogs[] = $newLogEntry; + $loggingModel->logs = json_encode($previousLogs, flags: JSON_THROW_ON_ERROR); + $loggingModel->save(); + } }); - $loggingModel->update([ - 'current_process_id' => $process->id(), - ]); - + if ($loggingModel) { + $loggingModel->update([ + 'current_process_id' => $process->id(), + ]); + } $processResult = $process->wait(); if ($processResult->exitCode() !== 0) { if (!$ignoreErrors) { - $status = ApplicationDeploymentStatus::FAILED->value; - $loggingModel->status = $status; - $loggingModel->save(); + if ($loggingModel) { + $status = ApplicationDeploymentStatus::FAILED->value; + $loggingModel->status = $status; + $loggingModel->save(); + } throw new \RuntimeException($processResult->errorOutput()); } } diff --git a/app/Models/Service.php b/app/Models/Service.php index 8cd195bce..2d3e17e98 100644 --- a/app/Models/Service.php +++ b/app/Models/Service.php @@ -371,521 +371,6 @@ class Service extends BaseModel public function parse(bool $isNew = false): Collection { - // ray()->clearAll(); - if ($this->docker_compose_raw) { - try { - $yaml = Yaml::parse($this->docker_compose_raw); - } catch (\Exception $e) { - throw new \Exception($e->getMessage()); - } - - $topLevelVolumes = collect(data_get($yaml, 'volumes', [])); - $topLevelNetworks = collect(data_get($yaml, 'networks', [])); - $dockerComposeVersion = data_get($yaml, 'version') ?? '3.8'; - $services = data_get($yaml, 'services'); - - $generatedServiceFQDNS = collect([]); - if (is_null($this->destination)) { - $destination = $this->server->destinations()->first(); - if ($destination) { - $this->destination()->associate($destination); - $this->save(); - } - } - $definedNetwork = collect([$this->uuid]); - - $services = collect($services)->map(function ($service, $serviceName) use ($topLevelVolumes, $topLevelNetworks, $definedNetwork, $isNew, $generatedServiceFQDNS) { - $serviceVolumes = collect(data_get($service, 'volumes', [])); - $servicePorts = collect(data_get($service, 'ports', [])); - $serviceNetworks = collect(data_get($service, 'networks', [])); - $serviceVariables = collect(data_get($service, 'environment', [])); - $serviceLabels = collect(data_get($service, 'labels', [])); - if ($serviceLabels->count() > 0) { - $removedLabels = collect([]); - $serviceLabels = $serviceLabels->filter(function ($serviceLabel, $serviceLabelName) use ($removedLabels) { - if (!str($serviceLabel)->contains('=')) { - $removedLabels->put($serviceLabelName, $serviceLabel); - return false; - } - return $serviceLabel; - }); - foreach($removedLabels as $removedLabelName =>$removedLabel) { - $serviceLabels->push("$removedLabelName=$removedLabel"); - } - } - - $containerName = "$serviceName-{$this->uuid}"; - - // Decide if the service is a database - $isDatabase = false; - $image = data_get_str($service, 'image'); - if ($image->contains(':')) { - $image = Str::of($image); - } else { - $image = Str::of($image)->append(':latest'); - } - $imageName = $image->before(':'); - - if (collect(DATABASE_DOCKER_IMAGES)->contains($imageName)) { - $isDatabase = true; - } - data_set($service, 'is_database', $isDatabase); - - // Create new serviceApplication or serviceDatabase - if ($isDatabase) { - if ($isNew) { - $savedService = ServiceDatabase::create([ - 'name' => $serviceName, - 'image' => $image, - 'service_id' => $this->id - ]); - } else { - $savedService = ServiceDatabase::where([ - 'name' => $serviceName, - 'service_id' => $this->id - ])->first(); - } - } else { - if ($isNew) { - $savedService = ServiceApplication::create([ - 'name' => $serviceName, - 'image' => $image, - 'service_id' => $this->id - ]); - } else { - $savedService = ServiceApplication::where([ - 'name' => $serviceName, - 'service_id' => $this->id - ])->first(); - } - } - if (is_null($savedService)) { - if ($isDatabase) { - $savedService = ServiceDatabase::create([ - 'name' => $serviceName, - 'image' => $image, - 'service_id' => $this->id - ]); - } else { - $savedService = ServiceApplication::create([ - 'name' => $serviceName, - 'image' => $image, - 'service_id' => $this->id - ]); - } - } - - // Check if image changed - if ($savedService->image !== $image) { - $savedService->image = $image; - $savedService->save(); - } - - // Collect/create/update networks - if ($serviceNetworks->count() > 0) { - foreach ($serviceNetworks as $networkName => $networkDetails) { - $networkExists = $topLevelNetworks->contains(function ($value, $key) use ($networkName) { - return $value == $networkName || $key == $networkName; - }); - if (!$networkExists) { - $topLevelNetworks->put($networkDetails, null); - } - } - } - - // Collect/create/update ports - $collectedPorts = collect([]); - if ($servicePorts->count() > 0) { - foreach ($servicePorts as $sport) { - if (is_string($sport) || is_numeric($sport)) { - $collectedPorts->push($sport); - } - if (is_array($sport)) { - $target = data_get($sport, 'target'); - $published = data_get($sport, 'published'); - $protocol = data_get($sport, 'protocol'); - $collectedPorts->push("$target:$published/$protocol"); - } - } - } - $savedService->ports = $collectedPorts->implode(','); - $savedService->save(); - - // Add Coolify specific networks - $definedNetworkExists = $topLevelNetworks->contains(function ($value, $_) use ($definedNetwork) { - return $value == $definedNetwork; - }); - if (!$definedNetworkExists) { - foreach ($definedNetwork as $network) { - $topLevelNetworks->put($network, [ - 'name' => $network, - 'external' => true - ]); - } - } - $networks = collect(); - foreach ($serviceNetworks as $key => $serviceNetwork) { - if (gettype($serviceNetwork) === 'string') { - // networks: - // - appwrite - $networks->put($serviceNetwork, null); - } else if (gettype($serviceNetwork) === 'array') { - // networks: - // default: - // ipv4_address: 192.168.203.254 - // $networks->put($serviceNetwork, null); - ray($key); - $networks->put($key, $serviceNetwork); - } - } - foreach ($definedNetwork as $key => $network) { - $networks->put($network, null); - } - data_set($service, 'networks', $networks->toArray()); - - // Collect/create/update volumes - if ($serviceVolumes->count() > 0) { - $serviceVolumes = $serviceVolumes->map(function ($volume) use ($savedService, $topLevelVolumes) { - $type = null; - $source = null; - $target = null; - $content = null; - $isDirectory = false; - if (is_string($volume)) { - $source = Str::of($volume)->before(':'); - $target = Str::of($volume)->after(':')->beforeLast(':'); - if ($source->startsWith('./') || $source->startsWith('/') || $source->startsWith('~')) { - $type = Str::of('bind'); - } else { - $type = Str::of('volume'); - } - } else if (is_array($volume)) { - $type = data_get_str($volume, 'type'); - $source = data_get_str($volume, 'source'); - $target = data_get_str($volume, 'target'); - $content = data_get($volume, 'content'); - $isDirectory = (bool) data_get($volume, 'isDirectory', false); - $foundConfig = $savedService->fileStorages()->whereMountPath($target)->first(); - if ($foundConfig) { - $contentNotNull = data_get($foundConfig, 'content'); - if ($contentNotNull) { - $content = $contentNotNull; - } - $isDirectory = (bool) data_get($foundConfig, 'is_directory'); - } - } - if ($type->value() === 'bind') { - if ($source->value() === "/var/run/docker.sock") { - return $volume; - } - if ($source->value() === '/tmp' || $source->value() === '/tmp/') { - return $volume; - } - LocalFileVolume::updateOrCreate( - [ - 'mount_path' => $target, - 'resource_id' => $savedService->id, - 'resource_type' => get_class($savedService) - ], - [ - 'fs_path' => $source, - 'mount_path' => $target, - 'content' => $content, - 'is_directory' => $isDirectory, - 'resource_id' => $savedService->id, - 'resource_type' => get_class($savedService) - ] - ); - } else if ($type->value() === 'volume') { - $slugWithoutUuid = Str::slug($source, '-'); - $name = "{$savedService->service->uuid}_{$slugWithoutUuid}"; - if (is_string($volume)) { - $source = Str::of($volume)->before(':'); - $target = Str::of($volume)->after(':')->beforeLast(':'); - $source = $name; - $volume = "$source:$target"; - } else if (is_array($volume)) { - data_set($volume, 'source', $name); - } - $topLevelVolumes->put($name, [ - 'name' => $name, - ]); - LocalPersistentVolume::updateOrCreate( - [ - 'mount_path' => $target, - 'resource_id' => $savedService->id, - 'resource_type' => get_class($savedService) - ], - [ - 'name' => $name, - 'mount_path' => $target, - 'resource_id' => $savedService->id, - 'resource_type' => get_class($savedService) - ] - ); - } - $savedService->getFilesFromServer(isInit: true); - return $volume; - }); - data_set($service, 'volumes', $serviceVolumes->toArray()); - } - - // Add env_file with at least .env to the service - // $envFile = collect(data_get($service, 'env_file', [])); - // if ($envFile->count() > 0) { - // if (!$envFile->contains('.env')) { - // $envFile->push('.env'); - // } - // } else { - // $envFile = collect(['.env']); - // } - // data_set($service, 'env_file', $envFile->toArray()); - - - // Get variables from the service - foreach ($serviceVariables as $variableName => $variable) { - if (is_numeric($variableName)) { - $variable = Str::of($variable); - if ($variable->contains('=')) { - // - SESSION_SECRET=123 - // - SESSION_SECRET= - $key = $variable->before('='); - $value = $variable->after('='); - } else { - // - SESSION_SECRET - $key = $variable; - $value = null; - } - } else { - // SESSION_SECRET: 123 - // SESSION_SECRET: - $key = Str::of($variableName); - $value = Str::of($variable); - } - // TODO: here is the problem - if ($key->startsWith('SERVICE_FQDN')) { - if ($isNew || $savedService->fqdn === null) { - $name = $key->after('SERVICE_FQDN_')->beforeLast('_')->lower(); - $fqdn = generateFqdn($this->server, "{$name->value()}-{$this->uuid}"); - if (substr_count($key->value(), '_') === 3) { - // SERVICE_FQDN_UMAMI_1000 - $port = $key->afterLast('_'); - } else { - // SERVICE_FQDN_UMAMI - $port = null; - } - if ($port) { - $fqdn = "$fqdn:$port"; - } - if (substr_count($key->value(), '_') >= 2) { - if (is_null($value)) { - $value = Str::of('/'); - } - $path = $value->value(); - if ($generatedServiceFQDNS->count() > 0) { - $alreadyGenerated = $generatedServiceFQDNS->has($key->value()); - if ($alreadyGenerated) { - $fqdn = $generatedServiceFQDNS->get($key->value()); - } else { - $generatedServiceFQDNS->put($key->value(), $fqdn); - } - } else { - $generatedServiceFQDNS->put($key->value(), $fqdn); - } - $fqdn = "$fqdn$path"; - } - - if (!$isDatabase) { - if ($savedService->fqdn) { - $fqdn = $savedService->fqdn . ',' . $fqdn; - } else { - $fqdn = $fqdn; - } - $savedService->fqdn = $fqdn; - $savedService->save(); - } - } - // data_forget($service, "environment.$variableName"); - // $yaml = data_forget($yaml, "services.$serviceName.environment.$variableName"); - // if (count(data_get($yaml, 'services.' . $serviceName . '.environment')) === 0) { - // $yaml = data_forget($yaml, "services.$serviceName.environment"); - // } - continue; - } - if ($value?->startsWith('$')) { - $value = Str::of(replaceVariables($value)); - $key = $value; - $foundEnv = EnvironmentVariable::where([ - 'key' => $key, - 'service_id' => $this->id, - ])->first(); - if ($value->startsWith('SERVICE_')) { - // Count _ in $value - $count = substr_count($value->value(), '_'); - if ($count === 2) { - // SERVICE_FQDN_UMAMI - $command = $value->after('SERVICE_')->beforeLast('_'); - $forService = $value->afterLast('_'); - $generatedValue = null; - $port = null; - } - if ($count === 3) { - // SERVICE_FQDN_UMAMI_1000 - $command = $value->after('SERVICE_')->before('_'); - $forService = $value->after('SERVICE_')->after('_')->before('_'); - $generatedValue = null; - $port = $value->afterLast('_'); - } - if ($command->value() === 'FQDN' || $command->value() === 'URL') { - if (Str::lower($forService) === $serviceName) { - $fqdn = generateFqdn($this->server, $containerName); - } else { - $fqdn = generateFqdn($this->server, Str::lower($forService) . '-' . $this->uuid); - } - if ($port) { - $fqdn = "$fqdn:$port"; - } - if ($foundEnv) { - $fqdn = data_get($foundEnv, 'value'); - } else { - if ($command->value() === 'URL') { - $fqdn = Str::of($fqdn)->after('://')->value(); - } - EnvironmentVariable::create([ - 'key' => $key, - 'value' => $fqdn, - 'is_build_time' => false, - 'service_id' => $this->id, - 'is_preview' => false, - ]); - } - if (!$isDatabase) { - if ($command->value() === 'FQDN' && is_null($savedService->fqdn)) { - $savedService->fqdn = $fqdn; - $savedService->save(); - } - } - } else { - switch ($command) { - case 'PASSWORD': - $generatedValue = Str::password(symbols: false); - break; - case 'PASSWORD_64': - $generatedValue = Str::password(length: 64, symbols: false); - break; - case 'BASE64_64': - $generatedValue = Str::random(64); - break; - case 'BASE64_128': - $generatedValue = Str::random(128); - break; - case 'BASE64': - $generatedValue = Str::random(32); - break; - case 'USER': - $generatedValue = Str::random(16); - break; - } - - if (!$foundEnv) { - EnvironmentVariable::create([ - 'key' => $key, - 'value' => $generatedValue, - 'is_build_time' => false, - 'service_id' => $this->id, - 'is_preview' => false, - ]); - } - } - } else { - if ($value->contains(':-')) { - $key = $value->before(':'); - $defaultValue = $value->after(':-'); - } else if ($value->contains('-')) { - $key = $value->before('-'); - $defaultValue = $value->after('-'); - } else if ($value->contains(':?')) { - $key = $value->before(':'); - $defaultValue = $value->after(':?'); - } else if ($value->contains('?')) { - $key = $value->before('?'); - $defaultValue = $value->after('?'); - } else { - $key = $value; - $defaultValue = null; - } - if ($foundEnv) { - $defaultValue = data_get($foundEnv, 'value'); - } - EnvironmentVariable::updateOrCreate([ - 'key' => $key, - 'service_id' => $this->id, - ], [ - 'value' => $defaultValue, - 'is_build_time' => false, - 'service_id' => $this->id, - 'is_preview' => false, - ]); - } - } - } - - // Add labels to the service - if ($savedService->serviceType()) { - $fqdns = generateServiceSpecificFqdns($savedService, forTraefik: true); - } else { - $fqdns = collect(data_get($savedService, 'fqdns')); - } - $defaultLabels = defaultLabels($this->id, $containerName, type: 'service', subType: $isDatabase ? 'database' : 'application', subId: $savedService->id); - $serviceLabels = $serviceLabels->merge($defaultLabels); - if (!$isDatabase && $fqdns->count() > 0) { - if ($fqdns) { - $serviceLabels = $serviceLabels->merge(fqdnLabelsForTraefik($this->uuid, $fqdns, true)); - } - } - if ($this->server->isLogDrainEnabled() && $savedService->isLogDrainEnabled()) { - data_set($service, 'logging', [ - 'driver' => 'fluentd', - 'options' => [ - 'fluentd-address' => "tcp://127.0.0.1:24224", - 'fluentd-async' => "true", - 'fluentd-sub-second-precision' => "true", - ] - ]); - } - data_set($service, 'labels', $serviceLabels->toArray()); - data_forget($service, 'is_database'); - data_set($service, 'restart', RESTART_MODE); - data_set($service, 'container_name', $containerName); - data_forget($service, 'volumes.*.content'); - data_forget($service, 'volumes.*.isDirectory'); - // Remove unnecessary variables from service.environment - // $withoutServiceEnvs = collect([]); - // collect(data_get($service, 'environment'))->each(function ($value, $key) use ($withoutServiceEnvs) { - // ray($key, $value); - // if (!Str::of($key)->startsWith('$SERVICE_') && !Str::of($value)->startsWith('SERVICE_')) { - // $k = Str::of($value)->before("="); - // $v = Str::of($value)->after("="); - // $withoutServiceEnvs->put($k->value(), $v->value()); - // } - // }); - // ray($withoutServiceEnvs); - // data_set($service, 'environment', $withoutServiceEnvs->toArray()); - return $service; - }); - $finalServices = [ - 'version' => $dockerComposeVersion, - 'services' => $services->toArray(), - 'volumes' => $topLevelVolumes->toArray(), - 'networks' => $topLevelNetworks->toArray(), - ]; - $this->docker_compose_raw = Yaml::dump($yaml, 10, 2); - $this->docker_compose = Yaml::dump($finalServices, 10, 2); - $this->save(); - $this->saveComposeConfigs(); - return collect([]); - } else { - return collect([]); - } + return parseDockerComposeFile($this, $isNew); } } diff --git a/app/Traits/ExecuteRemoteCommand.php b/app/Traits/ExecuteRemoteCommand.php index 1306f645c..42d0d9e0f 100644 --- a/app/Traits/ExecuteRemoteCommand.php +++ b/app/Traits/ExecuteRemoteCommand.php @@ -12,6 +12,7 @@ use Illuminate\Support\Str; trait ExecuteRemoteCommand { public ?string $save = null; + public static int $batch_counter = 0; public function execute_remote_command(...$commands) { static::$batch_counter++; @@ -24,7 +25,6 @@ trait ExecuteRemoteCommand throw new \RuntimeException('Server is not set or is not an instance of Server model'); } - $commandsText->each(function ($single_command) { $command = data_get($single_command, 'command') ?? $single_command[0] ?? null; if ($command === null) { diff --git a/bootstrap/helpers/docker.php b/bootstrap/helpers/docker.php index 36f8733b3..fef40aed8 100644 --- a/bootstrap/helpers/docker.php +++ b/bootstrap/helpers/docker.php @@ -3,6 +3,9 @@ use App\Models\Application; use App\Models\ApplicationPreview; use App\Models\Server; +use App\Models\Service; +use App\Models\ServiceApplication; +use App\Models\ServiceDatabase; use Illuminate\Support\Collection; use Illuminate\Support\Str; use Spatie\Url\Url; @@ -137,18 +140,28 @@ function defaultLabels($id, $name, $pull_request_id = 0, string $type = 'applica $labels->push('coolify.name=' . $name); $labels->push('coolify.pullRequestId=' . $pull_request_id); if ($type === 'service') { - $labels->push('coolify.service.subId=' . $subId); - $labels->push('coolify.service.subType=' . $subType); + $subId && $labels->push('coolify.service.subId=' . $subId); + $subType && $labels->push('coolify.service.subType=' . $subType); } return $labels; } -function generateServiceSpecificFqdns($service, $forTraefik = false) +function generateServiceSpecificFqdns(ServiceApplication|Application $resource, $forTraefik = false) { - $variables = collect($service->service->environment_variables); - $type = $service->serviceType(); + if ($resource->getMorphClass() === 'App\Models\ServiceApplication') { + $uuid = $resource->uuid; + $server = $resource->service->server; + $environment_variables = $resource->service->environment_variables; + $type = $resource->serviceType(); + } else if ($resource->getMorphClass() === 'App\Models\Application') { + $uuid = $resource->uuid; + $server = $resource->destination->server; + $environment_variables = $resource->environment_variables; + $type = $resource->serviceType(); + } + $variables = collect($environment_variables); $payload = collect([]); switch ($type) { - case $type->contains('minio'): + case $type?->contains('minio'): $MINIO_BROWSER_REDIRECT_URL = $variables->where('key', 'MINIO_BROWSER_REDIRECT_URL')->first(); $MINIO_SERVER_URL = $variables->where('key', 'MINIO_SERVER_URL')->first(); if (is_null($MINIO_BROWSER_REDIRECT_URL) || is_null($MINIO_SERVER_URL)) { @@ -156,12 +169,12 @@ function generateServiceSpecificFqdns($service, $forTraefik = false) } if (is_null($MINIO_BROWSER_REDIRECT_URL?->value)) { $MINIO_BROWSER_REDIRECT_URL?->update([ - "value" => generateFqdn($service->service->server, 'console-' . $service->uuid) + "value" => generateFqdn($server, 'console-' . $uuid) ]); } if (is_null($MINIO_SERVER_URL?->value)) { $MINIO_SERVER_URL?->update([ - "value" => generateFqdn($service->service->server, 'minio-' . $service->uuid) + "value" => generateFqdn($server, 'minio-' . $uuid) ]); } if ($forTraefik) { diff --git a/bootstrap/helpers/proxy.php b/bootstrap/helpers/proxy.php index ee9c624f5..595d6c5ad 100644 --- a/bootstrap/helpers/proxy.php +++ b/bootstrap/helpers/proxy.php @@ -12,6 +12,7 @@ function get_proxy_path() } function connectProxyToNetworks(Server $server) { + // TODO: Connect to service + compose based application networks as well. $networks = collect($server->standaloneDockers)->map(function ($docker) { return $docker['network']; })->unique(); diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php index 839f86835..cff60233a 100644 --- a/bootstrap/helpers/shared.php +++ b/bootstrap/helpers/shared.php @@ -1,9 +1,14 @@ source_id !== 0 && !is_null($resource->source_id)) { return null; } @@ -487,3 +494,852 @@ function removeAnsiColors($text) { return preg_replace('/\e[[][A-Za-z0-9];?[0-9]*m?/', '', $text); } + +function parseDockerComposeFile(Service|Application $resource, bool $isNew = false) +{ + ray()->clearAll(); + if ($resource->getMorphClass() === 'App\Models\Service') { + if ($resource->docker_compose_raw) { + try { + $yaml = Yaml::parse($resource->docker_compose_raw); + } catch (\Exception $e) { + throw new \Exception($e->getMessage()); + } + + $topLevelVolumes = collect(data_get($yaml, 'volumes', [])); + $topLevelNetworks = collect(data_get($yaml, 'networks', [])); + $dockerComposeVersion = data_get($yaml, 'version') ?? '3.8'; + $services = data_get($yaml, 'services'); + + $generatedServiceFQDNS = collect([]); + if (is_null($resource->destination)) { + $destination = $resource->server->destinations()->first(); + if ($destination) { + $resource->destination()->associate($destination); + $resource->save(); + } + } + $definedNetwork = collect([$resource->uuid]); + + $services = collect($services)->map(function ($service, $serviceName) use ($topLevelVolumes, $topLevelNetworks, $definedNetwork, $isNew, $generatedServiceFQDNS, $resource) { + $serviceVolumes = collect(data_get($service, 'volumes', [])); + $servicePorts = collect(data_get($service, 'ports', [])); + $serviceNetworks = collect(data_get($service, 'networks', [])); + $serviceVariables = collect(data_get($service, 'environment', [])); + $serviceLabels = collect(data_get($service, 'labels', [])); + if ($serviceLabels->count() > 0) { + $removedLabels = collect([]); + $serviceLabels = $serviceLabels->filter(function ($serviceLabel, $serviceLabelName) use ($removedLabels) { + if (!str($serviceLabel)->contains('=')) { + $removedLabels->put($serviceLabelName, $serviceLabel); + return false; + } + return $serviceLabel; + }); + foreach ($removedLabels as $removedLabelName => $removedLabel) { + $serviceLabels->push("$removedLabelName=$removedLabel"); + } + } + + $containerName = "$serviceName-{$resource->uuid}"; + + // Decide if the service is a database + $isDatabase = false; + $image = data_get_str($service, 'image'); + if ($image->contains(':')) { + $image = Str::of($image); + } else { + $image = Str::of($image)->append(':latest'); + } + $imageName = $image->before(':'); + + if (collect(DATABASE_DOCKER_IMAGES)->contains($imageName)) { + $isDatabase = true; + } + data_set($service, 'is_database', $isDatabase); + + // Create new serviceApplication or serviceDatabase + if ($isDatabase) { + if ($isNew) { + $savedService = ServiceDatabase::create([ + 'name' => $serviceName, + 'image' => $image, + 'service_id' => $resource->id + ]); + } else { + $savedService = ServiceDatabase::where([ + 'name' => $serviceName, + 'service_id' => $resource->id + ])->first(); + } + } else { + if ($isNew) { + $savedService = ServiceApplication::create([ + 'name' => $serviceName, + 'image' => $image, + 'service_id' => $resource->id + ]); + } else { + $savedService = ServiceApplication::where([ + 'name' => $serviceName, + 'service_id' => $resource->id + ])->first(); + } + } + if (is_null($savedService)) { + if ($isDatabase) { + $savedService = ServiceDatabase::create([ + 'name' => $serviceName, + 'image' => $image, + 'service_id' => $resource->id + ]); + } else { + $savedService = ServiceApplication::create([ + 'name' => $serviceName, + 'image' => $image, + 'service_id' => $resource->id + ]); + } + } + + // Check if image changed + if ($savedService->image !== $image) { + $savedService->image = $image; + $savedService->save(); + } + + // Collect/create/update networks + if ($serviceNetworks->count() > 0) { + foreach ($serviceNetworks as $networkName => $networkDetails) { + $networkExists = $topLevelNetworks->contains(function ($value, $key) use ($networkName) { + return $value == $networkName || $key == $networkName; + }); + if (!$networkExists) { + $topLevelNetworks->put($networkDetails, null); + } + } + } + + // Collect/create/update ports + $collectedPorts = collect([]); + if ($servicePorts->count() > 0) { + foreach ($servicePorts as $sport) { + if (is_string($sport) || is_numeric($sport)) { + $collectedPorts->push($sport); + } + if (is_array($sport)) { + $target = data_get($sport, 'target'); + $published = data_get($sport, 'published'); + $protocol = data_get($sport, 'protocol'); + $collectedPorts->push("$target:$published/$protocol"); + } + } + } + $savedService->ports = $collectedPorts->implode(','); + $savedService->save(); + + // Add Coolify specific networks + $definedNetworkExists = $topLevelNetworks->contains(function ($value, $_) use ($definedNetwork) { + return $value == $definedNetwork; + }); + if (!$definedNetworkExists) { + foreach ($definedNetwork as $network) { + $topLevelNetworks->put($network, [ + 'name' => $network, + 'external' => true + ]); + } + } + $networks = collect(); + foreach ($serviceNetworks as $key => $serviceNetwork) { + if (gettype($serviceNetwork) === 'string') { + // networks: + // - appwrite + $networks->put($serviceNetwork, null); + } else if (gettype($serviceNetwork) === 'array') { + // networks: + // default: + // ipv4_address: 192.168.203.254 + // $networks->put($serviceNetwork, null); + ray($key); + $networks->put($key, $serviceNetwork); + } + } + foreach ($definedNetwork as $key => $network) { + $networks->put($network, null); + } + data_set($service, 'networks', $networks->toArray()); + + // Collect/create/update volumes + if ($serviceVolumes->count() > 0) { + $serviceVolumes = $serviceVolumes->map(function ($volume) use ($savedService, $topLevelVolumes) { + $type = null; + $source = null; + $target = null; + $content = null; + $isDirectory = false; + if (is_string($volume)) { + $source = Str::of($volume)->before(':'); + $target = Str::of($volume)->after(':')->beforeLast(':'); + if ($source->startsWith('./') || $source->startsWith('/') || $source->startsWith('~')) { + $type = Str::of('bind'); + } else { + $type = Str::of('volume'); + } + } else if (is_array($volume)) { + $type = data_get_str($volume, 'type'); + $source = data_get_str($volume, 'source'); + $target = data_get_str($volume, 'target'); + $content = data_get($volume, 'content'); + $isDirectory = (bool) data_get($volume, 'isDirectory', false); + $foundConfig = $savedService->fileStorages()->whereMountPath($target)->first(); + if ($foundConfig) { + $contentNotNull = data_get($foundConfig, 'content'); + if ($contentNotNull) { + $content = $contentNotNull; + } + $isDirectory = (bool) data_get($foundConfig, 'is_directory'); + } + } + if ($type->value() === 'bind') { + if ($source->value() === "/var/run/docker.sock") { + return $volume; + } + if ($source->value() === '/tmp' || $source->value() === '/tmp/') { + return $volume; + } + LocalFileVolume::updateOrCreate( + [ + 'mount_path' => $target, + 'resource_id' => $savedService->id, + 'resource_type' => get_class($savedService) + ], + [ + 'fs_path' => $source, + 'mount_path' => $target, + 'content' => $content, + 'is_directory' => $isDirectory, + 'resource_id' => $savedService->id, + 'resource_type' => get_class($savedService) + ] + ); + } else if ($type->value() === 'volume') { + $slugWithoutUuid = Str::slug($source, '-'); + $name = "{$savedService->service->uuid}_{$slugWithoutUuid}"; + if (is_string($volume)) { + $source = Str::of($volume)->before(':'); + $target = Str::of($volume)->after(':')->beforeLast(':'); + $source = $name; + $volume = "$source:$target"; + } else if (is_array($volume)) { + data_set($volume, 'source', $name); + } + $topLevelVolumes->put($name, [ + 'name' => $name, + ]); + LocalPersistentVolume::updateOrCreate( + [ + 'mount_path' => $target, + 'resource_id' => $savedService->id, + 'resource_type' => get_class($savedService) + ], + [ + 'name' => $name, + 'mount_path' => $target, + 'resource_id' => $savedService->id, + 'resource_type' => get_class($savedService) + ] + ); + } + $savedService->getFilesFromServer(isInit: true); + return $volume; + }); + data_set($service, 'volumes', $serviceVolumes->toArray()); + } + + // Add env_file with at least .env to the service + // $envFile = collect(data_get($service, 'env_file', [])); + // if ($envFile->count() > 0) { + // if (!$envFile->contains('.env')) { + // $envFile->push('.env'); + // } + // } else { + // $envFile = collect(['.env']); + // } + // data_set($service, 'env_file', $envFile->toArray()); + + + // Get variables from the service + foreach ($serviceVariables as $variableName => $variable) { + if (is_numeric($variableName)) { + $variable = Str::of($variable); + if ($variable->contains('=')) { + // - SESSION_SECRET=123 + // - SESSION_SECRET= + $key = $variable->before('='); + $value = $variable->after('='); + } else { + // - SESSION_SECRET + $key = $variable; + $value = null; + } + } else { + // SESSION_SECRET: 123 + // SESSION_SECRET: + $key = Str::of($variableName); + $value = Str::of($variable); + } + // TODO: here is the problem + if ($key->startsWith('SERVICE_FQDN')) { + if ($isNew || $savedService->fqdn === null) { + $name = $key->after('SERVICE_FQDN_')->beforeLast('_')->lower(); + $fqdn = generateFqdn($resource->server, "{$name->value()}-{$resource->uuid}"); + if (substr_count($key->value(), '_') === 3) { + // SERVICE_FQDN_UMAMI_1000 + $port = $key->afterLast('_'); + } else { + // SERVICE_FQDN_UMAMI + $port = null; + } + if ($port) { + $fqdn = "$fqdn:$port"; + } + if (substr_count($key->value(), '_') >= 2) { + if (is_null($value)) { + $value = Str::of('/'); + } + $path = $value->value(); + if ($generatedServiceFQDNS->count() > 0) { + $alreadyGenerated = $generatedServiceFQDNS->has($key->value()); + if ($alreadyGenerated) { + $fqdn = $generatedServiceFQDNS->get($key->value()); + } else { + $generatedServiceFQDNS->put($key->value(), $fqdn); + } + } else { + $generatedServiceFQDNS->put($key->value(), $fqdn); + } + $fqdn = "$fqdn$path"; + } + + if (!$isDatabase) { + if ($savedService->fqdn) { + $fqdn = $savedService->fqdn . ',' . $fqdn; + } else { + $fqdn = $fqdn; + } + $savedService->fqdn = $fqdn; + $savedService->save(); + } + } + // data_forget($service, "environment.$variableName"); + // $yaml = data_forget($yaml, "services.$serviceName.environment.$variableName"); + // if (count(data_get($yaml, 'services.' . $serviceName . '.environment')) === 0) { + // $yaml = data_forget($yaml, "services.$serviceName.environment"); + // } + continue; + } + if ($value?->startsWith('$')) { + $value = Str::of(replaceVariables($value)); + $key = $value; + $foundEnv = EnvironmentVariable::where([ + 'key' => $key, + 'service_id' => $resource->id, + ])->first(); + if ($value->startsWith('SERVICE_')) { + // Count _ in $value + $count = substr_count($value->value(), '_'); + if ($count === 2) { + // SERVICE_FQDN_UMAMI + $command = $value->after('SERVICE_')->beforeLast('_'); + $forService = $value->afterLast('_'); + $generatedValue = null; + $port = null; + } + if ($count === 3) { + // SERVICE_FQDN_UMAMI_1000 + $command = $value->after('SERVICE_')->before('_'); + $forService = $value->after('SERVICE_')->after('_')->before('_'); + $generatedValue = null; + $port = $value->afterLast('_'); + } + if ($command->value() === 'FQDN' || $command->value() === 'URL') { + if (Str::lower($forService) === $serviceName) { + $fqdn = generateFqdn($resource->server, $containerName); + } else { + $fqdn = generateFqdn($resource->server, Str::lower($forService) . '-' . $resource->uuid); + } + if ($port) { + $fqdn = "$fqdn:$port"; + } + if ($foundEnv) { + $fqdn = data_get($foundEnv, 'value'); + } else { + if ($command->value() === 'URL') { + $fqdn = Str::of($fqdn)->after('://')->value(); + } + EnvironmentVariable::create([ + 'key' => $key, + 'value' => $fqdn, + 'is_build_time' => false, + 'service_id' => $resource->id, + 'is_preview' => false, + ]); + } + if (!$isDatabase) { + if ($command->value() === 'FQDN' && is_null($savedService->fqdn)) { + $savedService->fqdn = $fqdn; + $savedService->save(); + } + } + } else { + switch ($command) { + case 'PASSWORD': + $generatedValue = Str::password(symbols: false); + break; + case 'PASSWORD_64': + $generatedValue = Str::password(length: 64, symbols: false); + break; + case 'BASE64_64': + $generatedValue = Str::random(64); + break; + case 'BASE64_128': + $generatedValue = Str::random(128); + break; + case 'BASE64': + $generatedValue = Str::random(32); + break; + case 'USER': + $generatedValue = Str::random(16); + break; + } + + if (!$foundEnv) { + EnvironmentVariable::create([ + 'key' => $key, + 'value' => $generatedValue, + 'is_build_time' => false, + 'service_id' => $resource->id, + 'is_preview' => false, + ]); + } + } + } else { + if ($value->contains(':-')) { + $key = $value->before(':'); + $defaultValue = $value->after(':-'); + } else if ($value->contains('-')) { + $key = $value->before('-'); + $defaultValue = $value->after('-'); + } else if ($value->contains(':?')) { + $key = $value->before(':'); + $defaultValue = $value->after(':?'); + } else if ($value->contains('?')) { + $key = $value->before('?'); + $defaultValue = $value->after('?'); + } else { + $key = $value; + $defaultValue = null; + } + if ($foundEnv) { + $defaultValue = data_get($foundEnv, 'value'); + } + EnvironmentVariable::updateOrCreate([ + 'key' => $key, + 'service_id' => $resource->id, + ], [ + 'value' => $defaultValue, + 'is_build_time' => false, + 'service_id' => $resource->id, + 'is_preview' => false, + ]); + } + } + } + + // Add labels to the service + if ($savedService->serviceType()) { + $fqdns = generateServiceSpecificFqdns($savedService, forTraefik: true); + } else { + $fqdns = collect(data_get($savedService, 'fqdns')); + } + $defaultLabels = defaultLabels($resource->id, $containerName, type: 'service', subType: $isDatabase ? 'database' : 'application', subId: $savedService->id); + $serviceLabels = $serviceLabels->merge($defaultLabels); + if (!$isDatabase && $fqdns->count() > 0) { + if ($fqdns) { + $serviceLabels = $serviceLabels->merge(fqdnLabelsForTraefik($resource->uuid, $fqdns, true)); + } + } + if ($resource->server->isLogDrainEnabled() && $savedService->isLogDrainEnabled()) { + data_set($service, 'logging', [ + 'driver' => 'fluentd', + 'options' => [ + 'fluentd-address' => "tcp://127.0.0.1:24224", + 'fluentd-async' => "true", + 'fluentd-sub-second-precision' => "true", + ] + ]); + } + data_set($service, 'labels', $serviceLabels->toArray()); + data_forget($service, 'is_database'); + data_set($service, 'restart', RESTART_MODE); + data_set($service, 'container_name', $containerName); + data_forget($service, 'volumes.*.content'); + data_forget($service, 'volumes.*.isDirectory'); + // Remove unnecessary variables from service.environment + // $withoutServiceEnvs = collect([]); + // collect(data_get($service, 'environment'))->each(function ($value, $key) use ($withoutServiceEnvs) { + // ray($key, $value); + // if (!Str::of($key)->startsWith('$SERVICE_') && !Str::of($value)->startsWith('SERVICE_')) { + // $k = Str::of($value)->before("="); + // $v = Str::of($value)->after("="); + // $withoutServiceEnvs->put($k->value(), $v->value()); + // } + // }); + // ray($withoutServiceEnvs); + // data_set($service, 'environment', $withoutServiceEnvs->toArray()); + return $service; + }); + $finalServices = [ + 'version' => $dockerComposeVersion, + 'services' => $services->toArray(), + 'volumes' => $topLevelVolumes->toArray(), + 'networks' => $topLevelNetworks->toArray(), + ]; + $resource->docker_compose_raw = Yaml::dump($yaml, 10, 2); + $resource->docker_compose = Yaml::dump($finalServices, 10, 2); + $resource->save(); + $resource->saveComposeConfigs(); + return collect([]); + } else { + return collect([]); + } + } else if ($resource->getMorphClass() === 'App\Models\Application') { + try { + $yaml = Yaml::parse($resource->docker_compose_raw); + } catch (\Exception $e) { + throw new \Exception($e->getMessage()); + } + $server = $resource->destination->server; + $topLevelVolumes = collect(data_get($yaml, 'volumes', [])); + $topLevelNetworks = collect(data_get($yaml, 'networks', [])); + $dockerComposeVersion = data_get($yaml, 'version') ?? '3.8'; + $services = data_get($yaml, 'services'); + + $generatedServiceFQDNS = collect([]); + if (is_null($resource->destination)) { + $destination = $server->destinations()->first(); + if ($destination) { + $resource->destination()->associate($destination); + $resource->save(); + } + } + $definedNetwork = collect([$resource->uuid]); + $services = collect($services)->map(function ($service, $serviceName) use ($topLevelVolumes, $topLevelNetworks, $definedNetwork, $isNew, $generatedServiceFQDNS, $resource, $server) { + $serviceVolumes = collect(data_get($service, 'volumes', [])); + $servicePorts = collect(data_get($service, 'ports', [])); + $serviceNetworks = collect(data_get($service, 'networks', [])); + $serviceVariables = collect(data_get($service, 'environment', [])); + $serviceLabels = collect(data_get($service, 'labels', [])); + if ($serviceLabels->count() > 0) { + $removedLabels = collect([]); + $serviceLabels = $serviceLabels->filter(function ($serviceLabel, $serviceLabelName) use ($removedLabels) { + if (!str($serviceLabel)->contains('=')) { + $removedLabels->put($serviceLabelName, $serviceLabel); + return false; + } + return $serviceLabel; + }); + foreach ($removedLabels as $removedLabelName => $removedLabel) { + $serviceLabels->push("$removedLabelName=$removedLabel"); + } + } + + $containerName = "$serviceName-{$resource->uuid}"; + + // Decide if the service is a database + $isDatabase = false; + $image = data_get_str($service, 'image'); + if ($image->contains(':')) { + $image = Str::of($image); + } else { + $image = Str::of($image)->append(':latest'); + } + $imageName = $image->before(':'); + + if (collect(DATABASE_DOCKER_IMAGES)->contains($imageName)) { + $isDatabase = true; + } + data_set($service, 'is_database', $isDatabase); + + // Collect/create/update networks + if ($serviceNetworks->count() > 0) { + foreach ($serviceNetworks as $networkName => $networkDetails) { + $networkExists = $topLevelNetworks->contains(function ($value, $key) use ($networkName) { + return $value == $networkName || $key == $networkName; + }); + if (!$networkExists) { + $topLevelNetworks->put($networkDetails, null); + } + } + } + // Collect/create/update ports + $collectedPorts = collect([]); + if ($servicePorts->count() > 0) { + foreach ($servicePorts as $sport) { + if (is_string($sport) || is_numeric($sport)) { + $collectedPorts->push($sport); + } + if (is_array($sport)) { + $target = data_get($sport, 'target'); + $published = data_get($sport, 'published'); + $protocol = data_get($sport, 'protocol'); + $collectedPorts->push("$target:$published/$protocol"); + } + } + } + if ($collectedPorts->count() > 0) { + // ray($collectedPorts->implode(',')); + } + $definedNetworkExists = $topLevelNetworks->contains(function ($value, $_) use ($definedNetwork) { + return $value == $definedNetwork; + }); + if (!$definedNetworkExists) { + foreach ($definedNetwork as $network) { + $topLevelNetworks->put($network, [ + 'name' => $network, + 'external' => true + ]); + } + } + $networks = collect(); + foreach ($serviceNetworks as $key => $serviceNetwork) { + if (gettype($serviceNetwork) === 'string') { + // networks: + // - appwrite + $networks->put($serviceNetwork, null); + } else if (gettype($serviceNetwork) === 'array') { + // networks: + // default: + // ipv4_address: 192.168.203.254 + // $networks->put($serviceNetwork, null); + $networks->put($key, $serviceNetwork); + } + } + foreach ($definedNetwork as $key => $network) { + $networks->put($network, null); + } + data_set($service, 'networks', $networks->toArray()); + // Get variables from the service + foreach ($serviceVariables as $variableName => $variable) { + if (is_numeric($variableName)) { + $variable = Str::of($variable); + if ($variable->contains('=')) { + // - SESSION_SECRET=123 + // - SESSION_SECRET= + $key = $variable->before('='); + $value = $variable->after('='); + } else { + // - SESSION_SECRET + $key = $variable; + $value = null; + } + } else { + // SESSION_SECRET: 123 + // SESSION_SECRET: + $key = Str::of($variableName); + $value = Str::of($variable); + } + // TODO: here is the problem + if ($key->startsWith('SERVICE_FQDN')) { + if ($isNew) { + $name = $key->after('SERVICE_FQDN_')->beforeLast('_')->lower(); + $fqdn = generateFqdn($server, "{$name->value()}-{$resource->uuid}"); + if (substr_count($key->value(), '_') === 3) { + // SERVICE_FQDN_UMAMI_1000 + $port = $key->afterLast('_'); + } else { + // SERVICE_FQDN_UMAMI + $port = null; + } + if ($port) { + $fqdn = "$fqdn:$port"; + } + if (substr_count($key->value(), '_') >= 2) { + if (is_null($value)) { + $value = Str::of('/'); + } + $path = $value->value(); + if ($generatedServiceFQDNS->count() > 0) { + $alreadyGenerated = $generatedServiceFQDNS->has($key->value()); + if ($alreadyGenerated) { + $fqdn = $generatedServiceFQDNS->get($key->value()); + } else { + $generatedServiceFQDNS->put($key->value(), $fqdn); + } + } else { + $generatedServiceFQDNS->put($key->value(), $fqdn); + } + $fqdn = "$fqdn$path"; + } + } + continue; + } + if ($value?->startsWith('$')) { + $value = Str::of(replaceVariables($value)); + $key = $value; + $foundEnv = EnvironmentVariable::where([ + 'key' => $key, + 'application_id' => $resource->id, + ])->first(); + if ($value->startsWith('SERVICE_')) { + // Count _ in $value + $count = substr_count($value->value(), '_'); + if ($count === 2) { + // SERVICE_FQDN_UMAMI + $command = $value->after('SERVICE_')->beforeLast('_'); + $forService = $value->afterLast('_'); + $generatedValue = null; + $port = null; + } + if ($count === 3) { + // SERVICE_FQDN_UMAMI_1000 + $command = $value->after('SERVICE_')->before('_'); + $forService = $value->after('SERVICE_')->after('_')->before('_'); + $generatedValue = null; + $port = $value->afterLast('_'); + } + if ($command->value() === 'FQDN' || $command->value() === 'URL') { + if (Str::lower($forService) === $serviceName) { + $fqdn = generateFqdn($server, $containerName); + } else { + $fqdn = generateFqdn($server, Str::lower($forService) . '-' . $resource->uuid); + } + if ($port) { + $fqdn = "$fqdn:$port"; + } + if ($foundEnv) { + $fqdn = data_get($foundEnv, 'value'); + } else { + if ($command->value() === 'URL') { + $fqdn = Str::of($fqdn)->after('://')->value(); + } + EnvironmentVariable::create([ + 'key' => $key, + 'value' => $fqdn, + 'is_build_time' => false, + 'application_id' => $resource->id, + 'is_preview' => false, + ]); + } + } else { + switch ($command) { + case 'PASSWORD': + $generatedValue = Str::password(symbols: false); + break; + case 'PASSWORD_64': + $generatedValue = Str::password(length: 64, symbols: false); + break; + case 'BASE64_64': + $generatedValue = Str::random(64); + break; + case 'BASE64_128': + $generatedValue = Str::random(128); + break; + case 'BASE64': + $generatedValue = Str::random(32); + break; + case 'USER': + $generatedValue = Str::random(16); + break; + } + + if (!$foundEnv) { + EnvironmentVariable::create([ + 'key' => $key, + 'value' => $generatedValue, + 'is_build_time' => false, + 'application_id' => $resource->id, + 'is_preview' => false, + ]); + } + } + } else { + if ($value->contains(':-')) { + $key = $value->before(':'); + $defaultValue = $value->after(':-'); + } else if ($value->contains('-')) { + $key = $value->before('-'); + $defaultValue = $value->after('-'); + } else if ($value->contains(':?')) { + $key = $value->before(':'); + $defaultValue = $value->after(':?'); + } else if ($value->contains('?')) { + $key = $value->before('?'); + $defaultValue = $value->after('?'); + } else { + $key = $value; + $defaultValue = null; + } + if ($foundEnv) { + $defaultValue = data_get($foundEnv, 'value'); + } + EnvironmentVariable::updateOrCreate([ + 'key' => $key, + 'application_id' => $resource->id, + ], [ + 'value' => $defaultValue, + 'is_build_time' => false, + 'service_id' => $resource->id, + 'is_preview' => false, + ]); + } + } + } + // Add labels to the service + if ($resource->serviceType()) { + $fqdns = generateServiceSpecificFqdns($resource, forTraefik: true); + } else { + $domains = collect(json_decode($resource->docker_compose_domains)) ?? []; + if ($domains) { + $fqdns = data_get($domains, "$serviceName.domain"); + if ($fqdns) { + $fqdns = str($fqdns)->explode(','); + $serviceLabels = $serviceLabels->merge(fqdnLabelsForTraefik($resource->uuid, $fqdns, true)); + } + } + } + $defaultLabels = defaultLabels($resource->id, $containerName, type: 'application'); + + $serviceLabels = $serviceLabels->merge($defaultLabels); + + if ($server->isLogDrainEnabled() && $resource->isLogDrainEnabled()) { + data_set($service, 'logging', [ + 'driver' => 'fluentd', + 'options' => [ + 'fluentd-address' => "tcp://127.0.0.1:24224", + 'fluentd-async' => "true", + 'fluentd-sub-second-precision' => "true", + ] + ]); + } + data_set($service, 'labels', $serviceLabels->toArray()); + data_forget($service, 'is_database'); + data_set($service, 'restart', RESTART_MODE); + data_set($service, 'container_name', $containerName); + data_forget($service, 'volumes.*.content'); + data_forget($service, 'volumes.*.isDirectory'); + return $service; + }); + $finalServices = [ + 'version' => $dockerComposeVersion, + 'services' => $services->toArray(), + 'volumes' => $topLevelVolumes->toArray(), + 'networks' => $topLevelNetworks->toArray(), + ]; + $resource->docker_compose_raw = Yaml::dump($yaml, 10, 2); + $resource->docker_compose = Yaml::dump($finalServices, 10, 2); + $resource->save(); + return collect($finalServices); + } +} diff --git a/database/migrations/2023_11_24_080341_add_docker_compose_location.php b/database/migrations/2023_11_24_080341_add_docker_compose_location.php new file mode 100644 index 000000000..b811aa4d1 --- /dev/null +++ b/database/migrations/2023_11_24_080341_add_docker_compose_location.php @@ -0,0 +1,35 @@ +string('docker_compose_location')->nullable()->default('/docker-compose.yml')->after('dockerfile_location'); + $table->longText('docker_compose')->nullable()->after('docker_compose_location'); + $table->longText('docker_compose_raw')->nullable()->after('docker_compose'); + $table->text('docker_compose_domains')->nullable()->after('docker_compose_raw'); + + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('applications', function (Blueprint $table) { + $table->dropColumn('docker_compose_location'); + $table->dropColumn('docker_compose'); + $table->dropColumn('docker_compose_raw'); + $table->dropColumn('docker_compose_domains'); + }); + } +}; diff --git a/resources/views/components/applications/navbar.blade.php b/resources/views/components/applications/navbar.blade.php index cab687b98..09035d370 100644 --- a/resources/views/components/applications/navbar.blade.php +++ b/resources/views/components/applications/navbar.blade.php @@ -16,7 +16,8 @@ @if ($application->status !== 'exited') - - + @if ($application->build_pack !== 'dockercompose') + + @endif
    -
    - - Generate Domain - -
    - @if (!$application->dockerfile) + @if ($application->build_pack !== 'dockercompose') +
    + + Generate Domain + +
    + @endif + @if (!$application->dockerfile && $application->build_pack !== 'dockerimage')
    - + + {{-- --}} @if ($application->settings->is_static || $application->build_pack === 'static') @@ -47,26 +50,27 @@ @endif
    @endif -

    Docker Registry

    - @if ($application->build_pack !== 'dockerimage') + @if ($application->build_pack !== 'dockerimage' && $application->build_pack !== 'dockercompose') +

    Docker Registry

    Push the built image to a docker registry. More info here.
    - @endif -
    - @if ($application->build_pack === 'dockerimage') - - - @else - - - @endif +
    + @if ($application->build_pack === 'dockerimage') + + + @else + + + @endif -
    +
    + @endif @if ($application->build_pack !== 'dockerimage')

    Build

    @@ -91,7 +95,13 @@ @if ($application->build_pack === 'dockerfile' && !$application->dockerfile) + helper="It is calculated together with the Base Directory:
    {{ Str::start($application->base_directory . $application->dockerfile_location, '/') }}" /> + @endif + @if ($application->build_pack === 'dockercompose') + + @endif @if ($application->build_pack === 'dockerfile') @endif + @if ($application->build_pack === 'dockercompose') + Reload Compose File + @if (count($parsedServices) > 0) + @foreach (data_get($parsedServices, 'services') as $serviceName => $service) + @if (!collect(DATABASE_DOCKER_IMAGES)->contains(data_get($service, 'image'))) + + @endif + @endforeach + @endif + + @endif @if ($application->dockerfile) @endif -

    Network

    -
    - @if ($application->settings->is_static || $application->build_pack === 'static') - - @else - - @endif - -
    - - Reset to Coolify Generated Labels + @if ($application->build_pack !== 'dockercompose') +

    Network

    +
    + @if ($application->settings->is_static || $application->build_pack === 'static') + + @else + + @endif + +
    + + Reset to Coolify Generated Labels + @endif
    diff --git a/resources/views/livewire/project/new/select.blade.php b/resources/views/livewire/project/new/select.blade.php index 60da34cd1..b4818d8c6 100644 --- a/resources/views/livewire/project/new/select.blade.php +++ b/resources/views/livewire/project/new/select.blade.php @@ -67,7 +67,7 @@ Based on a Docker Compose
    - You can deploy complex application easily with Docker Compose. + You can deploy complex application easily with Docker Compose, without Git.
    @@ -77,7 +77,7 @@ Based on an existing Docker Image
    - You can deploy an existing Docker Image form any Registry. + You can deploy an existing Docker Image form any Registry, without Git.
    From 6f886e8b6f2118e8867c09823ae7ff825d708d68 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Fri, 24 Nov 2023 21:03:59 +0100 Subject: [PATCH 18/46] Update Ghost configuration with mail options --- templates/compose/ghost.yaml | 6 ++++++ templates/service-templates.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/templates/compose/ghost.yaml b/templates/compose/ghost.yaml index 593619158..5e79f1f8c 100644 --- a/templates/compose/ghost.yaml +++ b/templates/compose/ghost.yaml @@ -14,6 +14,12 @@ services: - database__connection__user=$SERVICE_USER_MYSQL - database__connection__password=$SERVICE_PASSWORD_MYSQL - database__connection__database=${MYSQL_DATABASE-ghost} + - mail__options__auth__pass=${MAIL_OPTIONS_AUTH_PASS} + - mail__options__auth__user=${MAIL_OPTIONS_AUTH_USER} + - mail__options__secure=${MAIL_OPTIONS_SECURE:-true} + - mail__options__port=${MAIL_OPTIONS_PORT:-465} + - mail__options__service=${MAIL_OPTIONS_SERVICE:-Mailgun} + - mail__options__host=${MAIL_OPTIONS_HOST} depends_on: mysql: condition: service_healthy diff --git a/templates/service-templates.json b/templates/service-templates.json index 3b5b7c4e7..6c6d6d03b 100644 --- a/templates/service-templates.json +++ b/templates/service-templates.json @@ -155,7 +155,7 @@ "ghost": { "documentation": "https:\/\/ghost.org\/docs", "slogan": "Ghost is a popular open-source content management system (CMS) and blogging platform, known for its simplicity and focus on content creation.", - "compose": "c2VydmljZXM6CiAgZ2hvc3Q6CiAgICBpbWFnZTogJ2dob3N0OjUnCiAgICB2b2x1bWVzOgogICAgICAtICdnaG9zdC1jb250ZW50LWRhdGE6L3Zhci9saWIvZ2hvc3QvY29udGVudCcKICAgIGVudmlyb25tZW50OgogICAgICAtIHVybD0kU0VSVklDRV9GUUROX0dIT1NUCiAgICAgIC0gZGF0YWJhc2VfX2NsaWVudD1teXNxbAogICAgICAtIGRhdGFiYXNlX19jb25uZWN0aW9uX19ob3N0PW15c3FsCiAgICAgIC0gZGF0YWJhc2VfX2Nvbm5lY3Rpb25fX3VzZXI9JFNFUlZJQ0VfVVNFUl9NWVNRTAogICAgICAtIGRhdGFiYXNlX19jb25uZWN0aW9uX19wYXNzd29yZD0kU0VSVklDRV9QQVNTV09SRF9NWVNRTAogICAgICAtICdkYXRhYmFzZV9fY29ubmVjdGlvbl9fZGF0YWJhc2U9JHtNWVNRTF9EQVRBQkFTRS1naG9zdH0nCiAgICBkZXBlbmRzX29uOgogICAgICBteXNxbDoKICAgICAgICBjb25kaXRpb246IHNlcnZpY2VfaGVhbHRoeQogIG15c3FsOgogICAgaW1hZ2U6ICdteXNxbDo4LjAnCiAgICB2b2x1bWVzOgogICAgICAtICdnaG9zdC1teXNxbC1kYXRhOi92YXIvbGliL215c3FsJwogICAgZW52aXJvbm1lbnQ6CiAgICAgIC0gJ01ZU1FMX1VTRVI9JHtTRVJWSUNFX1VTRVJfTVlTUUx9JwogICAgICAtICdNWVNRTF9QQVNTV09SRD0ke1NFUlZJQ0VfUEFTU1dPUkRfTVlTUUx9JwogICAgICAtICdNWVNRTF9EQVRBQkFTRT0ke01ZU1FMX0RBVEFCQVNFfScKICAgICAgLSAnTVlTUUxfUk9PVF9QQVNTV09SRD0ke1NFUlZJQ0VfUEFTU1dPUkRfTVlTUUxST09UfScKICAgIGhlYWx0aGNoZWNrOgogICAgICB0ZXN0OgogICAgICAgIC0gQ01ECiAgICAgICAgLSBteXNxbGFkbWluCiAgICAgICAgLSBwaW5nCiAgICAgICAgLSAnLWgnCiAgICAgICAgLSBsb2NhbGhvc3QKICAgICAgaW50ZXJ2YWw6IDVzCiAgICAgIHRpbWVvdXQ6IDIwcwogICAgICByZXRyaWVzOiAxMAo=", + "compose": "c2VydmljZXM6CiAgZ2hvc3Q6CiAgICBpbWFnZTogJ2dob3N0OjUnCiAgICB2b2x1bWVzOgogICAgICAtICdnaG9zdC1jb250ZW50LWRhdGE6L3Zhci9saWIvZ2hvc3QvY29udGVudCcKICAgIGVudmlyb25tZW50OgogICAgICAtIHVybD0kU0VSVklDRV9GUUROX0dIT1NUCiAgICAgIC0gZGF0YWJhc2VfX2NsaWVudD1teXNxbAogICAgICAtIGRhdGFiYXNlX19jb25uZWN0aW9uX19ob3N0PW15c3FsCiAgICAgIC0gZGF0YWJhc2VfX2Nvbm5lY3Rpb25fX3VzZXI9JFNFUlZJQ0VfVVNFUl9NWVNRTAogICAgICAtIGRhdGFiYXNlX19jb25uZWN0aW9uX19wYXNzd29yZD0kU0VSVklDRV9QQVNTV09SRF9NWVNRTAogICAgICAtICdkYXRhYmFzZV9fY29ubmVjdGlvbl9fZGF0YWJhc2U9JHtNWVNRTF9EQVRBQkFTRS1naG9zdH0nCiAgICAgIC0gJ21haWxfX29wdGlvbnNfX2F1dGhfX3Bhc3M9JHtNQUlMX09QVElPTlNfQVVUSF9QQVNTfScKICAgICAgLSAnbWFpbF9fb3B0aW9uc19fYXV0aF9fdXNlcj0ke01BSUxfT1BUSU9OU19BVVRIX1VTRVJ9JwogICAgICAtICdtYWlsX19vcHRpb25zX19zZWN1cmU9JHtNQUlMX09QVElPTlNfU0VDVVJFOi10cnVlfScKICAgICAgLSAnbWFpbF9fb3B0aW9uc19fcG9ydD0ke01BSUxfT1BUSU9OU19QT1JUOi00NjV9JwogICAgICAtICdtYWlsX19vcHRpb25zX19zZXJ2aWNlPSR7TUFJTF9PUFRJT05TX1NFUlZJQ0U6LU1haWxndW59JwogICAgICAtICdtYWlsX19vcHRpb25zX19ob3N0PSR7TUFJTF9PUFRJT05TX0hPU1R9JwogICAgZGVwZW5kc19vbjoKICAgICAgbXlzcWw6CiAgICAgICAgY29uZGl0aW9uOiBzZXJ2aWNlX2hlYWx0aHkKICBteXNxbDoKICAgIGltYWdlOiAnbXlzcWw6OC4wJwogICAgdm9sdW1lczoKICAgICAgLSAnZ2hvc3QtbXlzcWwtZGF0YTovdmFyL2xpYi9teXNxbCcKICAgIGVudmlyb25tZW50OgogICAgICAtICdNWVNRTF9VU0VSPSR7U0VSVklDRV9VU0VSX01ZU1FMfScKICAgICAgLSAnTVlTUUxfUEFTU1dPUkQ9JHtTRVJWSUNFX1BBU1NXT1JEX01ZU1FMfScKICAgICAgLSAnTVlTUUxfREFUQUJBU0U9JHtNWVNRTF9EQVRBQkFTRX0nCiAgICAgIC0gJ01ZU1FMX1JPT1RfUEFTU1dPUkQ9JHtTRVJWSUNFX1BBU1NXT1JEX01ZU1FMUk9PVH0nCiAgICBoZWFsdGhjaGVjazoKICAgICAgdGVzdDoKICAgICAgICAtIENNRAogICAgICAgIC0gbXlzcWxhZG1pbgogICAgICAgIC0gcGluZwogICAgICAgIC0gJy1oJwogICAgICAgIC0gbG9jYWxob3N0CiAgICAgIGludGVydmFsOiA1cwogICAgICB0aW1lb3V0OiAyMHMKICAgICAgcmV0cmllczogMTAK", "tags": [ "cms", "blog", From 10e5a58b9e2c03067dd2c8973804791a6366a083 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Fri, 24 Nov 2023 21:04:15 +0100 Subject: [PATCH 19/46] Add extra fields for MinIO, Weblate, and Ghost services --- .../Livewire/Project/Service/StackForm.php | 1 + app/Models/Service.php | 65 ++++++++++++++++++- bootstrap/helpers/docker.php | 1 + .../project/service/stack-form.blade.php | 3 +- 4 files changed, 67 insertions(+), 3 deletions(-) diff --git a/app/Http/Livewire/Project/Service/StackForm.php b/app/Http/Livewire/Project/Service/StackForm.php index ebdb2d481..44caa29ff 100644 --- a/app/Http/Livewire/Project/Service/StackForm.php +++ b/app/Http/Livewire/Project/Service/StackForm.php @@ -31,6 +31,7 @@ class StackForm extends Component "name" => $fieldKey, "value" => $value, "isPassword" => $isPassword, + "rules" => $rules ]; $this->rules["fields.$key.value"] = $rules; $this->validationAttributes["fields.$key.value"] = $fieldKey; diff --git a/app/Models/Service.php b/app/Models/Service.php index 2d3e17e98..a05c989b8 100644 --- a/app/Models/Service.php +++ b/app/Models/Service.php @@ -52,7 +52,7 @@ class Service extends BaseModel foreach ($applications as $application) { $image = str($application->image)->before(':')->value(); switch ($image) { - case str($image)->contains('minio'): + case str($image)?->contains('minio'): $data = collect([]); $console_url = $this->environment_variables()->where('key', 'MINIO_BROWSER_REDIRECT_URL')->first(); $s3_api_url = $this->environment_variables()->where('key', 'MINIO_SERVER_URL')->first(); @@ -105,7 +105,7 @@ class Service extends BaseModel $fields->put('MinIO', $data->toArray()); break; - case str($image)->contains('weblate'): + case str($image)?->contains('weblate'): $data = collect([]); $admin_email = $this->environment_variables()->where('key', 'WEBLATE_ADMIN_EMAIL')->first(); $admin_password = $this->environment_variables()->where('key', 'SERVICE_PASSWORD_WEBLATE')->first(); @@ -130,6 +130,67 @@ class Service extends BaseModel ]); } $fields->put('Weblate', $data); + break; + case str($image)?->contains('ghost'): + $data = collect([]); + $MAIL_OPTIONS_AUTH_PASS = $this->environment_variables()->where('key', 'MAIL_OPTIONS_AUTH_PASS')->first(); + $MAIL_OPTIONS_AUTH_USER = $this->environment_variables()->where('key', 'MAIL_OPTIONS_AUTH_USER')->first(); + $MAIL_OPTIONS_SECURE = $this->environment_variables()->where('key', 'MAIL_OPTIONS_SECURE')->first(); + $MAIL_OPTIONS_PORT = $this->environment_variables()->where('key', 'MAIL_OPTIONS_PORT')->first(); + $MAIL_OPTIONS_SERVICE = $this->environment_variables()->where('key', 'MAIL_OPTIONS_SERVICE')->first(); + $MAIL_OPTIONS_HOST = $this->environment_variables()->where('key', 'MAIL_OPTIONS_HOST')->first(); + if ($MAIL_OPTIONS_AUTH_PASS) { + $data = $data->merge([ + 'Mail Password' => [ + 'key' => data_get($MAIL_OPTIONS_AUTH_PASS, 'key'), + 'value' => data_get($MAIL_OPTIONS_AUTH_PASS, 'value'), + 'isPassword' => true, + ], + ]); + } + if ($MAIL_OPTIONS_AUTH_USER) { + $data = $data->merge([ + 'Mail User' => [ + 'key' => data_get($MAIL_OPTIONS_AUTH_USER, 'key'), + 'value' => data_get($MAIL_OPTIONS_AUTH_USER, 'value'), + ], + ]); + } + if ($MAIL_OPTIONS_SECURE) { + $data = $data->merge([ + 'Mail Secure' => [ + 'key' => data_get($MAIL_OPTIONS_SECURE, 'key'), + 'value' => data_get($MAIL_OPTIONS_SECURE, 'value'), + ], + ]); + } + if ($MAIL_OPTIONS_PORT) { + $data = $data->merge([ + 'Mail Port' => [ + 'key' => data_get($MAIL_OPTIONS_PORT, 'key'), + 'value' => data_get($MAIL_OPTIONS_PORT, 'value'), + ], + ]); + } + if ($MAIL_OPTIONS_SERVICE) { + $data = $data->merge([ + 'Mail Service' => [ + 'key' => data_get($MAIL_OPTIONS_SERVICE, 'key'), + 'value' => data_get($MAIL_OPTIONS_SERVICE, 'value'), + ], + ]); + } + if ($MAIL_OPTIONS_HOST) { + $data = $data->merge([ + 'Mail Host' => [ + 'key' => data_get($MAIL_OPTIONS_HOST, 'key'), + 'value' => data_get($MAIL_OPTIONS_HOST, 'value'), + ], + ]); + } + + $fields->put('Ghost', $data); + break; } } $databases = $this->databases()->get(); diff --git a/bootstrap/helpers/docker.php b/bootstrap/helpers/docker.php index fef40aed8..d6ef6f1ce 100644 --- a/bootstrap/helpers/docker.php +++ b/bootstrap/helpers/docker.php @@ -188,6 +188,7 @@ function generateServiceSpecificFqdns(ServiceApplication|Application $resource, $MINIO_SERVER_URL->value, ]); } + break; } return $payload; } diff --git a/resources/views/livewire/project/service/stack-form.blade.php b/resources/views/livewire/project/service/stack-form.blade.php index c8a853b37..65ead1777 100644 --- a/resources/views/livewire/project/service/stack-form.blade.php +++ b/resources/views/livewire/project/service/stack-form.blade.php @@ -18,7 +18,8 @@
    @foreach ($fields as $serviceName => $field) - From 2cbe1e8489f41c246a7977745c332b6ec482145f Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Fri, 24 Nov 2023 21:23:48 +0100 Subject: [PATCH 20/46] Add SMTP mail transport option to Ghost compose file --- templates/compose/ghost.yaml | 1 + templates/service-templates.json | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/templates/compose/ghost.yaml b/templates/compose/ghost.yaml index 5e79f1f8c..d0d9c27a5 100644 --- a/templates/compose/ghost.yaml +++ b/templates/compose/ghost.yaml @@ -14,6 +14,7 @@ services: - database__connection__user=$SERVICE_USER_MYSQL - database__connection__password=$SERVICE_PASSWORD_MYSQL - database__connection__database=${MYSQL_DATABASE-ghost} + - mail__transport=SMTP - mail__options__auth__pass=${MAIL_OPTIONS_AUTH_PASS} - mail__options__auth__user=${MAIL_OPTIONS_AUTH_USER} - mail__options__secure=${MAIL_OPTIONS_SECURE:-true} diff --git a/templates/service-templates.json b/templates/service-templates.json index 6c6d6d03b..c48488a90 100644 --- a/templates/service-templates.json +++ b/templates/service-templates.json @@ -155,7 +155,7 @@ "ghost": { "documentation": "https:\/\/ghost.org\/docs", "slogan": "Ghost is a popular open-source content management system (CMS) and blogging platform, known for its simplicity and focus on content creation.", - "compose": "c2VydmljZXM6CiAgZ2hvc3Q6CiAgICBpbWFnZTogJ2dob3N0OjUnCiAgICB2b2x1bWVzOgogICAgICAtICdnaG9zdC1jb250ZW50LWRhdGE6L3Zhci9saWIvZ2hvc3QvY29udGVudCcKICAgIGVudmlyb25tZW50OgogICAgICAtIHVybD0kU0VSVklDRV9GUUROX0dIT1NUCiAgICAgIC0gZGF0YWJhc2VfX2NsaWVudD1teXNxbAogICAgICAtIGRhdGFiYXNlX19jb25uZWN0aW9uX19ob3N0PW15c3FsCiAgICAgIC0gZGF0YWJhc2VfX2Nvbm5lY3Rpb25fX3VzZXI9JFNFUlZJQ0VfVVNFUl9NWVNRTAogICAgICAtIGRhdGFiYXNlX19jb25uZWN0aW9uX19wYXNzd29yZD0kU0VSVklDRV9QQVNTV09SRF9NWVNRTAogICAgICAtICdkYXRhYmFzZV9fY29ubmVjdGlvbl9fZGF0YWJhc2U9JHtNWVNRTF9EQVRBQkFTRS1naG9zdH0nCiAgICAgIC0gJ21haWxfX29wdGlvbnNfX2F1dGhfX3Bhc3M9JHtNQUlMX09QVElPTlNfQVVUSF9QQVNTfScKICAgICAgLSAnbWFpbF9fb3B0aW9uc19fYXV0aF9fdXNlcj0ke01BSUxfT1BUSU9OU19BVVRIX1VTRVJ9JwogICAgICAtICdtYWlsX19vcHRpb25zX19zZWN1cmU9JHtNQUlMX09QVElPTlNfU0VDVVJFOi10cnVlfScKICAgICAgLSAnbWFpbF9fb3B0aW9uc19fcG9ydD0ke01BSUxfT1BUSU9OU19QT1JUOi00NjV9JwogICAgICAtICdtYWlsX19vcHRpb25zX19zZXJ2aWNlPSR7TUFJTF9PUFRJT05TX1NFUlZJQ0U6LU1haWxndW59JwogICAgICAtICdtYWlsX19vcHRpb25zX19ob3N0PSR7TUFJTF9PUFRJT05TX0hPU1R9JwogICAgZGVwZW5kc19vbjoKICAgICAgbXlzcWw6CiAgICAgICAgY29uZGl0aW9uOiBzZXJ2aWNlX2hlYWx0aHkKICBteXNxbDoKICAgIGltYWdlOiAnbXlzcWw6OC4wJwogICAgdm9sdW1lczoKICAgICAgLSAnZ2hvc3QtbXlzcWwtZGF0YTovdmFyL2xpYi9teXNxbCcKICAgIGVudmlyb25tZW50OgogICAgICAtICdNWVNRTF9VU0VSPSR7U0VSVklDRV9VU0VSX01ZU1FMfScKICAgICAgLSAnTVlTUUxfUEFTU1dPUkQ9JHtTRVJWSUNFX1BBU1NXT1JEX01ZU1FMfScKICAgICAgLSAnTVlTUUxfREFUQUJBU0U9JHtNWVNRTF9EQVRBQkFTRX0nCiAgICAgIC0gJ01ZU1FMX1JPT1RfUEFTU1dPUkQ9JHtTRVJWSUNFX1BBU1NXT1JEX01ZU1FMUk9PVH0nCiAgICBoZWFsdGhjaGVjazoKICAgICAgdGVzdDoKICAgICAgICAtIENNRAogICAgICAgIC0gbXlzcWxhZG1pbgogICAgICAgIC0gcGluZwogICAgICAgIC0gJy1oJwogICAgICAgIC0gbG9jYWxob3N0CiAgICAgIGludGVydmFsOiA1cwogICAgICB0aW1lb3V0OiAyMHMKICAgICAgcmV0cmllczogMTAK", + "compose": "c2VydmljZXM6CiAgZ2hvc3Q6CiAgICBpbWFnZTogJ2dob3N0OjUnCiAgICB2b2x1bWVzOgogICAgICAtICdnaG9zdC1jb250ZW50LWRhdGE6L3Zhci9saWIvZ2hvc3QvY29udGVudCcKICAgIGVudmlyb25tZW50OgogICAgICAtIHVybD0kU0VSVklDRV9GUUROX0dIT1NUCiAgICAgIC0gZGF0YWJhc2VfX2NsaWVudD1teXNxbAogICAgICAtIGRhdGFiYXNlX19jb25uZWN0aW9uX19ob3N0PW15c3FsCiAgICAgIC0gZGF0YWJhc2VfX2Nvbm5lY3Rpb25fX3VzZXI9JFNFUlZJQ0VfVVNFUl9NWVNRTAogICAgICAtIGRhdGFiYXNlX19jb25uZWN0aW9uX19wYXNzd29yZD0kU0VSVklDRV9QQVNTV09SRF9NWVNRTAogICAgICAtICdkYXRhYmFzZV9fY29ubmVjdGlvbl9fZGF0YWJhc2U9JHtNWVNRTF9EQVRBQkFTRS1naG9zdH0nCiAgICAgIC0gbWFpbF9fdHJhbnNwb3J0PVNNVFAKICAgICAgLSAnbWFpbF9fb3B0aW9uc19fYXV0aF9fcGFzcz0ke01BSUxfT1BUSU9OU19BVVRIX1BBU1N9JwogICAgICAtICdtYWlsX19vcHRpb25zX19hdXRoX191c2VyPSR7TUFJTF9PUFRJT05TX0FVVEhfVVNFUn0nCiAgICAgIC0gJ21haWxfX29wdGlvbnNfX3NlY3VyZT0ke01BSUxfT1BUSU9OU19TRUNVUkU6LXRydWV9JwogICAgICAtICdtYWlsX19vcHRpb25zX19wb3J0PSR7TUFJTF9PUFRJT05TX1BPUlQ6LTQ2NX0nCiAgICAgIC0gJ21haWxfX29wdGlvbnNfX3NlcnZpY2U9JHtNQUlMX09QVElPTlNfU0VSVklDRTotTWFpbGd1bn0nCiAgICAgIC0gJ21haWxfX29wdGlvbnNfX2hvc3Q9JHtNQUlMX09QVElPTlNfSE9TVH0nCiAgICBkZXBlbmRzX29uOgogICAgICBteXNxbDoKICAgICAgICBjb25kaXRpb246IHNlcnZpY2VfaGVhbHRoeQogIG15c3FsOgogICAgaW1hZ2U6ICdteXNxbDo4LjAnCiAgICB2b2x1bWVzOgogICAgICAtICdnaG9zdC1teXNxbC1kYXRhOi92YXIvbGliL215c3FsJwogICAgZW52aXJvbm1lbnQ6CiAgICAgIC0gJ01ZU1FMX1VTRVI9JHtTRVJWSUNFX1VTRVJfTVlTUUx9JwogICAgICAtICdNWVNRTF9QQVNTV09SRD0ke1NFUlZJQ0VfUEFTU1dPUkRfTVlTUUx9JwogICAgICAtICdNWVNRTF9EQVRBQkFTRT0ke01ZU1FMX0RBVEFCQVNFfScKICAgICAgLSAnTVlTUUxfUk9PVF9QQVNTV09SRD0ke1NFUlZJQ0VfUEFTU1dPUkRfTVlTUUxST09UfScKICAgIGhlYWx0aGNoZWNrOgogICAgICB0ZXN0OgogICAgICAgIC0gQ01ECiAgICAgICAgLSBteXNxbGFkbWluCiAgICAgICAgLSBwaW5nCiAgICAgICAgLSAnLWgnCiAgICAgICAgLSBsb2NhbGhvc3QKICAgICAgaW50ZXJ2YWw6IDVzCiAgICAgIHRpbWVvdXQ6IDIwcwogICAgICByZXRyaWVzOiAxMAo=", "tags": [ "cms", "blog", From 58111f53b9d31688057a3a0eeb9fd0f6b189ee8f Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Fri, 24 Nov 2023 21:35:01 +0100 Subject: [PATCH 21/46] test wire:ignore --- .../views/livewire/project/service/compose-modal.blade.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/views/livewire/project/service/compose-modal.blade.php b/resources/views/livewire/project/service/compose-modal.blade.php index d3c16dfd4..77365cc0c 100644 --- a/resources/views/livewire/project/service/compose-modal.blade.php +++ b/resources/views/livewire/project/service/compose-modal.blade.php @@ -1,4 +1,4 @@ - +

    Docker Compose

    From ec21155c9ede67db4a43399f9c1a9a2b8fc2dd8e Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Fri, 24 Nov 2023 21:38:39 +0100 Subject: [PATCH 22/46] Update rules for field validation in StackForm.php --- app/Http/Livewire/Project/Service/StackForm.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Http/Livewire/Project/Service/StackForm.php b/app/Http/Livewire/Project/Service/StackForm.php index 44caa29ff..1014756ce 100644 --- a/app/Http/Livewire/Project/Service/StackForm.php +++ b/app/Http/Livewire/Project/Service/StackForm.php @@ -23,7 +23,7 @@ class StackForm extends Component foreach ($fields as $fieldKey => $field) { $key = data_get($field, 'key'); $value = data_get($field, 'value'); - $rules = data_get($field, 'rules'); + $rules = data_get($field, 'rules', 'nullable'); $isPassword = data_get($field, 'isPassword'); $this->fields[$key] = [ "serviceName" => $serviceName, From d396f649dfc607ff245135e3cf0e8ddb0be352d5 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Mon, 27 Nov 2023 09:39:43 +0100 Subject: [PATCH 23/46] fix: show defined resources in server tab, so you will know what you need to delete before you can delete the server. --- app/Http/Controllers/ProjectController.php | 2 +- app/Http/Livewire/Project/New/DockerCompose.php | 2 +- .../Livewire/Project/Service/Application.php | 2 +- app/Models/Application.php | 9 ++++++++- app/Models/Server.php | 6 ++++++ app/Models/Service.php | 8 ++++++++ app/Models/StandaloneMariadb.php | 8 ++++++++ app/Models/StandaloneMongodb.php | 8 ++++++++ app/Models/StandaloneMysql.php | 8 ++++++++ app/Models/StandalonePostgresql.php | 9 ++++++++- app/Models/StandaloneRedis.php | 9 ++++++++- .../views/components/services/navbar.blade.php | 4 ++-- .../livewire/project/service/show.blade.php | 4 ++-- resources/views/livewire/server/delete.blade.php | 16 ++++++++++++++-- resources/views/project/resources.blade.php | 2 +- routes/web.php | 2 +- 16 files changed, 85 insertions(+), 14 deletions(-) diff --git a/app/Http/Controllers/ProjectController.php b/app/Http/Controllers/ProjectController.php index 73cf55fcd..58d0dbb6a 100644 --- a/app/Http/Controllers/ProjectController.php +++ b/app/Http/Controllers/ProjectController.php @@ -137,7 +137,7 @@ class ProjectController extends Controller } $service->parse(isNew: true); - return redirect()->route('project.service', [ + return redirect()->route('project.service.configuration', [ 'service_uuid' => $service->uuid, 'environment_name' => $environment->name, 'project_uuid' => $project->uuid, diff --git a/app/Http/Livewire/Project/New/DockerCompose.php b/app/Http/Livewire/Project/New/DockerCompose.php index 81c487ccf..cd0d86c08 100644 --- a/app/Http/Livewire/Project/New/DockerCompose.php +++ b/app/Http/Livewire/Project/New/DockerCompose.php @@ -129,7 +129,7 @@ class DockerCompose extends Component $service->parse(isNew: true); - return redirect()->route('project.service', [ + return redirect()->route('project.service.configuration', [ 'service_uuid' => $service->uuid, 'environment_name' => $environment->name, 'project_uuid' => $project->uuid, diff --git a/app/Http/Livewire/Project/Service/Application.php b/app/Http/Livewire/Project/Service/Application.php index 14fd3d5ab..622b1e0e2 100644 --- a/app/Http/Livewire/Project/Service/Application.php +++ b/app/Http/Livewire/Project/Service/Application.php @@ -41,7 +41,7 @@ class Application extends Component try { $this->application->delete(); $this->emit('success', 'Application deleted successfully.'); - return redirect()->route('project.service', $this->parameters); + return redirect()->route('project.service.configuration', $this->parameters); } catch (\Throwable $e) { return handleError($e, $this); } diff --git a/app/Models/Application.php b/app/Models/Application.php index ee7ccf60b..20f61ed39 100644 --- a/app/Models/Application.php +++ b/app/Models/Application.php @@ -47,7 +47,14 @@ class Application extends BaseModel $application->environment_variables_preview()->delete(); }); } - + public function link() + { + return route('project.application.configuration', [ + 'project_uuid' => $this->environment->project->uuid, + 'environment_name' => $this->environment->name, + 'application_uuid' => $this->uuid + ]); + } public function settings() { return $this->hasOne(ApplicationSetting::class); diff --git a/app/Models/Server.php b/app/Models/Server.php index 8df4c1e3f..5aa571452 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -194,6 +194,12 @@ class Server extends BaseModel { return instant_remote_process(["df /| tail -1 | awk '{ print $5}' | sed 's/%//g'"], $this, false); } + public function definedResources() { + $applications = $this->applications(); + $databases = $this->databases(); + $services = $this->services(); + return $applications->concat($databases)->concat($services->get()); + } public function hasDefinedResources() { $applications = $this->applications()->count() > 0; diff --git a/app/Models/Service.php b/app/Models/Service.php index a05c989b8..345e4db92 100644 --- a/app/Models/Service.php +++ b/app/Models/Service.php @@ -361,6 +361,14 @@ class Service extends BaseModel } } } + public function link() + { + return route('project.service.configuration', [ + 'project_uuid' => $this->environment->project->uuid, + 'environment_name' => $this->environment->name, + 'service_uuid' => $this->uuid + ]); + } public function documentation() { $services = getServiceTemplates(); diff --git a/app/Models/StandaloneMariadb.php b/app/Models/StandaloneMariadb.php index cb9fa9aa2..ca7221d7c 100644 --- a/app/Models/StandaloneMariadb.php +++ b/app/Models/StandaloneMariadb.php @@ -41,6 +41,14 @@ class StandaloneMariadb extends BaseModel $database->environment_variables()->delete(); }); } + public function link() + { + return route('project.database.configuration', [ + 'project_uuid' => $this->environment->project->uuid, + 'environment_name' => $this->environment->name, + 'database_uuid' => $this->uuid + ]); + } public function isLogDrainEnabled() { return data_get($this, 'is_log_drain_enabled', false); diff --git a/app/Models/StandaloneMongodb.php b/app/Models/StandaloneMongodb.php index 3a5b7a52a..598719acf 100644 --- a/app/Models/StandaloneMongodb.php +++ b/app/Models/StandaloneMongodb.php @@ -48,6 +48,14 @@ class StandaloneMongodb extends BaseModel { return data_get($this, 'is_log_drain_enabled', false); } + public function link() + { + return route('project.database.configuration', [ + 'project_uuid' => $this->environment->project->uuid, + 'environment_name' => $this->environment->name, + 'database_uuid' => $this->uuid + ]); + } public function mongoInitdbRootPassword(): Attribute { return Attribute::make( diff --git a/app/Models/StandaloneMysql.php b/app/Models/StandaloneMysql.php index 6de57cb73..4cbee006e 100644 --- a/app/Models/StandaloneMysql.php +++ b/app/Models/StandaloneMysql.php @@ -41,6 +41,14 @@ class StandaloneMysql extends BaseModel $database->environment_variables()->delete(); }); } + public function link() + { + return route('project.database.configuration', [ + 'project_uuid' => $this->environment->project->uuid, + 'environment_name' => $this->environment->name, + 'database_uuid' => $this->uuid + ]); + } public function type(): string { return 'standalone-mysql'; diff --git a/app/Models/StandalonePostgresql.php b/app/Models/StandalonePostgresql.php index 1bef3b6c9..0056f9b51 100644 --- a/app/Models/StandalonePostgresql.php +++ b/app/Models/StandalonePostgresql.php @@ -41,7 +41,14 @@ class StandalonePostgresql extends BaseModel $database->environment_variables()->delete(); }); } - + public function link() + { + return route('project.database.configuration', [ + 'project_uuid' => $this->environment->project->uuid, + 'environment_name' => $this->environment->name, + 'database_uuid' => $this->uuid + ]); + } public function isLogDrainEnabled() { return data_get($this, 'is_log_drain_enabled', false); diff --git a/app/Models/StandaloneRedis.php b/app/Models/StandaloneRedis.php index fe1281c22..b2038d8af 100644 --- a/app/Models/StandaloneRedis.php +++ b/app/Models/StandaloneRedis.php @@ -36,7 +36,14 @@ class StandaloneRedis extends BaseModel $database->environment_variables()->delete(); }); } - + public function link() + { + return route('project.database.configuration', [ + 'project_uuid' => $this->environment->project->uuid, + 'environment_name' => $this->environment->name, + 'database_uuid' => $this->uuid + ]); + } public function isLogDrainEnabled() { return data_get($this, 'is_log_drain_enabled', false); diff --git a/resources/views/components/services/navbar.blade.php b/resources/views/components/services/navbar.blade.php index 6c9af5f07..83facccec 100644 --- a/resources/views/components/services/navbar.blade.php +++ b/resources/views/components/services/navbar.blade.php @@ -1,6 +1,6 @@
    - + diff --git a/resources/views/livewire/project/service/show.blade.php b/resources/views/livewire/project/service/show.blade.php index 427904595..f556d6df8 100644 --- a/resources/views/livewire/project/service/show.blade.php +++ b/resources/views/livewire/project/service/show.blade.php @@ -2,8 +2,8 @@
    - + This will remove this server from Coolify. Beware! There is no coming back!
    - @if ($server->hasDefinedResources()) -
    Please delete all resources before deleting this server.
    + @if ($server->definedResources()->count() > 0) + + Delete + @else Delete @endif + @forelse ($server->definedResources() as $resource) + @if ($loop->first) +
    Please delete all resources before deleting this server.
    + @endif +
    +
    {{ str($resource->type())->headline() }}
    +     {{ $resource->name }} +
    + @empty + @endforelse @endif
    diff --git a/resources/views/project/resources.blade.php b/resources/views/project/resources.blade.php index a81982ec7..8e97bf324 100644 --- a/resources/views/project/resources.blade.php +++ b/resources/views/project/resources.blade.php @@ -79,7 +79,7 @@ @endforeach @foreach ($environment->services->sortBy('name') as $service) + href="{{ route('project.service.configuration', [$project->uuid, $environment->name, $service->uuid]) }}">
    {{ $service->name }}
    {{ $service->description }}
    diff --git a/routes/web.php b/routes/web.php index 18bad3bf5..1e276e738 100644 --- a/routes/web.php +++ b/routes/web.php @@ -120,7 +120,7 @@ Route::middleware(['auth', 'verified'])->group(function () { // Services - Route::get('/project/{project_uuid}/{environment_name}/service/{service_uuid}', ServiceIndex::class)->name('project.service'); + Route::get('/project/{project_uuid}/{environment_name}/service/{service_uuid}', ServiceIndex::class)->name('project.service.configuration'); Route::get('/project/{project_uuid}/{environment_name}/service/{service_uuid}/{service_name}', ServiceShow::class)->name('project.service.show'); Route::get('/project/{project_uuid}/{environment_name}/service/{service_uuid}/{service_name}/logs', Logs::class)->name('project.service.logs'); }); From 8d0c3abf2e6b4b2d0a8b5b3273d3bd3a29d1c509 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Mon, 27 Nov 2023 09:42:23 +0100 Subject: [PATCH 24/46] Refactor server delete view to display defined resources --- resources/views/livewire/server/delete.blade.php | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/resources/views/livewire/server/delete.blade.php b/resources/views/livewire/server/delete.blade.php index bc33f0233..b568306ad 100644 --- a/resources/views/livewire/server/delete.blade.php +++ b/resources/views/livewire/server/delete.blade.php @@ -22,7 +22,19 @@ @endif @forelse ($server->definedResources() as $resource) @if ($loop->first) -
    Please delete all resources before deleting this server.
    +

    Defined resources

    +
    Please delete all resources before deleting this server.
    + @endif +
    +
    {{ str($resource->type())->headline() }}
    +     {{ $resource->name }} +
    + @empty + @endforelse + @else + @forelse ($server->definedResources() as $resource) + @if ($loop->first) +

    Defined resources

    @endif
    {{ str($resource->type())->headline() }}
    From fae97e4dee198d2bfd86c82290f397e51b198027 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Mon, 27 Nov 2023 09:58:31 +0100 Subject: [PATCH 25/46] Fix network connection issues in Server and Service models --- app/Http/Livewire/Server/Show.php | 1 + app/Models/Service.php | 6 +++ bootstrap/helpers/proxy.php | 10 +++- bootstrap/helpers/shared.php | 85 ++++++++++++++++++++++++++++++- 4 files changed, 99 insertions(+), 3 deletions(-) diff --git a/app/Http/Livewire/Server/Show.php b/app/Http/Livewire/Server/Show.php index 3863381b2..a72abdf94 100644 --- a/app/Http/Livewire/Server/Show.php +++ b/app/Http/Livewire/Server/Show.php @@ -19,6 +19,7 @@ class Show extends Component if (is_null($this->server)) { return redirect()->route('server.all'); } + } catch (\Throwable $e) { return handleError($e, $this); } diff --git a/app/Models/Service.php b/app/Models/Service.php index 345e4db92..3a02ed560 100644 --- a/app/Models/Service.php +++ b/app/Models/Service.php @@ -442,4 +442,10 @@ class Service extends BaseModel { return parseDockerComposeFile($this, $isNew); } + public function networks() + { + $networks = getTopLevelNetworks($this); + // ray($networks); + return $networks; + } } diff --git a/bootstrap/helpers/proxy.php b/bootstrap/helpers/proxy.php index 595d6c5ad..86946231b 100644 --- a/bootstrap/helpers/proxy.php +++ b/bootstrap/helpers/proxy.php @@ -12,10 +12,16 @@ function get_proxy_path() } function connectProxyToNetworks(Server $server) { - // TODO: Connect to service + compose based application networks as well. + // TODO: Connect to compose based application networks as well. + // Standalone networks $networks = collect($server->standaloneDockers)->map(function ($docker) { return $docker['network']; - })->unique(); + }); + // Service networks + foreach($server->services()->get() as $service) { + $networks->push($service->networks()); + } + $networks = collect($networks)->flatten()->unique(); if ($networks->count() === 0) { $networks = collect(['coolify']); } diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php index cff60233a..4f47d2332 100644 --- a/bootstrap/helpers/shared.php +++ b/bootstrap/helpers/shared.php @@ -495,6 +495,89 @@ function removeAnsiColors($text) return preg_replace('/\e[[][A-Za-z0-9];?[0-9]*m?/', '', $text); } +function getTopLevelNetworks(Service|Application $resource) +{ + if ($resource->getMorphClass() === 'App\Models\Service') { + if ($resource->docker_compose_raw) { + try { + $yaml = Yaml::parse($resource->docker_compose_raw); + } catch (\Exception $e) { + throw new \Exception($e->getMessage()); + } + $services = data_get($yaml, 'services'); + $topLevelNetworks = collect(data_get($yaml, 'networks', [])); + $definedNetwork = collect([$resource->uuid]); + $services = collect($services)->map(function ($service, $_) use ($topLevelNetworks, $definedNetwork) { + $serviceNetworks = collect(data_get($service, 'networks', [])); + + // Collect/create/update networks + if ($serviceNetworks->count() > 0) { + foreach ($serviceNetworks as $networkName => $networkDetails) { + $networkExists = $topLevelNetworks->contains(function ($value, $key) use ($networkName) { + return $value == $networkName || $key == $networkName; + }); + if (!$networkExists) { + $topLevelNetworks->put($networkDetails, null); + } + } + } + + $definedNetworkExists = $topLevelNetworks->contains(function ($value, $_) use ($definedNetwork) { + return $value == $definedNetwork; + }); + if (!$definedNetworkExists) { + foreach ($definedNetwork as $network) { + $topLevelNetworks->put($network, [ + 'name' => $network, + 'external' => true + ]); + } + } + + return $service; + }); + return $topLevelNetworks->keys(); + } + } else if ($resource->getMorphClass() === 'App\Models\Application') { + try { + $yaml = Yaml::parse($resource->docker_compose_raw); + } catch (\Exception $e) { + throw new \Exception($e->getMessage()); + } + $server = $resource->destination->server; + $topLevelNetworks = collect(data_get($yaml, 'networks', [])); + $services = data_get($yaml, 'services'); + $definedNetwork = collect([$resource->uuid]); + $services = collect($services)->map(function ($service, $_) use ($topLevelNetworks, $definedNetwork) { + $serviceNetworks = collect(data_get($service, 'networks', [])); + + // Collect/create/update networks + if ($serviceNetworks->count() > 0) { + foreach ($serviceNetworks as $networkName => $networkDetails) { + $networkExists = $topLevelNetworks->contains(function ($value, $key) use ($networkName) { + return $value == $networkName || $key == $networkName; + }); + if (!$networkExists) { + $topLevelNetworks->put($networkDetails, null); + } + } + } + $definedNetworkExists = $topLevelNetworks->contains(function ($value, $_) use ($definedNetwork) { + return $value == $definedNetwork; + }); + if (!$definedNetworkExists) { + foreach ($definedNetwork as $network) { + $topLevelNetworks->put($network, [ + 'name' => $network, + 'external' => true + ]); + } + } + return $service; + }); + return $topLevelNetworks->keys(); + } +} function parseDockerComposeFile(Service|Application $resource, bool $isNew = false) { ray()->clearAll(); @@ -1010,7 +1093,7 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal $resource->docker_compose = Yaml::dump($finalServices, 10, 2); $resource->save(); $resource->saveComposeConfigs(); - return collect([]); + return $finalServices; } else { return collect([]); } From 8d86d53292359efe98b30adff8ecfa5806e3021f Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Mon, 27 Nov 2023 11:54:55 +0100 Subject: [PATCH 26/46] fix: new logging for deployment jobs fix: git based docker compose files --- app/Console/Commands/ServicesGenerate.php | 2 +- .../Livewire/Project/Application/General.php | 55 ++++++------ .../Livewire/Project/Application/Heading.php | 7 +- .../Shared/EnvironmentVariable/Show.php | 1 + app/Http/Livewire/Project/Shared/Logs.php | 8 +- app/Jobs/ApplicationDeployDockerImageJob.php | 2 +- app/Jobs/ApplicationDeploymentJob.php | 29 ++---- app/Jobs/MultipleApplicationDeploymentJob.php | 1 - app/Models/Application.php | 43 +++++++++ app/Models/ApplicationDeploymentQueue.php | 9 +- app/Traits/ExecuteRemoteCommand.php | 18 ++-- bootstrap/helpers/docker.php | 15 ++++ bootstrap/helpers/remoteProcess.php | 2 + bootstrap/helpers/shared.php | 36 ++------ .../components/applications/links.blade.php | 53 +++++++---- .../components/applications/navbar.blade.php | 90 ++++++++++--------- .../application/configuration.blade.php | 14 +-- .../application/deployment-navbar.blade.php | 2 +- .../project/application/general.blade.php | 15 +++- .../project/shared/get-logs.blade.php | 4 +- .../livewire/project/shared/logs.blade.php | 9 +- 21 files changed, 247 insertions(+), 168 deletions(-) diff --git a/app/Console/Commands/ServicesGenerate.php b/app/Console/Commands/ServicesGenerate.php index 530dd259a..802b3180f 100644 --- a/app/Console/Commands/ServicesGenerate.php +++ b/app/Console/Commands/ServicesGenerate.php @@ -26,7 +26,7 @@ class ServicesGenerate extends Command */ public function handle() { - ray()->clearAll(); + // ray()->clearAll(); $files = array_diff(scandir(base_path('templates/compose')), ['.', '..']); $files = array_filter($files, function ($file) { return strpos($file, '.yaml') !== false; diff --git a/app/Http/Livewire/Project/Application/General.php b/app/Http/Livewire/Project/Application/General.php index a90fadb53..d108bca14 100644 --- a/app/Http/Livewire/Project/Application/General.php +++ b/app/Http/Livewire/Project/Application/General.php @@ -26,6 +26,8 @@ class General extends Component public bool $labelsChanged = false; public bool $isConfigurationChanged = false; + public ?string $initialDockerComposeLocation = null; + public bool $is_static; public $parsedServices = []; @@ -109,6 +111,7 @@ class General extends Component } else { $this->customLabels = str($this->application->custom_labels)->replace(',', "\n"); } + $this->initialDockerComposeLocation = $this->application->docker_compose_location; $this->checkLabelUpdates(); } public function instantSave() @@ -118,35 +121,30 @@ class General extends Component } public function loadComposeFile($isInit = false) { - if ($isInit && $this->application->docker_compose_raw) { - return; - } - $uuid = new Cuid2(); - ['commands' => $cloneCommand] = $this->application->generateGitImportCommands(deployment_uuid: $uuid, only_checkout: true, exec_in_docker: false, custom_base_dir: '.'); - $workdir = rtrim($this->application->base_directory, '/'); - $composeFile = $this->application->docker_compose_location; - $commands = collect([ - "mkdir -p /tmp/{$uuid} && cd /tmp/{$uuid}", - $cloneCommand, - "git sparse-checkout init --cone", - "git sparse-checkout set .$workdir$composeFile", - "git read-tree -mu HEAD", - "cat .$workdir$composeFile", - ]); - $composeFileContent = instant_remote_process($commands, $this->application->destination->server, false); - if (!$composeFileContent) { - $this->emit('error', "Could not load compose file from $workdir$composeFile"); - return; - } else { - $this->application->docker_compose_raw = $composeFileContent; + try { + if ($isInit && $this->application->docker_compose_raw) { + return; + } + ['parsedServices' => $this->parsedServices, 'initialDockerComposeLocation' => $this->initialDockerComposeLocation] = $this->application->loadComposeFile($isInit); + $this->emit('success', 'Docker compose file loaded.'); + } catch (\Throwable $e) { + $this->application->docker_compose_location = $this->initialDockerComposeLocation; $this->application->save(); + return handleError($e, $this); } - $commands = collect([ - "rm -rf /tmp/{$uuid}", - ]); - instant_remote_process($commands, $this->application->destination->server, false); - $this->parsedServices = $this->application->parseCompose(); - $this->emit('success', 'Compose file loaded.'); + } + public function generateDomain(string $serviceName) + { + $domain = $this->parsedServiceDomains[$serviceName]['domain'] ?? null; + if (!$domain) { + $uuid = new Cuid2(7); + $domain = generateFqdn($this->application->destination->server, $uuid); + $this->parsedServiceDomains[$serviceName]['domain'] = $domain; + $this->application->docker_compose_domains = json_encode($this->parsedServiceDomains); + $this->application->save(); + $this->emit('success', 'Domain generated.'); + } + return $domain; } public function updatedApplicationBuildPack() { @@ -190,6 +188,9 @@ class General extends Component public function submit($showToaster = true) { try { + if ($this->initialDockerComposeLocation !== $this->application->docker_compose_location) { + $this->loadComposeFile(); + } $this->validate(); if ($this->ports_exposes !== $this->application->ports_exposes) { $this->resetDefaultLabels(false); diff --git a/app/Http/Livewire/Project/Application/Heading.php b/app/Http/Livewire/Project/Application/Heading.php index e09e469a9..dae105cd6 100644 --- a/app/Http/Livewire/Project/Application/Heading.php +++ b/app/Http/Livewire/Project/Application/Heading.php @@ -41,6 +41,10 @@ class Heading extends Component public function deploy(bool $force_rebuild = false) { + if (!$this->application->deployableComposeBuildPack()) { + $this->emit('error', 'Please load a Compose file first.'); + return; + } $this->setDeploymentUuid(); queue_application_deployment( application_id: $this->application->id, @@ -68,7 +72,8 @@ class Heading extends Component $this->application->save(); $this->application->refresh(); } - public function restart() { + public function restart() + { $this->setDeploymentUuid(); queue_application_deployment( application_id: $this->application->id, diff --git a/app/Http/Livewire/Project/Shared/EnvironmentVariable/Show.php b/app/Http/Livewire/Project/Shared/EnvironmentVariable/Show.php index eed0f7052..dc2808d08 100644 --- a/app/Http/Livewire/Project/Shared/EnvironmentVariable/Show.php +++ b/app/Http/Livewire/Project/Shared/EnvironmentVariable/Show.php @@ -59,6 +59,7 @@ class Show extends Component { $this->validate(); $this->env->save(); + ray($this->env); $this->emit('success', 'Environment variable updated successfully.'); $this->emit('refreshEnvs'); } diff --git a/app/Http/Livewire/Project/Shared/Logs.php b/app/Http/Livewire/Project/Shared/Logs.php index f58ec672a..982f729b4 100644 --- a/app/Http/Livewire/Project/Shared/Logs.php +++ b/app/Http/Livewire/Project/Shared/Logs.php @@ -17,13 +17,15 @@ class Logs extends Component public ?string $type = null; public Application|Service|StandalonePostgresql|StandaloneRedis|StandaloneMongodb|StandaloneMysql|StandaloneMariadb $resource; public Server $server; - public ?string $container = null; + public $container = []; + public $containers; public $parameters; public $query; public $status; public function mount() { + $this->containers = collect(); $this->parameters = get_route_parameters(); $this->query = request()->query(); if (data_get($this->parameters, 'application_uuid')) { @@ -33,7 +35,9 @@ class Logs extends Component $this->server = $this->resource->destination->server; $containers = getCurrentApplicationContainerStatus($this->server, $this->resource->id, 0); if ($containers->count() > 0) { - $this->container = data_get($containers[0], 'Names'); + $containers->each(function ($container) { + $this->containers->push(str_replace('/', '', $container['Names'])); + }); } } else if (data_get($this->parameters, 'database_uuid')) { $this->type = 'database'; diff --git a/app/Jobs/ApplicationDeployDockerImageJob.php b/app/Jobs/ApplicationDeployDockerImageJob.php index c21ad5680..9dfdad1f1 100644 --- a/app/Jobs/ApplicationDeployDockerImageJob.php +++ b/app/Jobs/ApplicationDeployDockerImageJob.php @@ -30,7 +30,7 @@ class ApplicationDeployDockerImageJob implements ShouldQueue, ShouldBeEncrypted } public function handle() { - ray()->clearAll(); + // ray()->clearAll(); ray('Deploying Docker Image'); static::$batch_counter = 0; try { diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index ac7d4dde7..f4982ad7d 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -76,7 +76,6 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted private string $docker_compose_location = '/docker-compose.yml'; private ?string $addHosts = null; private ?string $buildTarget = null; - private $log_model; private Collection $saved_outputs; private ?string $full_healthcheck_url = null; @@ -93,9 +92,7 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted public $tries = 1; public function __construct(int $application_deployment_queue_id) { - // ray()->clearScreen(); $this->application_deployment_queue = ApplicationDeploymentQueue::find($application_deployment_queue_id); - $this->log_model = $this->application_deployment_queue; $this->application = Application::find($this->application_deployment_queue->application_id); $this->build_pack = data_get($this->application, 'build_pack'); @@ -187,7 +184,6 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted ['repository' => $this->customRepository, 'port' => $this->customPort] = $this->application->customRepository(); try { - ray($this->application->build_pack); if ($this->restart_only && $this->application->build_pack !== 'dockerimage') { $this->just_restart(); if ($this->server->isProxyShouldRun()) { @@ -451,7 +447,6 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted $this->docker_compose_location = $this->application->docker_compose_location; } $this->application_deployment_queue->addLogEntry("Starting deployment of {$this->application->name}."); - $this->server->executeRemoteCommand( commands: $this->application->prepareHelperImage($this->deployment_uuid), loggingModel: $this->application_deployment_queue @@ -474,6 +469,7 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted $this->save_environment_variables(); $this->stop_running_container(force: true); $this->start_by_compose_file(); + $this->application->loadComposeFile(isInit: false); } private function deploy_dockerfile_buildpack() { @@ -752,14 +748,9 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted private function clone_repository() { $importCommands = $this->generate_git_import_commands(); - ray($importCommands); + $this->application_deployment_queue->addLogEntry("\n----------------------------------------"); + $this->application_deployment_queue->addLogEntry("Importing {$this->customRepository}:{$this->application->git_branch} (commit sha {$this->application->git_commit_sha}) to {$this->basedir}."); $this->execute_remote_command( - [ - "echo '\n----------------------------------------'", - ], - [ - "echo -n 'Importing {$this->customRepository}:{$this->application->git_branch} (commit sha {$this->application->git_commit_sha}) to {$this->basedir}. '" - ], [ $importCommands, "hidden" => true ] @@ -768,7 +759,6 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted private function generate_git_import_commands() { - ['commands' => $commands, 'branch' => $this->branch, 'fullRepoUrl' => $this->fullRepoUrl] = $this->application->generateGitImportCommands($this->deployment_uuid, $this->pull_request_id, $this->git_type); return $commands; } @@ -1178,10 +1168,9 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf"); private function stop_running_container(bool $force = false) { - $this->execute_remote_command(["echo -n 'Removing old container.'"]); + $this->application_deployment_queue->addLogEntry("Removing old containers."); if ($this->newVersionIsHealthy || $force) { $containers = getCurrentApplicationContainerStatus($this->server, $this->application->id, $this->pull_request_id); - ray($containers); if ($this->pull_request_id !== 0) { $containers = $containers->filter(function ($container) { return data_get($container, 'Names') === $this->container_name; @@ -1197,14 +1186,10 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf"); [executeInDocker($this->deployment_uuid, "docker rm -f $containerName >/dev/null 2>&1"), "hidden" => true, "ignore_errors" => true], ); }); - $this->execute_remote_command( - [ - "echo 'Rolling update completed.'" - ], - ); + $this->application_deployment_queue->addLogEntry("Rolling update completed."); } else { + $this->application_deployment_queue->addLogEntry("New container is not healthy, rolling back to the old container."); $this->execute_remote_command( - ["echo -n 'New container is not healthy, rolling back to the old container.'"], [executeInDocker($this->deployment_uuid, "docker rm -f $this->container_name >/dev/null 2>&1"), "hidden" => true, "ignore_errors" => true], ); } @@ -1213,8 +1198,8 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf"); private function start_by_compose_file() { if ($this->application->build_pack === 'dockerimage') { + $this->application_deployment_queue->addLogEntry("Pulling latest images from the registry."); $this->execute_remote_command( - ["echo -n 'Pulling latest images from the registry.'"], [executeInDocker($this->deployment_uuid, "docker compose --project-directory {$this->workdir} pull"), "hidden" => true], [executeInDocker($this->deployment_uuid, "docker compose --project-directory {$this->workdir} up --build -d"), "hidden" => true], ); diff --git a/app/Jobs/MultipleApplicationDeploymentJob.php b/app/Jobs/MultipleApplicationDeploymentJob.php index 32c98d3b0..4c3cc29c9 100644 --- a/app/Jobs/MultipleApplicationDeploymentJob.php +++ b/app/Jobs/MultipleApplicationDeploymentJob.php @@ -91,7 +91,6 @@ class MultipleApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted public $tries = 1; public function __construct(int $application_deployment_queue_id) { - // ray()->clearScreen(); $this->application_deployment_queue = ApplicationDeploymentQueue::find($application_deployment_queue_id); $this->log_model = $this->application_deployment_queue; $this->application = Application::find($this->application_deployment_queue->application_id); diff --git a/app/Models/Application.php b/app/Models/Application.php index 20f61ed39..3b0926659 100644 --- a/app/Models/Application.php +++ b/app/Models/Application.php @@ -8,6 +8,7 @@ use Spatie\Activitylog\Models\Activity; use Illuminate\Support\Str; use RuntimeException; use Symfony\Component\Yaml\Yaml; +use Visus\Cuid2\Cuid2; class Application extends BaseModel { @@ -47,6 +48,10 @@ class Application extends BaseModel $application->environment_variables_preview()->delete(); }); } + public function deployableComposeBuildPack() + { + return $this->build_pack === 'dockercompose' && $this->docker_compose_raw; + } public function link() { return route('project.application.configuration', [ @@ -592,4 +597,42 @@ class Application extends BaseModel return collect([]); } } + function loadComposeFile($isInit = false) + { + $initialDockerComposeLocation = $this->docker_compose_location; + if ($this->build_pack === 'dockercompose') { + if ($isInit && $this->docker_compose_raw) { + return; + } + $uuid = new Cuid2(); + ['commands' => $cloneCommand] = $this->generateGitImportCommands(deployment_uuid: $uuid, only_checkout: true, exec_in_docker: false, custom_base_dir: '.'); + $workdir = rtrim($this->base_directory, '/'); + $composeFile = $this->docker_compose_location; + $commands = collect([ + "mkdir -p /tmp/{$uuid} && cd /tmp/{$uuid}", + $cloneCommand, + "git sparse-checkout init --cone", + "git sparse-checkout set .$workdir$composeFile", + "git read-tree -mu HEAD", + "cat .$workdir$composeFile", + ]); + $composeFileContent = instant_remote_process($commands, $this->destination->server, false); + if (!$composeFileContent) { + $this->docker_compose_location = $initialDockerComposeLocation; + $this->save(); + throw new \Exception("Could not load compose file from $workdir$composeFile"); + } else { + $this->docker_compose_raw = $composeFileContent; + $this->save(); + } + $commands = collect([ + "rm -rf /tmp/{$uuid}", + ]); + instant_remote_process($commands, $this->destination->server, false); + return [ + 'parsedServices' => $this->parseCompose(), + 'initialDockerComposeLocation' => $this->docker_compose_location + ]; + } + } } diff --git a/app/Models/ApplicationDeploymentQueue.php b/app/Models/ApplicationDeploymentQueue.php index bc9b9471a..99aed3750 100644 --- a/app/Models/ApplicationDeploymentQueue.php +++ b/app/Models/ApplicationDeploymentQueue.php @@ -9,7 +9,8 @@ class ApplicationDeploymentQueue extends Model { protected $guarded = []; - public function getOutput($name) { + public function getOutput($name) + { if (!$this->logs) { return null; } @@ -21,9 +22,13 @@ class ApplicationDeploymentQueue extends Model if ($type === 'error') { $type = 'stderr'; } + $message = str($message)->trim(); + if ($message->startsWith('â•”')) { + $message = "\n" . $message; + } $newLogEntry = [ 'command' => null, - 'output' => $message, + 'output' => remove_iip($message), 'type' => $type, 'timestamp' => Carbon::now('UTC'), 'hidden' => $hidden, diff --git a/app/Traits/ExecuteRemoteCommand.php b/app/Traits/ExecuteRemoteCommand.php index 42d0d9e0f..6c0dc5d03 100644 --- a/app/Traits/ExecuteRemoteCommand.php +++ b/app/Traits/ExecuteRemoteCommand.php @@ -24,7 +24,6 @@ trait ExecuteRemoteCommand if ($this->server instanceof Server === false) { throw new \RuntimeException('Server is not set or is not an instance of Server model'); } - $commandsText->each(function ($single_command) { $command = data_get($single_command, 'command') ?? $single_command[0] ?? null; if ($command === null) { @@ -49,32 +48,29 @@ trait ExecuteRemoteCommand 'hidden' => $hidden, 'batch' => static::$batch_counter, ]; - - if (!$this->log_model->logs) { + if (!$this->application_deployment_queue->logs) { $new_log_entry['order'] = 1; } else { - $previous_logs = json_decode($this->log_model->logs, associative: true, flags: JSON_THROW_ON_ERROR); + $previous_logs = json_decode($this->application_deployment_queue->logs, associative: true, flags: JSON_THROW_ON_ERROR); $new_log_entry['order'] = count($previous_logs) + 1; } - $previous_logs[] = $new_log_entry; - $this->log_model->logs = json_encode($previous_logs, flags: JSON_THROW_ON_ERROR); - $this->log_model->save(); + $this->application_deployment_queue->logs = json_encode($previous_logs, flags: JSON_THROW_ON_ERROR); + $this->application_deployment_queue->save(); if ($this->save) { $this->saved_outputs[$this->save] = Str::of($output)->trim(); } }); - $this->log_model->update([ + $this->application_deployment_queue->update([ 'current_process_id' => $process->id(), ]); $process_result = $process->wait(); if ($process_result->exitCode() !== 0) { if (!$ignore_errors) { - $status = ApplicationDeploymentStatus::FAILED->value; - $this->log_model->status = $status; - $this->log_model->save(); + $this->application_deployment_queue->status = ApplicationDeploymentStatus::FAILED->value; + $this->application_deployment_queue->save(); throw new \RuntimeException($process_result->errorOutput()); } } diff --git a/bootstrap/helpers/docker.php b/bootstrap/helpers/docker.php index d6ef6f1ce..616e0e864 100644 --- a/bootstrap/helpers/docker.php +++ b/bootstrap/helpers/docker.php @@ -274,3 +274,18 @@ function generateLabelsApplication(Application $application, ?ApplicationPreview } return $labels->all(); } + +function isDatabaseImage(string $image) +{ + $image = str($image); + if ($image->contains(':')) { + $image = str($image); + } else { + $image = str($image)->append(':latest'); + } + $imageName = $image->before(':'); + if (collect(DATABASE_DOCKER_IMAGES)->contains($imageName)) { + return true; + } + return false; +} diff --git a/bootstrap/helpers/remoteProcess.php b/bootstrap/helpers/remoteProcess.php index 4f755c89e..45c8ae0e5 100644 --- a/bootstrap/helpers/remoteProcess.php +++ b/bootstrap/helpers/remoteProcess.php @@ -151,6 +151,7 @@ function decode_remote_command_output(?ApplicationDeploymentQueue $application_d if (is_null($application_deployment_queue)) { return collect([]); } + // ray(data_get($application_deployment_queue, 'logs')); try { $decoded = json_decode( data_get($application_deployment_queue, 'logs'), @@ -160,6 +161,7 @@ function decode_remote_command_output(?ApplicationDeploymentQueue $application_d } catch (\JsonException $exception) { return collect([]); } + // ray($decoded ); $formatted = collect($decoded); if (!$is_debug_enabled) { $formatted = $formatted->filter(fn ($i) => $i['hidden'] === false ?? false); diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php index 4f47d2332..a01017c13 100644 --- a/bootstrap/helpers/shared.php +++ b/bootstrap/helpers/shared.php @@ -580,7 +580,7 @@ function getTopLevelNetworks(Service|Application $resource) } function parseDockerComposeFile(Service|Application $resource, bool $isNew = false) { - ray()->clearAll(); + // ray()->clearAll(); if ($resource->getMorphClass() === 'App\Models\Service') { if ($resource->docker_compose_raw) { try { @@ -627,18 +627,8 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal $containerName = "$serviceName-{$resource->uuid}"; // Decide if the service is a database - $isDatabase = false; + $isDatabase = isDatabaseImage(data_get_str($service, 'image')); $image = data_get_str($service, 'image'); - if ($image->contains(':')) { - $image = Str::of($image); - } else { - $image = Str::of($image)->append(':latest'); - } - $imageName = $image->before(':'); - - if (collect(DATABASE_DOCKER_IMAGES)->contains($imageName)) { - $isDatabase = true; - } data_set($service, 'is_database', $isDatabase); // Create new serviceApplication or serviceDatabase @@ -1093,7 +1083,7 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal $resource->docker_compose = Yaml::dump($finalServices, 10, 2); $resource->save(); $resource->saveComposeConfigs(); - return $finalServices; + return collect($finalServices); } else { return collect([]); } @@ -1141,18 +1131,8 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal $containerName = "$serviceName-{$resource->uuid}"; // Decide if the service is a database - $isDatabase = false; + $isDatabase = isDatabaseImage(data_get_str($service, 'image')); $image = data_get_str($service, 'image'); - if ($image->contains(':')) { - $image = Str::of($image); - } else { - $image = Str::of($image)->append(':latest'); - } - $imageName = $image->before(':'); - - if (collect(DATABASE_DOCKER_IMAGES)->contains($imageName)) { - $isDatabase = true; - } data_set($service, 'is_database', $isDatabase); // Collect/create/update networks @@ -1367,14 +1347,14 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal if ($foundEnv) { $defaultValue = data_get($foundEnv, 'value'); } + $isBuildTime = data_get($foundEnv, 'is_build_time', false); EnvironmentVariable::updateOrCreate([ - 'key' => $key, + 'key' => $key->value(), 'application_id' => $resource->id, ], [ 'value' => $defaultValue, - 'is_build_time' => false, - 'service_id' => $resource->id, - 'is_preview' => false, + 'is_build_time' => $isBuildTime, + 'application_id' => $resource->id, ]); } } diff --git a/resources/views/components/applications/links.blade.php b/resources/views/components/applications/links.blade.php index 1c716a3af..44d69df62 100644 --- a/resources/views/components/applications/links.blade.php +++ b/resources/views/components/applications/links.blade.php @@ -16,22 +16,43 @@
    • @endif @if (data_get($application, 'fqdn')) - @foreach (Str::of(data_get($application, 'fqdn'))->explode(',') as $fqdn) -
  • - - - - - - - {{ getFqdnWithoutPort($fqdn) }} - -
    • - @endforeach + @if (data_get($application, 'build_pack') === 'dockercompose') + @foreach (collect(json_decode($this->application->docker_compose_domains)) as $fqdn) + @if (data_get($fqdn, 'domain')) +
  • + + + + + + + {{ getFqdnWithoutPort(data_get($fqdn, 'domain')) }} + +
    • + @endif + @endforeach + @else + @foreach (Str::of(data_get($application, 'fqdn'))->explode(',') as $fqdn) +
  • + + + + + + + {{ getFqdnWithoutPort($fqdn) }} + +
    • + @endforeach + @endif @endif @if (data_get($application, 'previews', collect([]))->count() > 0) @foreach (data_get($application, 'previews') as $preview) diff --git a/resources/views/components/applications/navbar.blade.php b/resources/views/components/applications/navbar.blade.php index 09035d370..d0cd565dc 100644 --- a/resources/views/components/applications/navbar.blade.php +++ b/resources/views/components/applications/navbar.blade.php @@ -13,51 +13,57 @@
    - - - @if ($application->status !== 'exited') - - @if ($application->build_pack !== 'dockercompose') - + @if ($application->build_pack !== 'dockercompose') + + @endif + + @else + @endif - - @else - @endif diff --git a/resources/views/livewire/project/application/configuration.blade.php b/resources/views/livewire/project/application/configuration.blade.php index 7f2efbc5a..ecfb74779 100644 --- a/resources/views/livewire/project/application/configuration.blade.php +++ b/resources/views/livewire/project/application/configuration.blade.php @@ -20,7 +20,7 @@ Server - @if ($application->build_pack !== 'static') + @if ($application->build_pack !== 'static' && $application->build_pack !== 'dockercompose') Storages @@ -34,7 +34,7 @@ Deployments @endif - @if ($application->build_pack !== 'static') + @if ($application->build_pack !== 'static' && $application->build_pack !== 'dockercompose') Healthchecks @@ -42,10 +42,12 @@ Rollback - Resource Limits - + @if ($application->build_pack !== 'dockercompose') + Resource Limits + + @endif Danger Zone diff --git a/resources/views/livewire/project/application/deployment-navbar.blade.php b/resources/views/livewire/project/application/deployment-navbar.blade.php index d42fb2298..936a3fb6e 100644 --- a/resources/views/livewire/project/application/deployment-navbar.blade.php +++ b/resources/views/livewire/project/application/deployment-navbar.blade.php @@ -1,5 +1,5 @@
    -

    Logs

    +

    Deployment Log

    @if ($is_debug_enabled) Hide Debug Logs @else diff --git a/resources/views/livewire/project/application/general.blade.php b/resources/views/livewire/project/application/general.blade.php index 67574eca8..6a1ddb675 100644 --- a/resources/views/livewire/project/application/general.blade.php +++ b/resources/views/livewire/project/application/general.blade.php @@ -32,7 +32,6 @@ - {{-- --}} @if ($application->settings->is_static || $application->build_pack === 'static') @@ -122,9 +121,17 @@ Reload Compose File @if (count($parsedServices) > 0) @foreach (data_get($parsedServices, 'services') as $serviceName => $service) - @if (!collect(DATABASE_DOCKER_IMAGES)->contains(data_get($service, 'image'))) - + @if (!isDatabaseImage(data_get($service, 'image'))) +
    + + @if (!data_get($parsedServiceDomains, "$serviceName.domain")) + Generate + Domain + @endif +
    @endif @endforeach @endif diff --git a/resources/views/livewire/project/shared/get-logs.blade.php b/resources/views/livewire/project/shared/get-logs.blade.php index 7f4c4a74d..77c147c5f 100644 --- a/resources/views/livewire/project/shared/get-logs.blade.php +++ b/resources/views/livewire/project/shared/get-logs.blade.php @@ -1,6 +1,6 @@
    -

    Logs

    +

    Container: {{$container}}

    @if ($streamLogs) @endif @@ -13,7 +13,7 @@ Refresh -
    +
    @endif @if ($application->build_pack !== 'dockerimage' && $application->build_pack !== 'dockercompose') @@ -119,24 +137,9 @@ @endif @if ($application->build_pack === 'dockercompose') Reload Compose File - @if (count($parsedServices) > 0) - @foreach (data_get($parsedServices, 'services') as $serviceName => $service) - @if (!isDatabaseImage(data_get($service, 'image'))) -
    - - @if (!data_get($parsedServiceDomains, "$serviceName.domain")) - Generate - Domain - @endif -
    - @endif - @endforeach - @endif - + + @endif @if ($application->dockerfile) diff --git a/routes/webhooks.php b/routes/webhooks.php index f59f8f681..17efae2d2 100644 --- a/routes/webhooks.php +++ b/routes/webhooks.php @@ -225,13 +225,9 @@ Route::post('/source/github/events/manual', function () { return response("Nothing to do. No applications found with branch '$base_branch'."); } } - ray($applications); foreach ($applications as $application) { - ray($application); $webhook_secret = data_get($application, 'manual_webhook_secret_github'); - ray($webhook_secret); $hmac = hash_hmac('sha256', request()->getContent(), $webhook_secret); - ray($hmac, $x_hub_signature_256); if (!hash_equals($x_hub_signature_256, $hmac)) { ray('Invalid signature'); continue; @@ -324,7 +320,6 @@ Route::post('/source/github/events', function () { $webhook_secret = data_get($github_app, 'webhook_secret'); $hmac = hash_hmac('sha256', request()->getContent(), $webhook_secret); - ray($hmac, $x_hub_signature_256)->blue(); if (config('app.env') !== 'local') { if (!hash_equals($x_hub_signature_256, $hmac)) { return response('not cool'); @@ -661,12 +656,10 @@ Route::post('/payments/paddle/events', function () { $h1 = Str::of($signature)->after('h1='); $signedPayload = $ts->value . ':' . request()->getContent(); $verify = hash_hmac('sha256', $signedPayload, config('subscription.paddle_webhook_secret')); - ray($verify, $h1->value, hash_equals($verify, $h1->value)); if (!hash_equals($verify, $h1->value)) { return response('Invalid signature.', 400); } $eventType = data_get($payload, 'event_type'); - ray($eventType); $webhook = Webhook::create([ 'type' => 'paddle', 'payload' => $payload, From c1710c8f7bd24d10e1b50b87b35cef452cea47f7 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Mon, 27 Nov 2023 15:25:15 +0100 Subject: [PATCH 28/46] moar fixes --- app/Actions/Application/StopApplication.php | 10 ++ .../Livewire/Project/Application/General.php | 4 + .../Livewire/Project/Application/Heading.php | 2 +- .../Livewire/Project/Application/Previews.php | 2 +- app/Jobs/ApplicationDeploymentJob.php | 1 - app/Models/Application.php | 5 - app/Models/ApplicationPreview.php | 21 ++- bootstrap/helpers/shared.php | 15 +- .../components/applications/links.blade.php | 167 +++++++++--------- .../project/application/previews.blade.php | 7 +- 10 files changed, 138 insertions(+), 96 deletions(-) diff --git a/app/Actions/Application/StopApplication.php b/app/Actions/Application/StopApplication.php index a98384f45..1d09f0daf 100644 --- a/app/Actions/Application/StopApplication.php +++ b/app/Actions/Application/StopApplication.php @@ -25,5 +25,15 @@ class StopApplication // TODO: make notification for application // $application->environment->project->team->notify(new StatusChanged($application)); } + // Delete Preview Deployments + $previewDeployments = $application->previews; + foreach ($previewDeployments as $previewDeployment) { + $containers = getCurrentApplicationContainerStatus($server, $application->id, $previewDeployment->pull_request_id); + foreach ($containers as $container) { + $name = str_replace('/', '', $container['Names']); + instant_remote_process(["docker rm -f $name"], $application->destination->server, throwError: false); + } + } + } } diff --git a/app/Http/Livewire/Project/Application/General.php b/app/Http/Livewire/Project/Application/General.php index d108bca14..0284b40a9 100644 --- a/app/Http/Livewire/Project/Application/General.php +++ b/app/Http/Livewire/Project/Application/General.php @@ -152,6 +152,10 @@ class General extends Component $this->application->settings->is_static = $this->is_static = false; $this->application->settings->save(); } + if ($this->application->build_pack === 'dockercompose') { + $this->application->fqdn = null; + $this->application->settings->save(); + } $this->submit(); } public function checkLabelUpdates() diff --git a/app/Http/Livewire/Project/Application/Heading.php b/app/Http/Livewire/Project/Application/Heading.php index dae105cd6..14bdf1db0 100644 --- a/app/Http/Livewire/Project/Application/Heading.php +++ b/app/Http/Livewire/Project/Application/Heading.php @@ -41,7 +41,7 @@ class Heading extends Component public function deploy(bool $force_rebuild = false) { - if (!$this->application->deployableComposeBuildPack()) { + if ($this->application->build_pack === 'dockercompose' && is_null($this->application->docker_compose_raw)) { $this->emit('error', 'Please load a Compose file first.'); return; } diff --git a/app/Http/Livewire/Project/Application/Previews.php b/app/Http/Livewire/Project/Application/Previews.php index fd471a007..effdd7f7c 100644 --- a/app/Http/Livewire/Project/Application/Previews.php +++ b/app/Http/Livewire/Project/Application/Previews.php @@ -77,7 +77,7 @@ class Previews extends Component $name = str_replace('/', '', $container['Names']); instant_remote_process(["docker rm -f $name"], $this->application->destination->server, throwError: false); } - ApplicationPreview::where('application_id', $this->application->id)->where('pull_request_id', $pull_request_id)->delete(); + ApplicationPreview::where('application_id', $this->application->id)->where('pull_request_id', $pull_request_id)->first()->delete(); $this->application->refresh(); } catch (\Throwable $e) { return handleError($e, $this); diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index 6dca962bb..36907a917 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -949,7 +949,6 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted ]; } if ($this->application->settings->is_gpu_enabled) { - ray('asd'); $docker_compose['services'][$this->container_name]['deploy']['resources']['reservations']['devices'] = [ [ 'driver' => data_get($this->application, 'settings.gpu_driver', 'nvidia'), diff --git a/app/Models/Application.php b/app/Models/Application.php index 809d5ff6f..2e58d1891 100644 --- a/app/Models/Application.php +++ b/app/Models/Application.php @@ -48,10 +48,6 @@ class Application extends BaseModel $application->environment_variables_preview()->delete(); }); } - public function deployableComposeBuildPack() - { - return $this->build_pack === 'dockercompose' && $this->docker_compose_raw; - } public function link() { return route('project.application.configuration', [ @@ -262,7 +258,6 @@ class Application extends BaseModel { return $this->morphTo(); } - public function isDeploymentInprogress() { $deployments = ApplicationDeploymentQueue::where('application_id', $this->id)->where('status', 'in_progress')->count(); diff --git a/app/Models/ApplicationPreview.php b/app/Models/ApplicationPreview.php index 13775abae..87dce056e 100644 --- a/app/Models/ApplicationPreview.php +++ b/app/Models/ApplicationPreview.php @@ -5,7 +5,26 @@ namespace App\Models; class ApplicationPreview extends BaseModel { protected $guarded = []; - + protected static function booted() + { + static::deleting(function ($preview) { + if ($preview->application->build_pack === 'dockercompose') { + $server = $preview->application->destination->server; + $composeFile = $preview->application->parseCompose(pull_request_id: $preview->pull_request_id); + $volumes = data_get($composeFile, 'volumes'); + $networks = data_get($composeFile, 'networks'); + $networkKeys = collect($networks)->keys(); + $volumeKeys = collect($volumes)->keys(); + $volumeKeys->each(function ($key) use ($server) { + instant_remote_process(["docker volume rm -f $key"], $server, false); + }); + $networkKeys->each(function ($key) use ($server) { + instant_remote_process(["docker network disconnect $key coolify-proxy"], $server, false); + instant_remote_process(["docker network rm $key"], $server, false); + }); + } + }); + } static function findPreviewByApplicationAndPullId(int $application_id, int $pull_request_id) { return self::where('application_id', $application_id)->where('pull_request_id', $pull_request_id)->firstOrFail(); diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php index df562d5ff..ecb67cd0c 100644 --- a/bootstrap/helpers/shared.php +++ b/bootstrap/helpers/shared.php @@ -1144,14 +1144,21 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal if ($volume->contains(':')) { $name = $volume->before(':'); $mount = $volume->after(':'); - $newName = $name . "-{$resource->uuid}-$pull_request_id"; + $newName = $resource->uuid . "-{$name}-pr-$pull_request_id"; $volume = str("$newName:$mount"); $topLevelVolumes->put($newName, [ 'name' => $newName, ]); } } else if (is_array($volume)) { - $volume['source'] = str($volume['source'])->append("-{$resource->uuid}-$pull_request_id"); + $source = data_get($volume, 'source'); + if ($source) { + $newSource = $resource->uuid . "-{$source}-pr-$pull_request_id"; + data_set($volume, 'source', $newSource); + $topLevelVolumes->put($newSource, [ + 'name' => $newSource, + ]); + } } @@ -1159,6 +1166,8 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal }); data_set($service, 'volumes', $serviceVolumes->toArray()); } + } else { + } // Decide if the service is a database $isDatabase = isDatabaseImage(data_get_str($service, 'image')); @@ -1202,7 +1211,7 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal if ($pull_request_id !== 0) { $topLevelNetworks->put($network, [ 'name' => $network, - 'external' => false + 'external' => true ]); } else { $topLevelNetworks->put($network, [ diff --git a/resources/views/components/applications/links.blade.php b/resources/views/components/applications/links.blade.php index 44d69df62..c5bdc4a48 100644 --- a/resources/views/components/applications/links.blade.php +++ b/resources/views/components/applications/links.blade.php @@ -1,21 +1,25 @@
    - + @if (data_get($application, 'fqdn') || + collect(json_decode($this->application->docker_compose_domains))->count() > 0 || + data_get($application, 'previews', collect([]))->count() > 0 || + data_get($application, 'ports_mappings_array')) + - + @endif
    diff --git a/resources/views/livewire/project/application/previews.blade.php b/resources/views/livewire/project/application/previews.blade.php index b02acc4ab..6663c71d3 100644 --- a/resources/views/livewire/project/application/previews.blade.php +++ b/resources/views/livewire/project/application/previews.blade.php @@ -71,15 +71,16 @@
    - + @if (data_get($preview, 'status') === 'exited') Deploy @else Redeploy @endif - Remove - Preview + Remove Preview From 23571ae104fc7dcc0d817cd22f0fe9644feb9756 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Mon, 27 Nov 2023 15:50:22 +0100 Subject: [PATCH 29/46] wip --- .../Livewire/Project/Application/General.php | 6 +- app/Jobs/ApplicationDeployDockerImageJob.php | 151 --- .../ApplicationDeploySimpleDockerfileJob.php | 46 - app/Jobs/ApplicationDeploymentJob.php | 1 - app/Jobs/MultipleApplicationDeploymentJob.php | 1164 ----------------- app/Models/Application.php | 36 +- bootstrap/helpers/applications.php | 32 - bootstrap/helpers/shared.php | 13 +- ..._24_080341_add_docker_compose_location.php | 15 +- .../project/application/general.blade.php | 5 +- 10 files changed, 62 insertions(+), 1407 deletions(-) delete mode 100644 app/Jobs/ApplicationDeployDockerImageJob.php delete mode 100644 app/Jobs/ApplicationDeploySimpleDockerfileJob.php delete mode 100644 app/Jobs/MultipleApplicationDeploymentJob.php diff --git a/app/Http/Livewire/Project/Application/General.php b/app/Http/Livewire/Project/Application/General.php index 0284b40a9..da642ba77 100644 --- a/app/Http/Livewire/Project/Application/General.php +++ b/app/Http/Livewire/Project/Application/General.php @@ -27,6 +27,7 @@ class General extends Component public bool $isConfigurationChanged = false; public ?string $initialDockerComposeLocation = null; + public ?string $initialDockerComposePrLocation = null; public bool $is_static; @@ -57,6 +58,7 @@ class General extends Component 'application.docker_registry_image_tag' => 'nullable', 'application.dockerfile_location' => 'nullable', 'application.docker_compose_location' => 'nullable', + 'application.docker_compose_pr_location' => 'nullable', 'application.docker_compose' => 'nullable', 'application.docker_compose_raw' => 'nullable', 'application.custom_labels' => 'nullable', @@ -84,6 +86,7 @@ class General extends Component 'application.docker_registry_image_tag' => 'Docker registry image tag', 'application.dockerfile_location' => 'Dockerfile location', 'application.docker_compose_location' => 'Docker compose location', + 'application.docker_compose_pr_location' => 'Docker compose location', 'application.docker_compose' => 'Docker compose', 'application.docker_compose_raw' => 'Docker compose raw', 'application.custom_labels' => 'Custom labels', @@ -125,10 +128,11 @@ class General extends Component if ($isInit && $this->application->docker_compose_raw) { return; } - ['parsedServices' => $this->parsedServices, 'initialDockerComposeLocation' => $this->initialDockerComposeLocation] = $this->application->loadComposeFile($isInit); + ['parsedServices' => $this->parsedServices, 'initialDockerComposeLocation' => $this->initialDockerComposeLocation, 'initialDockerComposePrLocation' => $this->initialDockerComposePrLocation] = $this->application->loadComposeFile($isInit); $this->emit('success', 'Docker compose file loaded.'); } catch (\Throwable $e) { $this->application->docker_compose_location = $this->initialDockerComposeLocation; + $this->application->docker_compose_pr_location = $this->initialDockerComposePrLocation; $this->application->save(); return handleError($e, $this); } diff --git a/app/Jobs/ApplicationDeployDockerImageJob.php b/app/Jobs/ApplicationDeployDockerImageJob.php deleted file mode 100644 index 9dfdad1f1..000000000 --- a/app/Jobs/ApplicationDeployDockerImageJob.php +++ /dev/null @@ -1,151 +0,0 @@ -clearAll(); - ray('Deploying Docker Image'); - static::$batch_counter = 0; - try { - $deploymentUuid = data_get($this->deploymentQueueEntry, 'deployment_uuid'); - $pullRequestId = data_get($this->deploymentQueueEntry, 'pull_request_id'); - - $this->server = data_get($this->application->destination, 'server'); - $network = data_get($this->application->destination, 'network'); - - $dockerImage = data_get($this->application, 'docker_registry_image_name'); - $dockerImageTag = data_get($this->application, 'docker_registry_image_tag'); - - $productionImageName = str("{$dockerImage}:{$dockerImageTag}"); - $this->containerName = generateApplicationContainerName($this->application, $pullRequestId); - savePrivateKeyToFs($this->server); - - ray("echo 'Starting deployment of {$productionImageName}.'"); - - $this->deploymentQueueEntry->update([ - 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value, - ]); - - $this->deploymentQueueEntry->addLogEntry('Starting deployment of ' . $productionImageName); - - $this->server->executeRemoteCommand( - commands: collect( - [ - [ - "name" => "ls", - "command" => 'ls -la', - "hidden" => true, - ], - [ - "name" => "pwd", - "command" => 'pwd', - "hidden" => true, - ] - ], - ), - loggingModel: $this->deploymentQueueEntry - ); - $this->server->executeRemoteCommand( - commands: prepareHelperContainer($this->server, $network, $deploymentUuid), - loggingModel: $this->deploymentQueueEntry - ); - $this->server->executeRemoteCommand( - commands: generateComposeFile( - deploymentUuid: $deploymentUuid, - server: $this->server, - network: $network, - application: $this->application, - containerName: $this->containerName, - imageName: $productionImageName, - pullRequestId: $pullRequestId - ), - loggingModel: $this->deploymentQueueEntry - ); - $this->deploymentQueueEntry->addLogEntry('----------------------------------------'); - - // Rolling update not possible - if (count($this->application->ports_mappings_array) > 0) { - $this->deploymentQueueEntry->addLogEntry('Application has ports mapped to the host system, rolling update is not supported.'); - $this->deploymentQueueEntry->addLogEntry('Stopping running container.'); - $this->server->stopApplicationRelatedRunningContainers($this->application->id, $this->containerName); - } else { - $this->deploymentQueueEntry->addLogEntry('Rolling update started.'); - // TODO - $this->server->executeRemoteCommand( - commands: startNewApplication(application: $this->application, deploymentUuid: $deploymentUuid, loggingModel: $this->deploymentQueueEntry), - loggingModel: $this->deploymentQueueEntry - ); - // $this->server->executeRemoteCommand( - // commands: healthCheckContainer(application: $this->application, containerName: $this->containerName , loggingModel: $this->deploymentQueueEntry), - // loggingModel: $this->deploymentQueueEntry - // ); - - } - - ray($this->remoteCommandOutputs); - $this->deploymentQueueEntry->update([ - 'status' => ApplicationDeploymentStatus::FINISHED->value, - ]); - } catch (Throwable $e) { - $this->fail($e); - throw $e; - } - } - public function failed(Throwable $exception): void - { - $this->deploymentQueueEntry->addLogEntry('Oops something is not okay, are you okay? 😢', 'error'); - $this->deploymentQueueEntry->addLogEntry($exception->getMessage(), 'error'); - $this->deploymentQueueEntry->addLogEntry('Deployment failed. Removing the new version of your application.'); - - $this->server->executeRemoteCommand( - commands: removeOldDeployment($this->containerName), - loggingModel: $this->deploymentQueueEntry - ); - $this->deploymentQueueEntry->update([ - 'status' => ApplicationDeploymentStatus::FAILED->value, - ]); - } - // private function next(string $status) - // { - // // If the deployment is cancelled by the user, don't update the status - // if ($this->application_deployment_queue->status !== ApplicationDeploymentStatus::CANCELLED_BY_USER->value) { - // $this->application_deployment_queue->update([ - // 'status' => $status, - // ]); - // } - // queue_next_deployment($this->application); - // if ($status === ApplicationDeploymentStatus::FINISHED->value) { - // $this->application->environment->project->team->notify(new DeploymentSuccess($this->application, $this->deployment_uuid, $this->preview)); - // } - // if ($status === ApplicationDeploymentStatus::FAILED->value) { - // $this->application->environment->project->team->notify(new DeploymentFailed($this->application, $this->deployment_uuid, $this->preview)); - // } - // } -} diff --git a/app/Jobs/ApplicationDeploySimpleDockerfileJob.php b/app/Jobs/ApplicationDeploySimpleDockerfileJob.php deleted file mode 100644 index 0cf9f61c4..000000000 --- a/app/Jobs/ApplicationDeploySimpleDockerfileJob.php +++ /dev/null @@ -1,46 +0,0 @@ -applicationDeploymentQueueId = $applicationDeploymentQueueId; - } - public function handle() - { - ray('Deploying Simple Dockerfile'); - $applicationDeploymentQueue = ApplicationDeploymentQueue::find($this->applicationDeploymentQueueId); - $application = Application::find($applicationDeploymentQueue->application_id); - $destination = $application->destination->getMorphClass()::where('id', $application->destination->id)->first(); - $server = data_get($destination, 'server'); - $commands = collect([]); - $commands->push( - [ - 'command' => 'echo "Starting deployment of simple dockerfile."', - ], - [ - 'command' => 'ls -la', - ] - ); - $server->executeRemoteCommand(commands: $commands, logModel: $applicationDeploymentQueue); - } -} diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index 36907a917..eea334c8e 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -456,7 +456,6 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted if ($this->pull_request_id === 0) { $this->application_deployment_queue->addLogEntry("Starting deployment of {$this->application->name}."); } else { - ray('asd'); $this->application_deployment_queue->addLogEntry("Starting pull request (#{$this->pull_request_id}) deployment of {$this->customRepository}:{$this->application->git_branch}."); } $this->server->executeRemoteCommand( diff --git a/app/Jobs/MultipleApplicationDeploymentJob.php b/app/Jobs/MultipleApplicationDeploymentJob.php deleted file mode 100644 index 4c3cc29c9..000000000 --- a/app/Jobs/MultipleApplicationDeploymentJob.php +++ /dev/null @@ -1,1164 +0,0 @@ -application_deployment_queue = ApplicationDeploymentQueue::find($application_deployment_queue_id); - $this->log_model = $this->application_deployment_queue; - $this->application = Application::find($this->application_deployment_queue->application_id); - $this->build_pack = data_get($this->application, 'build_pack'); - - $this->application_deployment_queue_id = $application_deployment_queue_id; - $this->deployment_uuid = $this->application_deployment_queue->deployment_uuid; - $this->commit = $this->application_deployment_queue->commit; - $this->force_rebuild = $this->application_deployment_queue->force_rebuild; - $this->restart_only = $this->application_deployment_queue->restart_only; - - $this->git_type = data_get($this->application_deployment_queue, 'git_type'); - - $source = data_get($this->application, 'source'); - if ($source) { - $this->source = $source->getMorphClass()::where('id', $this->application->source->id)->first(); - } - $this->destination = $this->application->destination->getMorphClass()::where('id', $this->application->destination->id)->first(); - $this->server = $this->mainServer = $this->destination->server; - $this->serverUser = $this->server->user; - $this->basedir = generateBaseDir($this->deployment_uuid); - $this->workdir = "{$this->basedir}" . rtrim($this->application->base_directory, '/'); - $this->configuration_dir = application_configuration_dir() . "/{$this->application->uuid}"; - $this->is_debug_enabled = $this->application->settings->is_debug_enabled; - $this->saved_outputs = collect(); - $this->container_name = generateApplicationContainerName($this->application, 0); - } - - public function handle(): void - { - savePrivateKeyToFs($this->server); - $this->application_deployment_queue->update([ - 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value, - ]); - - $this->addHosts = generateHostIpMapping($this->server, $this->destination->network); - - if ($this->application->dockerfile_target_build) { - $this->buildTarget = " --target {$this->application->dockerfile_target_build} "; - } - - // Check custom port - preg_match('/(?<=:)\d+(?=\/)/', $this->application->git_repository, $matches); - if (count($matches) === 1) { - $this->customPort = $matches[0]; - $gitHost = str($this->application->git_repository)->before(':'); - $gitRepo = str($this->application->git_repository)->after('/'); - $this->customRepository = "$gitHost:$gitRepo"; - } else { - $this->customRepository = $this->application->git_repository; - } - try { - if ($this->application->isMultipleServerDeployment()) { - if ($this->application->build_pack === 'dockerimage') { - $this->dockerImage = $this->application->docker_registry_image_name; - $this->dockerImageTag = $this->application->docker_registry_image_tag; - $this->execute_remote_command( - [ - "echo 'Starting deployment of {$this->dockerImage}:{$this->dockerImageTag}.'" - ], - ); - $this->production_image_name = Str::lower("{$this->dockerImage}:{$this->dockerImageTag}"); - ray(prepareHelperContainer($this->server, $this->deployment_uuid)); - $this->execute_remote_command( - [prepareHelperContainer($this->server, $this->deployment_uuid)] - ); - } - } else { - throw new RuntimeException('Missing configuration for multiple server deployment.'); - } - // if ($this->restart_only && $this->application->build_pack !== 'dockerimage') { - // $this->just_restart(); - // if ($this->server->isProxyShouldRun()) { - // dispatch(new ContainerStatusJob($this->server)); - // } - // $this->next(ApplicationDeploymentStatus::FINISHED->value); - // $this->application->isConfigurationChanged(true); - // return; - // } else if ($this->application->dockerfile) { - // $this->deploy_simple_dockerfile(); - // } else if ($this->application->build_pack === 'dockerimage') { - // $this->deploy_dockerimage_buildpack(); - // } else if ($this->application->build_pack === 'dockerfile') { - // $this->deploy_dockerfile_buildpack(); - // } else if ($this->application->build_pack === 'static') { - // $this->deploy_static_buildpack(); - // } else { - // $this->deploy_nixpacks_buildpack(); - // } - // if ($this->server->isProxyShouldRun()) { - // dispatch(new ContainerStatusJob($this->server)); - // } - // if ($this->application->docker_registry_image_name) { - // $this->push_to_docker_registry(); - // } - // $this->next(ApplicationDeploymentStatus::FINISHED->value); - // $this->application->isConfigurationChanged(true); - } catch (Exception $e) { - $this->fail($e); - throw $e; - } finally { - // if (isset($this->docker_compose_base64)) { - // $readme = generate_readme_file($this->application->name, $this->application_deployment_queue->updated_at); - // $composeFileName = "$this->configuration_dir/docker-compose.yml"; - // $this->execute_remote_command( - // [ - // "mkdir -p $this->configuration_dir" - // ], - // [ - // "echo '{$this->docker_compose_base64}' | base64 -d > $composeFileName", - // ], - // [ - // "echo '{$readme}' > $this->configuration_dir/README.md", - // ] - // ); - // } - // $this->execute_remote_command( - // [ - // "docker rm -f {$this->deployment_uuid} >/dev/null 2>&1", - // "hidden" => true, - // "ignore_errors" => true, - // ] - // ); - // $this->execute_remote_command( - // [ - // "docker image prune -f >/dev/null 2>&1", - // "hidden" => true, - // "ignore_errors" => true, - // ] - // ); - } - } - private function push_to_docker_registry() - { - try { - instant_remote_process(["docker images --format '{{json .}}' {$this->production_image_name}"], $this->server); - $this->execute_remote_command( - [ - "echo '\n----------------------------------------'", - ], - ["echo -n 'Pushing image to docker registry ({$this->production_image_name}).'"], - [ - executeInDocker($this->deployment_uuid, "docker push {$this->production_image_name}"), 'hidden' => true - ], - ); - if ($this->application->docker_registry_image_tag) { - // Tag image with latest - $this->execute_remote_command( - ['echo -n "Tagging and pushing image with latest tag."'], - [ - executeInDocker($this->deployment_uuid, "docker tag {$this->production_image_name} {$this->application->docker_registry_image_name}:{$this->application->docker_registry_image_tag}"), 'ignore_errors' => true, 'hidden' => true - ], - [ - executeInDocker($this->deployment_uuid, "docker push {$this->application->docker_registry_image_name}:{$this->application->docker_registry_image_tag}"), 'ignore_errors' => true, 'hidden' => true - ], - ); - } - $this->execute_remote_command([ - "echo -n 'Image pushed to docker registry.'" - ]); - } catch (Exception $e) { - $this->execute_remote_command( - ["echo -n 'Failed to push image to docker registry. Please check debug logs for more information.'"], - ); - ray($e); - } - } - // private function deploy_docker_compose() - // { - // $dockercompose_base64 = base64_encode($this->application->dockercompose); - // $this->execute_remote_command( - // [ - // "echo 'Starting deployment of {$this->application->name}.'" - // ], - // ); - // $this->prepare_builder_image(); - // $this->execute_remote_command( - // [ - // executeInDocker($this->deployment_uuid, "echo '$dockercompose_base64' | base64 -d > $this->workdir/docker-compose.yaml") - // ], - // ); - // $this->build_image_name = Str::lower("{$this->customRepository}:build"); - // $this->production_image_name = Str::lower("{$this->application->uuid}:latest"); - // $this->save_environment_variables(); - // $containers = getCurrentApplicationContainerStatus($this->application->destination->server, $this->application->id); - // ray($containers); - // if ($containers->count() > 0) { - // foreach ($containers as $container) { - // $containerName = data_get($container, 'Names'); - // if ($containerName) { - // instant_remote_process( - // ["docker rm -f {$containerName}"], - // $this->application->destination->server - // ); - // } - // } - // } - - // $this->execute_remote_command( - // ["echo -n 'Starting services (could take a while)...'"], - // [executeInDocker($this->deployment_uuid, "docker compose --project-directory {$this->workdir} up -d"), "hidden" => true], - // ); - // } - private function generate_image_names() - { - if ($this->application->dockerfile) { - if ($this->application->docker_registry_image_name) { - $this->build_image_name = Str::lower("{$this->application->docker_registry_image_name}:build"); - $this->production_image_name = Str::lower("{$this->application->docker_registry_image_name}:latest"); - } else { - $this->build_image_name = Str::lower("{$this->application->uuid}:build"); - $this->production_image_name = Str::lower("{$this->application->uuid}:latest"); - } - } else if ($this->application->build_pack === 'dockerimage') { - $this->production_image_name = Str::lower("{$this->dockerImage}:{$this->dockerImageTag}"); - } else { - $this->dockerImageTag = str($this->commit)->substr(0, 128); - if ($this->application->docker_registry_image_name) { - $this->build_image_name = Str::lower("{$this->application->docker_registry_image_name}:{$this->dockerImageTag}-build"); - $this->production_image_name = Str::lower("{$this->application->docker_registry_image_name}:{$this->dockerImageTag}"); - } else { - $this->build_image_name = Str::lower("{$this->application->uuid}:{$this->dockerImageTag}-build"); - $this->production_image_name = Str::lower("{$this->application->uuid}:{$this->dockerImageTag}"); - } - } - } - private function just_restart() - { - $this->execute_remote_command( - [ - "echo 'Starting deployment of {$this->customRepository}:{$this->application->git_branch}.'" - ], - ); - $this->prepare_builder_image(); - $this->check_git_if_build_needed(); - $this->set_base_dir(); - $this->generate_image_names(); - $this->check_image_locally_or_remotely(); - if (str($this->saved_outputs->get('local_image_found'))->isNotEmpty()) { - $this->generate_compose_file(); - $this->rolling_update(); - return; - } - throw new RuntimeException('Cannot find image anywhere. Please redeploy the application.'); - } - private function check_image_locally_or_remotely() - { - $this->execute_remote_command([ - "docker images -q {$this->production_image_name} 2>/dev/null", "hidden" => true, "save" => "local_image_found" - ]); - if (str($this->saved_outputs->get('local_image_found'))->isEmpty() && $this->application->docker_registry_image_name) { - $this->execute_remote_command([ - "docker pull {$this->production_image_name} 2>/dev/null", "ignore_errors" => true, "hidden" => true - ]); - $this->execute_remote_command([ - "docker images -q {$this->production_image_name} 2>/dev/null", "hidden" => true, "save" => "local_image_found" - ]); - } - } - // private function save_environment_variables() - // { - // $envs = collect([]); - // foreach ($this->application->environment_variables as $env) { - // $envs->push($env->key . '=' . $env->value); - // } - // $envs_base64 = base64_encode($envs->implode("\n")); - // $this->execute_remote_command( - // [ - // executeInDocker($this->deployment_uuid, "echo '$envs_base64' | base64 -d > $this->workdir/.env") - // ], - // ); - // } - private function deploy_simple_dockerfile() - { - $dockerfile_base64 = base64_encode($this->application->dockerfile); - $this->execute_remote_command( - [ - "echo 'Starting deployment of {$this->application->name}.'" - ], - ); - $this->prepare_builder_image(); - $this->execute_remote_command( - [ - executeInDocker($this->deployment_uuid, "echo '$dockerfile_base64' | base64 -d > $this->workdir$this->dockerfile_location") - ], - ); - $this->generate_image_names(); - $this->generate_compose_file(); - $this->generate_build_env_variables(); - $this->add_build_env_variables_to_dockerfile(); - $this->build_image(); - $this->rolling_update(); - } - - private function deploy_dockerimage_buildpack() - { - // $this->dockerImage = $this->application->docker_registry_image_name; - // $this->dockerImageTag = $this->application->docker_registry_image_tag; - // ray("echo 'Starting deployment of {$this->dockerImage}:{$this->dockerImageTag}.'"); - // $this->execute_remote_command( - // [ - // "echo 'Starting deployment of {$this->dockerImage}:{$this->dockerImageTag}.'" - // ], - // ); - // $this->generate_image_names(); - // $this->prepare_builder_image(); - $this->generate_compose_file(); - $this->rolling_update(); - } - - private function deploy_dockerfile_buildpack() - { - if (data_get($this->application, 'dockerfile_location')) { - $this->dockerfile_location = $this->application->dockerfile_location; - } - $this->execute_remote_command( - [ - "echo 'Starting deployment of {$this->customRepository}:{$this->application->git_branch}.'" - ], - ); - $this->prepare_builder_image(); - $this->check_git_if_build_needed(); - $this->clone_repository(); - $this->set_base_dir(); - $this->generate_image_names(); - $this->cleanup_git(); - $this->generate_compose_file(); - $this->generate_build_env_variables(); - $this->add_build_env_variables_to_dockerfile(); - $this->build_image(); - // if ($this->application->additional_destinations) { - // $this->push_to_docker_registry(); - // $this->deploy_to_additional_destinations(); - // } else { - $this->rolling_update(); - // } - } - private function deploy_nixpacks_buildpack() - { - $this->execute_remote_command( - [ - "echo 'Starting deployment of {$this->customRepository}:{$this->application->git_branch}.'" - ], - ); - $this->prepare_builder_image(); - $this->check_git_if_build_needed(); - $this->set_base_dir(); - $this->generate_image_names(); - if (!$this->force_rebuild) { - $this->check_image_locally_or_remotely(); - if (str($this->saved_outputs->get('local_image_found'))->isNotEmpty() && !$this->application->isConfigurationChanged()) { - $this->execute_remote_command([ - "echo 'No configuration changed & image found ({$this->production_image_name}) with the same Git Commit SHA. Build step skipped.'", - ]); - $this->generate_compose_file(); - $this->rolling_update(); - return; - } - if ($this->application->isConfigurationChanged()) { - $this->execute_remote_command([ - "echo 'Configuration changed. Rebuilding image.'", - ]); - } - } - $this->clone_repository(); - $this->cleanup_git(); - $this->generate_nixpacks_confs(); - $this->generate_compose_file(); - $this->generate_build_env_variables(); - $this->add_build_env_variables_to_dockerfile(); - $this->build_image(); - $this->rolling_update(); - } - private function deploy_static_buildpack() - { - $this->execute_remote_command( - [ - "echo 'Starting deployment of {$this->customRepository}:{$this->application->git_branch}.'" - ], - ); - $this->prepare_builder_image(); - $this->check_git_if_build_needed(); - $this->set_base_dir(); - $this->generate_image_names(); - $this->clone_repository(); - $this->cleanup_git(); - $this->build_image(); - $this->generate_compose_file(); - $this->rolling_update(); - } - - private function rolling_update() - { - if (count($this->application->ports_mappings_array) > 0) { - $this->execute_remote_command( - [ - "echo '\n----------------------------------------'", - ], - ["echo -n 'Application has ports mapped to the host system, rolling update is not supported.'"], - ); - $this->stop_running_container(force: true); - $this->start_by_compose_file(); - } else { - $this->execute_remote_command( - [ - "echo '\n----------------------------------------'", - ], - ["echo -n 'Rolling update started.'"], - ); - $this->start_by_compose_file(); - $this->health_check(); - $this->stop_running_container(); - } - } - private function health_check() - { - if ($this->application->isHealthcheckDisabled()) { - $this->newVersionIsHealthy = true; - return; - } - // ray('New container name: ', $this->container_name); - if ($this->container_name) { - $counter = 1; - $this->execute_remote_command( - [ - "echo 'Waiting for healthcheck to pass on the new container.'" - ] - ); - if ($this->full_healthcheck_url) { - $this->execute_remote_command( - [ - "echo 'Healthcheck URL (inside the container): {$this->full_healthcheck_url}'" - ] - ); - } - while ($counter < $this->application->health_check_retries) { - $this->execute_remote_command( - [ - "docker inspect --format='{{json .State.Health.Status}}' {$this->container_name}", - "hidden" => true, - "save" => "health_check" - ], - - ); - $this->execute_remote_command( - [ - "echo 'Attempt {$counter} of {$this->application->health_check_retries} | Healthcheck status: {$this->saved_outputs->get('health_check')}'" - ], - ); - if (Str::of($this->saved_outputs->get('health_check'))->contains('healthy')) { - $this->newVersionIsHealthy = true; - $this->application->update(['status' => 'running']); - $this->execute_remote_command( - [ - "echo 'New container is healthy.'" - ], - ); - break; - } - $counter++; - sleep($this->application->health_check_interval); - } - } - } - - private function prepare_builder_image() - { - $helperImage = config('coolify.helper_image'); - // Get user home directory - $this->serverUserHomeDir = instant_remote_process(["echo \$HOME"], $this->server); - $this->dockerConfigFileExists = instant_remote_process(["test -f {$this->serverUserHomeDir}/.docker/config.json && echo 'OK' || echo 'NOK'"], $this->server); - - if ($this->dockerConfigFileExists === 'OK') { - $runCommand = "docker run -d --network {$this->destination->network} -v /:/host --name {$this->deployment_uuid} --rm -v {$this->serverUserHomeDir}/.docker/config.json:/root/.docker/config.json:ro -v /var/run/docker.sock:/var/run/docker.sock {$helperImage}"; - } else { - $runCommand = "docker run -d --network {$this->destination->network} -v /:/host --name {$this->deployment_uuid} --rm -v /var/run/docker.sock:/var/run/docker.sock {$helperImage}"; - } - $this->execute_remote_command( - [ - "echo -n 'Preparing container with helper image: $helperImage.'", - ], - [ - $runCommand, - "hidden" => true, - ], - [ - "command" => executeInDocker($this->deployment_uuid, "mkdir -p {$this->basedir}") - ], - ); - } - private function deploy_to_additional_destinations() - { - $destination_ids = collect(str($this->application->additional_destinations)->explode(',')); - foreach ($destination_ids as $destination_id) { - $destination = StandaloneDocker::find($destination_id); - $server = $destination->server; - if ($server->team_id !== $this->mainServer->team_id) { - $this->execute_remote_command( - [ - "echo -n 'Skipping deployment to {$server->name}. Not in the same team?!'", - ], - ); - continue; - } - $this->server = $server; - $this->execute_remote_command( - [ - "echo -n 'Deploying to {$this->server->name}.'", - ], - ); - $this->prepare_builder_image(); - $this->generate_image_names(); - $this->rolling_update(); - } - } - private function set_base_dir() - { - $this->execute_remote_command( - [ - "echo -n 'Setting base directory to {$this->workdir}.'" - ], - ); - } - private function check_git_if_build_needed() - { - $this->generate_git_import_commands(); - $private_key = data_get($this->application, 'private_key.private_key'); - if ($private_key) { - $private_key = base64_encode($private_key); - $this->execute_remote_command( - [ - executeInDocker($this->deployment_uuid, "mkdir -p /root/.ssh") - ], - [ - executeInDocker($this->deployment_uuid, "echo '{$private_key}' | base64 -d > /root/.ssh/id_rsa") - ], - [ - executeInDocker($this->deployment_uuid, "chmod 600 /root/.ssh/id_rsa") - ], - [ - executeInDocker($this->deployment_uuid, "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$this->customPort} -o Port={$this->customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa\" git ls-remote {$this->fullRepoUrl} {$this->branch}"), - "hidden" => true, - "save" => "git_commit_sha" - ], - ); - } else { - $this->execute_remote_command( - [ - executeInDocker($this->deployment_uuid, "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$this->customPort} -o Port={$this->customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null\" git ls-remote {$this->fullRepoUrl} {$this->branch}"), - "hidden" => true, - "save" => "git_commit_sha" - ], - ); - } - - if ($this->saved_outputs->get('git_commit_sha')) { - $this->commit = $this->saved_outputs->get('git_commit_sha')->before("\t"); - } - } - private function clone_repository() - { - $importCommands = $this->generate_git_import_commands(); - $this->execute_remote_command( - [ - "echo '\n----------------------------------------'", - ], - [ - "echo -n 'Importing {$this->customRepository}:{$this->application->git_branch} (commit sha {$this->application->git_commit_sha}) to {$this->basedir}. '" - ], - [ - $importCommands, "hidden" => true - ] - ); - } - - private function generate_git_import_commands() - { - $this->branch = $this->application->git_branch; - $commands = collect([]); - $git_clone_command = "git clone -q -b {$this->application->git_branch}"; - - if ($this->application->deploymentType() === 'source') { - $source_html_url = data_get($this->application, 'source.html_url'); - $url = parse_url(filter_var($source_html_url, FILTER_SANITIZE_URL)); - $source_html_url_host = $url['host']; - $source_html_url_scheme = $url['scheme']; - - if ($this->source->getMorphClass() == 'App\Models\GithubApp') { - if ($this->source->is_public) { - $this->fullRepoUrl = "{$this->source->html_url}/{$this->customRepository}"; - $git_clone_command = "{$git_clone_command} {$this->source->html_url}/{$this->customRepository} {$this->basedir}"; - $git_clone_command = $this->set_git_import_settings($git_clone_command); - - $commands->push(executeInDocker($this->deployment_uuid, $git_clone_command)); - } else { - $github_access_token = generate_github_installation_token($this->source); - $commands->push(executeInDocker($this->deployment_uuid, "git clone -q -b {$this->application->git_branch} $source_html_url_scheme://x-access-token:$github_access_token@$source_html_url_host/{$this->customRepository}.git {$this->basedir}")); - $this->fullRepoUrl = "$source_html_url_scheme://x-access-token:$github_access_token@$source_html_url_host/{$this->customRepository}.git"; - } - return $commands->implode(' && '); - } - } - if ($this->application->deploymentType() === 'deploy_key') { - $this->fullRepoUrl = $this->customRepository; - $private_key = data_get($this->application, 'private_key.private_key'); - if (is_null($private_key)) { - throw new RuntimeException('Private key not found. Please add a private key to the application and try again.'); - } - $private_key = base64_encode($private_key); - $git_clone_command_base = "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$this->customPort} -o Port={$this->customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa\" {$git_clone_command} {$this->customRepository} {$this->basedir}"; - $git_clone_command = $this->set_git_import_settings($git_clone_command_base); - $commands = collect([ - executeInDocker($this->deployment_uuid, "mkdir -p /root/.ssh"), - executeInDocker($this->deployment_uuid, "echo '{$private_key}' | base64 -d > /root/.ssh/id_rsa"), - executeInDocker($this->deployment_uuid, "chmod 600 /root/.ssh/id_rsa"), - ]); - - $commands->push(executeInDocker($this->deployment_uuid, $git_clone_command)); - return $commands->implode(' && '); - } - if ($this->application->deploymentType() === 'other') { - $this->fullRepoUrl = $this->customRepository; - $git_clone_command = "{$git_clone_command} {$this->customRepository} {$this->basedir}"; - $git_clone_command = $this->set_git_import_settings($git_clone_command); - $commands->push(executeInDocker($this->deployment_uuid, $git_clone_command)); - return $commands->implode(' && '); - } - } - - private function set_git_import_settings($git_clone_command) - { - if ($this->application->git_commit_sha !== 'HEAD') { - $git_clone_command = "{$git_clone_command} && cd {$this->basedir} && git -c advice.detachedHead=false checkout {$this->application->git_commit_sha} >/dev/null 2>&1"; - } - if ($this->application->settings->is_git_submodules_enabled) { - $git_clone_command = "{$git_clone_command} && cd {$this->basedir} && git submodule update --init --recursive"; - } - if ($this->application->settings->is_git_lfs_enabled) { - $git_clone_command = "{$git_clone_command} && cd {$this->basedir} && git lfs pull"; - } - return $git_clone_command; - } - - private function cleanup_git() - { - $this->execute_remote_command( - [executeInDocker($this->deployment_uuid, "rm -fr {$this->basedir}/.git")], - ); - } - - private function generate_nixpacks_confs() - { - $nixpacks_command = $this->nixpacks_build_cmd(); - $this->execute_remote_command( - [ - "echo -n 'Generating nixpacks configuration with: $nixpacks_command'", - ], - [executeInDocker($this->deployment_uuid, $nixpacks_command)], - [executeInDocker($this->deployment_uuid, "cp {$this->workdir}/.nixpacks/Dockerfile {$this->workdir}/Dockerfile")], - [executeInDocker($this->deployment_uuid, "rm -f {$this->workdir}/.nixpacks/Dockerfile")] - ); - } - - private function nixpacks_build_cmd() - { - $this->generate_env_variables(); - $nixpacks_command = "nixpacks build --cache-key '{$this->application->uuid}' -o {$this->workdir} {$this->env_args} --no-error-without-start"; - if ($this->application->build_command) { - $nixpacks_command .= " --build-cmd \"{$this->application->build_command}\""; - } - if ($this->application->start_command) { - $nixpacks_command .= " --start-cmd \"{$this->application->start_command}\""; - } - if ($this->application->install_command) { - $nixpacks_command .= " --install-cmd \"{$this->application->install_command}\""; - } - $nixpacks_command .= " {$this->workdir}"; - return $nixpacks_command; - } - - private function generate_env_variables() - { - $this->env_args = collect([]); - foreach ($this->application->nixpacks_environment_variables_preview as $env) { - $this->env_args->push("--env {$env->key}={$env->value}"); - } - $this->env_args = $this->env_args->implode(' '); - } - - private function generate_compose_file() - { - $ports = $this->application->settings->is_static ? [80] : $this->application->ports_exposes_array; - - $persistent_storages = $this->generate_local_persistent_volumes(); - $volume_names = $this->generate_local_persistent_volumes_only_volume_names(); - $environment_variables = $this->generate_environment_variables($ports); - - if (data_get($this->application, 'custom_labels')) { - $labels = collect(str($this->application->custom_labels)->explode(',')); - $labels = $labels->filter(function ($value, $key) { - return !Str::startsWith($value, 'coolify.'); - }); - $this->application->custom_labels = $labels->implode(','); - $this->application->save(); - } else { - $labels = collect(generateLabelsApplication($this->application, $this->preview)); - } - - $labels = $labels->merge(defaultLabels($this->application->id, $this->application->uuid, 0))->toArray(); - $docker_compose = [ - 'version' => '3.8', - 'services' => [ - $this->container_name => [ - 'image' => $this->production_image_name, - 'container_name' => $this->container_name, - 'restart' => RESTART_MODE, - 'environment' => $environment_variables, - 'labels' => $labels, - 'expose' => $ports, - 'networks' => [ - $this->destination->network, - ], - 'healthcheck' => [ - 'test' => [ - 'CMD-SHELL', - $this->generate_healthcheck_commands() - ], - 'interval' => $this->application->health_check_interval . 's', - 'timeout' => $this->application->health_check_timeout . 's', - 'retries' => $this->application->health_check_retries, - 'start_period' => $this->application->health_check_start_period . 's' - ], - 'mem_limit' => $this->application->limits_memory, - 'memswap_limit' => $this->application->limits_memory_swap, - 'mem_swappiness' => $this->application->limits_memory_swappiness, - 'mem_reservation' => $this->application->limits_memory_reservation, - 'cpus' => (int) $this->application->limits_cpus, - 'cpuset' => $this->application->limits_cpuset, - 'cpu_shares' => $this->application->limits_cpu_shares, - ] - ], - 'networks' => [ - $this->destination->network => [ - 'external' => true, - 'name' => $this->destination->network, - 'attachable' => true - ] - ] - ]; - if ($this->server->isLogDrainEnabled() && $this->application->isLogDrainEnabled()) { - $docker_compose['services'][$this->container_name]['logging'] = [ - 'driver' => 'fluentd', - 'options' => [ - 'fluentd-address' => "tcp://127.0.0.1:24224", - 'fluentd-async' => "true", - 'fluentd-sub-second-precision' => "true", - ] - ]; - } - if ($this->application->settings->is_gpu_enabled) { - ray('asd'); - $docker_compose['services'][$this->container_name]['deploy']['resources']['reservations']['devices'] = [ - [ - 'driver' => data_get($this->application, 'settings.gpu_driver', 'nvidia'), - 'capabilities' => ['gpu'], - 'options' => data_get($this->application, 'settings.gpu_options', []) - ] - ]; - if (data_get($this->application, 'settings.gpu_count')) { - $count = data_get($this->application, 'settings.gpu_count'); - if ($count === 'all') { - $docker_compose['services'][$this->container_name]['deploy']['resources']['reservations']['devices'][0]['count'] = $count; - } else { - $docker_compose['services'][$this->container_name]['deploy']['resources']['reservations']['devices'][0]['count'] = (int) $count; - } - } else if (data_get($this->application, 'settings.gpu_device_ids')) { - $docker_compose['services'][$this->container_name]['deploy']['resources']['reservations']['devices'][0]['ids'] = data_get($this->application, 'settings.gpu_device_ids'); - } - } - if ($this->application->isHealthcheckDisabled()) { - data_forget($docker_compose, 'services.' . $this->container_name . '.healthcheck'); - } - if (count($persistent_storages) > 0) { - $docker_compose['services'][$this->container_name]['volumes'] = $persistent_storages; - } - if (count($volume_names) > 0) { - $docker_compose['volumes'] = $volume_names; - } - // if ($this->build_pack === 'dockerfile') { - // $docker_compose['services'][$this->container_name]['build'] = [ - // 'context' => $this->workdir, - // 'dockerfile' => $this->workdir . $this->dockerfile_location, - // ]; - // } - $this->docker_compose = Yaml::dump($docker_compose, 10); - $this->docker_compose_base64 = base64_encode($this->docker_compose); - $this->execute_remote_command([executeInDocker($this->deployment_uuid, "echo '{$this->docker_compose_base64}' | base64 -d > {$this->workdir}/docker-compose.yml"), "hidden" => true]); - } - - private function generate_local_persistent_volumes() - { - $local_persistent_volumes = []; - foreach ($this->application->persistentStorages as $persistentStorage) { - $volume_name = $persistentStorage->host_path ?? $persistentStorage->name; - $local_persistent_volumes[] = $volume_name . ':' . $persistentStorage->mount_path; - } - return $local_persistent_volumes; - } - - private function generate_local_persistent_volumes_only_volume_names() - { - $local_persistent_volumes_names = []; - foreach ($this->application->persistentStorages as $persistentStorage) { - if ($persistentStorage->host_path) { - continue; - } - $name = $persistentStorage->name; - $local_persistent_volumes_names[$name] = [ - 'name' => $name, - 'external' => false, - ]; - } - return $local_persistent_volumes_names; - } - - private function generate_environment_variables($ports) - { - $environment_variables = collect(); - foreach ($this->application->runtime_environment_variables_preview as $env) { - $environment_variables->push("$env->key=$env->value"); - } - foreach ($this->application->nixpacks_environment_variables_preview as $env) { - $environment_variables->push("$env->key=$env->value"); - } - // Add PORT if not exists, use the first port as default - if ($environment_variables->filter(fn ($env) => Str::of($env)->contains('PORT'))->isEmpty()) { - $environment_variables->push("PORT={$ports[0]}"); - } - return $environment_variables->all(); - } - - private function generate_healthcheck_commands() - { - if ($this->application->dockerfile || $this->application->build_pack === 'dockerfile' || $this->application->build_pack === 'dockerimage') { - // TODO: disabled HC because there are several ways to hc a simple docker image, hard to figure out a good way. Like some docker images (pocketbase) does not have curl. - return 'exit 0'; - } - if (!$this->application->health_check_port) { - $health_check_port = $this->application->ports_exposes_array[0]; - } else { - $health_check_port = $this->application->health_check_port; - } - if ($this->application->health_check_path) { - $this->full_healthcheck_url = "{$this->application->health_check_method}: {$this->application->health_check_scheme}://{$this->application->health_check_host}:{$health_check_port}{$this->application->health_check_path}"; - $generated_healthchecks_commands = [ - "curl -s -X {$this->application->health_check_method} -f {$this->application->health_check_scheme}://{$this->application->health_check_host}:{$health_check_port}{$this->application->health_check_path} > /dev/null" - ]; - } else { - $this->full_healthcheck_url = "{$this->application->health_check_method}: {$this->application->health_check_scheme}://{$this->application->health_check_host}:{$health_check_port}/"; - $generated_healthchecks_commands = [ - "curl -s -X {$this->application->health_check_method} -f {$this->application->health_check_scheme}://{$this->application->health_check_host}:{$health_check_port}/" - ]; - } - return implode(' ', $generated_healthchecks_commands); - } - private function pull_latest_image($image) - { - $this->execute_remote_command( - ["echo -n 'Pulling latest image ($image) from the registry.'"], - - [ - executeInDocker($this->deployment_uuid, "docker pull {$image}"), "hidden" => true - ] - ); - } - private function build_image() - { - if ($this->application->build_pack === 'static') { - $this->execute_remote_command([ - "echo -n 'Static deployment. Copying static assets to the image.'", - ]); - } else { - $this->execute_remote_command( - [ - "echo -n 'Building docker image started.'", - ], - ["echo -n 'To check the current progress, click on Show Debug Logs.'"] - ); - } - - if ($this->application->settings->is_static || $this->application->build_pack === 'static') { - if ($this->application->static_image) { - $this->pull_latest_image($this->application->static_image); - } - if ($this->application->build_pack === 'static') { - $dockerfile = base64_encode("FROM {$this->application->static_image} -WORKDIR /usr/share/nginx/html/ -LABEL coolify.deploymentId={$this->deployment_uuid} -COPY . . -RUN rm -f /usr/share/nginx/html/nginx.conf -RUN rm -f /usr/share/nginx/html/Dockerfile -COPY ./nginx.conf /etc/nginx/conf.d/default.conf"); - $nginx_config = base64_encode("server { - listen 80; - listen [::]:80; - server_name localhost; - - location / { - root /usr/share/nginx/html; - index index.html; - try_files \$uri \$uri.html \$uri/index.html \$uri/ /index.html =404; - } - - error_page 500 502 503 504 /50x.html; - location = /50x.html { - root /usr/share/nginx/html; - } - }"); - } else { - $this->execute_remote_command([ - executeInDocker($this->deployment_uuid, "docker build $this->buildTarget $this->addHosts --network host -f {$this->workdir}{$this->dockerfile_location} {$this->build_args} --progress plain -t $this->build_image_name {$this->workdir}"), "hidden" => true - ]); - - $dockerfile = base64_encode("FROM {$this->application->static_image} -WORKDIR /usr/share/nginx/html/ -LABEL coolify.deploymentId={$this->deployment_uuid} -COPY --from=$this->build_image_name /app/{$this->application->publish_directory} . -COPY ./nginx.conf /etc/nginx/conf.d/default.conf"); - - $nginx_config = base64_encode("server { - listen 80; - listen [::]:80; - server_name localhost; - - location / { - root /usr/share/nginx/html; - index index.html; - try_files \$uri \$uri.html \$uri/index.html \$uri/ /index.html =404; - } - - error_page 500 502 503 504 /50x.html; - location = /50x.html { - root /usr/share/nginx/html; - } - }"); - } - $this->execute_remote_command( - [ - executeInDocker($this->deployment_uuid, "echo '{$dockerfile}' | base64 -d > {$this->workdir}/Dockerfile") - ], - [ - executeInDocker($this->deployment_uuid, "echo '{$nginx_config}' | base64 -d > {$this->workdir}/nginx.conf") - ], - [ - executeInDocker($this->deployment_uuid, "docker build $this->addHosts --network host -f {$this->workdir}/Dockerfile {$this->build_args} --progress plain -t $this->production_image_name {$this->workdir}"), "hidden" => true - ] - ); - } else { - // Pure Dockerfile based deployment - if ($this->application->dockerfile) { - $this->execute_remote_command([ - executeInDocker($this->deployment_uuid, "docker build --pull $this->buildTarget $this->addHosts --network host -f {$this->workdir}{$this->dockerfile_location} {$this->build_args} --progress plain -t $this->production_image_name {$this->workdir}"), "hidden" => true - ]); - } else { - $this->execute_remote_command([ - executeInDocker($this->deployment_uuid, "docker build $this->buildTarget $this->addHosts --network host -f {$this->workdir}{$this->dockerfile_location} {$this->build_args} --progress plain -t $this->production_image_name {$this->workdir}"), "hidden" => true - ]); - } - } - $this->execute_remote_command([ - "echo -n 'Building docker image completed.'", - ]); - } - - private function stop_running_container(bool $force = false) - { - $this->execute_remote_command(["echo -n 'Removing old container.'"]); - if ($this->newVersionIsHealthy || $force) { - $containers = getCurrentApplicationContainerStatus($this->server, $this->application->id, 0); - $containers = $containers->filter(function ($container) { - return data_get($container, 'Names') !== $this->container_name; - }); - $containers->each(function ($container) { - $containerName = data_get($container, 'Names'); - $this->execute_remote_command( - [executeInDocker($this->deployment_uuid, "docker rm -f $containerName >/dev/null 2>&1"), "hidden" => true, "ignore_errors" => true], - ); - }); - $this->execute_remote_command( - [ - "echo 'Rolling update completed.'" - ], - ); - } else { - $this->execute_remote_command( - ["echo -n 'New container is not healthy, rolling back to the old container.'"], - [executeInDocker($this->deployment_uuid, "docker rm -f $this->container_name >/dev/null 2>&1"), "hidden" => true, "ignore_errors" => true], - ); - } - } - - private function start_by_compose_file() - { - if ($this->application->build_pack === 'dockerimage') { - $this->execute_remote_command( - ["echo -n 'Pulling latest images from the registry.'"], - [executeInDocker($this->deployment_uuid, "docker compose --project-directory {$this->workdir} pull"), "hidden" => true], - [executeInDocker($this->deployment_uuid, "docker compose --project-directory {$this->workdir} up --build -d"), "hidden" => true], - ); - } else { - $this->execute_remote_command( - [executeInDocker($this->deployment_uuid, "docker compose --project-directory {$this->workdir} up --build -d"), "hidden" => true], - ); - } - } - - private function generate_build_env_variables() - { - $this->build_args = collect(["--build-arg SOURCE_COMMIT=\"{$this->commit}\""]); - foreach ($this->application->build_environment_variables_preview as $env) { - $this->build_args->push("--build-arg {$env->key}=\"{$env->value}\""); - } - $this->build_args = $this->build_args->implode(' '); - } - - private function add_build_env_variables_to_dockerfile() - { - $this->execute_remote_command([ - executeInDocker($this->deployment_uuid, "cat {$this->workdir}{$this->dockerfile_location}"), "hidden" => true, "save" => 'dockerfile' - ]); - $dockerfile = collect(Str::of($this->saved_outputs->get('dockerfile'))->trim()->explode("\n")); - - foreach ($this->application->build_environment_variables as $env) { - $dockerfile->splice(1, 0, "ARG {$env->key}={$env->value}"); - } - $dockerfile_base64 = base64_encode($dockerfile->implode("\n")); - $this->execute_remote_command([ - executeInDocker($this->deployment_uuid, "echo '{$dockerfile_base64}' | base64 -d > {$this->workdir}{$this->dockerfile_location}"), - "hidden" => true - ]); - } - - private function next(string $status) - { - // If the deployment is cancelled by the user, don't update the status - if ($this->application_deployment_queue->status !== ApplicationDeploymentStatus::CANCELLED_BY_USER->value) { - $this->application_deployment_queue->update([ - 'status' => $status, - ]); - } - queue_next_deployment($this->application); - if ($status === ApplicationDeploymentStatus::FINISHED->value) { - $this->application->environment->project->team->notify(new DeploymentSuccess($this->application, $this->deployment_uuid, $this->preview)); - } - if ($status === ApplicationDeploymentStatus::FAILED->value) { - $this->application->environment->project->team->notify(new DeploymentFailed($this->application, $this->deployment_uuid, $this->preview)); - } - } - - public function failed(Throwable $exception): void - { - $this->execute_remote_command( - ["echo 'Oops something is not okay, are you okay? 😢'", 'type' => 'err'], - ["echo '{$exception->getMessage()}'", 'type' => 'err'], - ["echo -n 'Deployment failed. Removing the new version of your application.'", 'type' => 'err'], - [executeInDocker($this->deployment_uuid, "docker rm -f $this->container_name >/dev/null 2>&1"), "hidden" => true, "ignore_errors" => true] - ); - - $this->next(ApplicationDeploymentStatus::FAILED->value); - } -} diff --git a/app/Models/Application.php b/app/Models/Application.php index 2e58d1891..a7c721800 100644 --- a/app/Models/Application.php +++ b/app/Models/Application.php @@ -138,7 +138,22 @@ class Application extends BaseModel return Attribute::make( set: function ($value) { if (is_null($value) || $value === '') { - return '/docker-compose.yml'; + return '/docker-compose.yaml'; + } else { + if ($value !== '/') { + return Str::start(Str::replaceEnd('/', '', $value), '/'); + } + return Str::start($value, '/'); + } + } + ); + } + public function dockerComposePrLocation(): Attribute + { + return Attribute::make( + set: function ($value) { + if (is_null($value) || $value === '') { + return '/docker-compose-pr.yaml'; } else { if ($value !== '/') { return Str::start(Str::replaceEnd('/', '', $value), '/'); @@ -595,6 +610,7 @@ class Application extends BaseModel function loadComposeFile($isInit = false) { $initialDockerComposeLocation = $this->docker_compose_location; + $initialDockerComposePrLocation = $this->docker_compose_pr_location; if ($this->build_pack === 'dockercompose') { if ($isInit && $this->docker_compose_raw) { return; @@ -603,11 +619,12 @@ class Application extends BaseModel ['commands' => $cloneCommand] = $this->generateGitImportCommands(deployment_uuid: $uuid, only_checkout: true, exec_in_docker: false, custom_base_dir: '.'); $workdir = rtrim($this->base_directory, '/'); $composeFile = $this->docker_compose_location; + $prComposeFile = $this->docker_compose_pr_location; $commands = collect([ "mkdir -p /tmp/{$uuid} && cd /tmp/{$uuid}", $cloneCommand, "git sparse-checkout init --cone", - "git sparse-checkout set .$workdir$composeFile", + "git sparse-checkout set .$workdir$composeFile .$workdir$prComposeFile", "git read-tree -mu HEAD", "cat .$workdir$composeFile", ]); @@ -620,13 +637,26 @@ class Application extends BaseModel $this->docker_compose_raw = $composeFileContent; $this->save(); } + $commands = collect([ + "cat .$workdir$prComposeFile", + ]); + $composePrFileContent = instant_remote_process($commands, $this->destination->server, false); + if (!$composePrFileContent) { + $this->docker_compose_pr_location = $initialDockerComposePrLocation; + $this->save(); + throw new \Exception("Could not load compose file from $workdir$prComposeFile"); + } else { + $this->docker_compose_pr_raw = $composePrFileContent; + $this->save(); + } $commands = collect([ "rm -rf /tmp/{$uuid}", ]); instant_remote_process($commands, $this->destination->server, false); return [ 'parsedServices' => $this->parseCompose(), - 'initialDockerComposeLocation' => $this->docker_compose_location + 'initialDockerComposeLocation' => $this->docker_compose_location, + 'initialDockerComposePrLocation' => $this->docker_compose_pr_location, ]; } } diff --git a/bootstrap/helpers/applications.php b/bootstrap/helpers/applications.php index 17c3aa486..bc0de5c47 100644 --- a/bootstrap/helpers/applications.php +++ b/bootstrap/helpers/applications.php @@ -36,10 +36,6 @@ function queue_application_deployment(int $application_id, string $deployment_uu if ($running_deployments->count() > 0) { return; } - // New deployment - // dispatchDeploymentJob($deployment); - - // Old deployment dispatch(new ApplicationDeploymentJob( application_deployment_queue_id: $deployment->id, ))->onConnection('long-running')->onQueue('long-running'); @@ -50,39 +46,11 @@ function queue_next_deployment(Application $application) { $next_found = ApplicationDeploymentQueue::where('application_id', $application->id)->where('status', 'queued')->first(); if ($next_found) { - // New deployment - // dispatchDeploymentJob($next_found->id); - - // Old deployment dispatch(new ApplicationDeploymentJob( application_deployment_queue_id: $next_found->id, ))->onConnection('long-running')->onQueue('long-running'); } } -function dispatchDeploymentJob(ApplicationDeploymentQueue $deploymentQueueEntry) -{ - $application = Application::find($deploymentQueueEntry->application_id); - - $isRestartOnly = data_get($deploymentQueueEntry, 'restart_only'); - $isSimpleDockerFile = data_get($application, 'dockerfile'); - $isDockerImage = data_get($application, 'build_pack') === 'dockerimage'; - - // if ($isRestartOnly) { - // ApplicationRestartJob::dispatch(queue: $deploymentQueueEntry, application: $application)->onConnection('long-running')->onQueue('long-running'); - // } else if ($isSimpleDockerFile) { - // ApplicationDeploySimpleDockerfileJob::dispatch(applicationDeploymentQueueId: $id)->onConnection('long-running')->onQueue('long-running'); - // } else - - if ($isDockerImage) { - ApplicationDeployDockerImageJob::dispatch( - deploymentQueueEntry: $deploymentQueueEntry, - application: $application - )->onConnection('long-running')->onQueue('long-running'); - } else { - throw new Exception('Unknown build pack'); - } -} - // Deployment things function generateHostIpMapping(Server $server, string $network) { diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php index ecb67cd0c..fcf32d215 100644 --- a/bootstrap/helpers/shared.php +++ b/bootstrap/helpers/shared.php @@ -1089,11 +1089,16 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal return collect([]); } } else if ($resource->getMorphClass() === 'App\Models\Application') { - try { - $yaml = Yaml::parse($resource->docker_compose_raw); - } catch (\Exception $e) { - throw new \Exception($e->getMessage()); + if ($pull_request_id !== 0 && $resource->dockerComposePrLocation() !== $resource->dockerComposeLocation()) { + + } else { + try { + $yaml = Yaml::parse($resource->docker_compose_raw); + } catch (\Exception $e) { + throw new \Exception($e->getMessage()); + } } + $server = $resource->destination->server; $topLevelVolumes = collect(data_get($yaml, 'volumes', [])); if ($pull_request_id !== 0) { diff --git a/database/migrations/2023_11_24_080341_add_docker_compose_location.php b/database/migrations/2023_11_24_080341_add_docker_compose_location.php index b811aa4d1..eab705ff0 100644 --- a/database/migrations/2023_11_24_080341_add_docker_compose_location.php +++ b/database/migrations/2023_11_24_080341_add_docker_compose_location.php @@ -12,11 +12,15 @@ return new class extends Migration public function up(): void { Schema::table('applications', function (Blueprint $table) { - $table->string('docker_compose_location')->nullable()->default('/docker-compose.yml')->after('dockerfile_location'); - $table->longText('docker_compose')->nullable()->after('docker_compose_location'); - $table->longText('docker_compose_raw')->nullable()->after('docker_compose'); - $table->text('docker_compose_domains')->nullable()->after('docker_compose_raw'); + $table->string('docker_compose_location')->nullable()->default('/docker-compose.yaml')->after('dockerfile_location'); + $table->string('docker_compose_pr_location')->nullable()->default('/docker-compose-pr.yaml')->after('docker_compose_location'); + $table->longText('docker_compose')->nullable()->after('docker_compose_location'); + $table->longText('docker_compose_pr')->nullable()->after('docker_compose_location'); + $table->longText('docker_compose_raw')->nullable()->after('docker_compose'); + $table->longText('docker_compose_pr_raw')->nullable()->after('docker_compose'); + + $table->text('docker_compose_domains')->nullable()->after('docker_compose_raw'); }); } @@ -27,8 +31,11 @@ return new class extends Migration { Schema::table('applications', function (Blueprint $table) { $table->dropColumn('docker_compose_location'); + $table->dropColumn('docker_compose_pr_location'); $table->dropColumn('docker_compose'); + $table->dropColumn('docker_compose_pr'); $table->dropColumn('docker_compose_raw'); + $table->dropColumn('docker_compose_pr_raw'); $table->dropColumn('docker_compose_domains'); }); } diff --git a/resources/views/livewire/project/application/general.blade.php b/resources/views/livewire/project/application/general.blade.php index 0975dbf4d..b08a9fb60 100644 --- a/resources/views/livewire/project/application/general.blade.php +++ b/resources/views/livewire/project/application/general.blade.php @@ -116,9 +116,12 @@ @endif @if ($application->build_pack === 'dockercompose') - + @endif @if ($application->build_pack === 'dockerfile') Date: Tue, 28 Nov 2023 10:11:53 +0100 Subject: [PATCH 30/46] Fix docker compose PR location default value --- .../Livewire/Project/Application/General.php | 7 +++- app/Models/Application.php | 41 +++++++++++++------ bootstrap/helpers/shared.php | 34 +++++++++++---- ..._24_080341_add_docker_compose_location.php | 2 +- .../project/application/general.blade.php | 7 ++-- 5 files changed, 65 insertions(+), 26 deletions(-) diff --git a/app/Http/Livewire/Project/Application/General.php b/app/Http/Livewire/Project/Application/General.php index da642ba77..2917fa438 100644 --- a/app/Http/Livewire/Project/Application/General.php +++ b/app/Http/Livewire/Project/Application/General.php @@ -60,7 +60,9 @@ class General extends Component 'application.docker_compose_location' => 'nullable', 'application.docker_compose_pr_location' => 'nullable', 'application.docker_compose' => 'nullable', + 'application.docker_compose_pr' => 'nullable', 'application.docker_compose_raw' => 'nullable', + 'application.docker_compose_pr_raw' => 'nullable', 'application.custom_labels' => 'nullable', 'application.dockerfile_target_build' => 'nullable', 'application.settings.is_static' => 'boolean|required', @@ -88,7 +90,9 @@ class General extends Component 'application.docker_compose_location' => 'Docker compose location', 'application.docker_compose_pr_location' => 'Docker compose location', 'application.docker_compose' => 'Docker compose', + 'application.docker_compose_pr' => 'Docker compose', 'application.docker_compose_raw' => 'Docker compose raw', + 'application.docker_compose_pr_raw' => 'Docker compose raw', 'application.custom_labels' => 'Custom labels', 'application.dockerfile_target_build' => 'Dockerfile target build', 'application.settings.is_static' => 'Is static', @@ -98,7 +102,6 @@ class General extends Component { try { $this->parsedServices = $this->application->parseCompose(); - ray($this->parsedServices); } catch (\Throwable $e) { $this->emit('error', $e->getMessage()); } @@ -196,7 +199,7 @@ class General extends Component public function submit($showToaster = true) { try { - if ($this->initialDockerComposeLocation !== $this->application->docker_compose_location) { + if ($this->initialDockerComposeLocation !== $this->application->docker_compose_location || $this->initialDockerComposePrLocation !== $this->application->docker_compose_pr_location) { $this->loadComposeFile(); } $this->validate(); diff --git a/app/Models/Application.php b/app/Models/Application.php index a7c721800..ea7e0a930 100644 --- a/app/Models/Application.php +++ b/app/Models/Application.php @@ -153,7 +153,7 @@ class Application extends BaseModel return Attribute::make( set: function ($value) { if (is_null($value) || $value === '') { - return '/docker-compose-pr.yaml'; + return '/docker-compose.yaml'; } else { if ($value !== '/') { return Str::start(Str::replaceEnd('/', '', $value), '/'); @@ -602,7 +602,11 @@ class Application extends BaseModel function parseCompose(int $pull_request_id = 0) { if ($this->docker_compose_raw) { - return parseDockerComposeFile(resource: $this, isNew: false, pull_request_id: $pull_request_id); + $mainCompose = parseDockerComposeFile(resource: $this, isNew: false, pull_request_id: $pull_request_id); + if ($this->getMorphClass() === 'App\Models\Application' && $this->docker_compose_pr_raw) { + parseDockerComposeFile(resource: $this, isNew: false, pull_request_id: $pull_request_id, is_pr: true); + } + return $mainCompose; } else { return collect([]); } @@ -620,11 +624,15 @@ class Application extends BaseModel $workdir = rtrim($this->base_directory, '/'); $composeFile = $this->docker_compose_location; $prComposeFile = $this->docker_compose_pr_location; + $fileList = collect([".$composeFile"]); + if ($composeFile !== $prComposeFile) { + $fileList->push(".$prComposeFile"); + } $commands = collect([ "mkdir -p /tmp/{$uuid} && cd /tmp/{$uuid}", $cloneCommand, "git sparse-checkout init --cone", - "git sparse-checkout set .$workdir$composeFile .$workdir$prComposeFile", + "git sparse-checkout set {$fileList->implode(' ')}", "git read-tree -mu HEAD", "cat .$workdir$composeFile", ]); @@ -632,23 +640,30 @@ class Application extends BaseModel if (!$composeFileContent) { $this->docker_compose_location = $initialDockerComposeLocation; $this->save(); - throw new \Exception("Could not load compose file from $workdir$composeFile"); + throw new \Exception("Could not load base compose file from $workdir$composeFile"); } else { $this->docker_compose_raw = $composeFileContent; $this->save(); } - $commands = collect([ - "cat .$workdir$prComposeFile", - ]); - $composePrFileContent = instant_remote_process($commands, $this->destination->server, false); - if (!$composePrFileContent) { - $this->docker_compose_pr_location = $initialDockerComposePrLocation; + if ($composeFile === $prComposeFile) { + $this->docker_compose_pr_raw = $composeFileContent; $this->save(); - throw new \Exception("Could not load compose file from $workdir$prComposeFile"); } else { - $this->docker_compose_pr_raw = $composePrFileContent; - $this->save(); + $commands = collect([ + "cd /tmp/{$uuid}", + "cat .$workdir$prComposeFile", + ]); + $composePrFileContent = instant_remote_process($commands, $this->destination->server, false); + if (!$composePrFileContent) { + $this->docker_compose_pr_location = $initialDockerComposePrLocation; + $this->save(); + throw new \Exception("Could not load compose file from $workdir$prComposeFile"); + } else { + $this->docker_compose_pr_raw = $composePrFileContent; + $this->save(); + } } + $commands = collect([ "rm -rf /tmp/{$uuid}", ]); diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php index fcf32d215..2b2f32d33 100644 --- a/bootstrap/helpers/shared.php +++ b/bootstrap/helpers/shared.php @@ -579,7 +579,7 @@ function getTopLevelNetworks(Service|Application $resource) return $topLevelNetworks->keys(); } } -function parseDockerComposeFile(Service|Application $resource, bool $isNew = false, int $pull_request_id) +function parseDockerComposeFile(Service|Application $resource, bool $isNew = false, int $pull_request_id, bool $is_pr = false) { // ray()->clearAll(); if ($resource->getMorphClass() === 'App\Models\Service') { @@ -1089,8 +1089,17 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal return collect([]); } } else if ($resource->getMorphClass() === 'App\Models\Application') { - if ($pull_request_id !== 0 && $resource->dockerComposePrLocation() !== $resource->dockerComposeLocation()) { - + $isSameDockerComposeFile = false; + if ($resource->dockerComposePrLocation() === $resource->dockerComposeLocation()) { + $isSameDockerComposeFile = true; + $is_pr = false; + } + if ($is_pr) { + try { + $yaml = Yaml::parse($resource->docker_compose_pr_raw); + } catch (\Exception $e) { + throw new \Exception($e->getMessage()); + } } else { try { $yaml = Yaml::parse($resource->docker_compose_raw); @@ -1098,7 +1107,7 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal throw new \Exception($e->getMessage()); } } - + ray($yaml); $server = $resource->destination->server; $topLevelVolumes = collect(data_get($yaml, 'volumes', [])); if ($pull_request_id !== 0) { @@ -1172,7 +1181,6 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal data_set($service, 'volumes', $serviceVolumes->toArray()); } } else { - } // Decide if the service is a database $isDatabase = isDatabaseImage(data_get_str($service, 'image')); @@ -1469,8 +1477,20 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal 'volumes' => $topLevelVolumes->toArray(), 'networks' => $topLevelNetworks->toArray(), ]; - $resource->docker_compose_raw = Yaml::dump($yaml, 10, 2); - $resource->docker_compose = Yaml::dump($finalServices, 10, 2); + if ($isSameDockerComposeFile) { + $resource->docker_compose_pr_raw = Yaml::dump($yaml, 10, 2); + $resource->docker_compose_pr = Yaml::dump($finalServices, 10, 2); + $resource->docker_compose_raw = Yaml::dump($yaml, 10, 2); + $resource->docker_compose = Yaml::dump($finalServices, 10, 2); + } else { + if ($is_pr) { + $resource->docker_compose_pr_raw = Yaml::dump($yaml, 10, 2); + $resource->docker_compose_pr = Yaml::dump($finalServices, 10, 2); + } else { + $resource->docker_compose_raw = Yaml::dump($yaml, 10, 2); + $resource->docker_compose = Yaml::dump($finalServices, 10, 2); + } + } $resource->save(); return collect($finalServices); } diff --git a/database/migrations/2023_11_24_080341_add_docker_compose_location.php b/database/migrations/2023_11_24_080341_add_docker_compose_location.php index eab705ff0..8508181a3 100644 --- a/database/migrations/2023_11_24_080341_add_docker_compose_location.php +++ b/database/migrations/2023_11_24_080341_add_docker_compose_location.php @@ -13,7 +13,7 @@ return new class extends Migration { Schema::table('applications', function (Blueprint $table) { $table->string('docker_compose_location')->nullable()->default('/docker-compose.yaml')->after('dockerfile_location'); - $table->string('docker_compose_pr_location')->nullable()->default('/docker-compose-pr.yaml')->after('docker_compose_location'); + $table->string('docker_compose_pr_location')->nullable()->default('/docker-compose.yaml')->after('docker_compose_location'); $table->longText('docker_compose')->nullable()->after('docker_compose_location'); $table->longText('docker_compose_pr')->nullable()->after('docker_compose_location'); diff --git a/resources/views/livewire/project/application/general.blade.php b/resources/views/livewire/project/application/general.blade.php index b08a9fb60..daeac9710 100644 --- a/resources/views/livewire/project/application/general.blade.php +++ b/resources/views/livewire/project/application/general.blade.php @@ -119,9 +119,9 @@ - + helper="It is calculated together with the Base Directory:
    {{ Str::start($application->base_directory . $application->docker_compose_pr_location, '/') }}" /> --}} @endif @if ($application->build_pack === 'dockerfile') build_pack === 'dockercompose') Reload Compose File - + {{-- --}} @endif @if ($application->dockerfile) From 3e2e1080f516d0e9326389810b580e58afbbfc52 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 28 Nov 2023 10:46:00 +0100 Subject: [PATCH 31/46] nothing to see here --- scripts/install.sh | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index 26ada577f..cad2599e9 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -1,11 +1,7 @@ #!/bin/bash ## Do not modify this file. You will lose the ability to install and auto-update! -########### -## Always run "php artisan app:sync-to-bunny-cdn --env=secrets" or "scripts/run sync-bunny" if you update this file. -########### - -VERSION="1.0.3" +VERSION="1.0.4" DOCKER_VERSION="24.0" CDN="https://cdn.coollabs.io/coolify" @@ -18,6 +14,28 @@ if [ $EUID != 0 ]; then echo "Please run as root" exit fi +# case "$OS_TYPE" in +# ubuntu | debian | raspbian) +# echo "Installing dependencies with APT..." +# apt update -y >/dev/null 2>&1 +# apt install -y curl wget git jq >/dev/null 2>&1 +# ;; +# centos | fedora | rhel | ol | rocky) +# echo "Installing dependencies with DNF..." +# dnf update -y >/dev/null 2>&1 +# dnf install -y curl wget git jq >/dev/null 2>&1 +# ;; +# sles | opensuse-leap | opensuse-tumbleweed) +# echo "Installing dependencies with Zypper..." +# zypper refresh >/dev/null 2>&1 +# zypper install -y curl wget git jq >/dev/null 2>&1 +# ;; +# *) +# echo "This script only supports Debian, Redhat or Sles based operating systems for now." +# exit +# ;; +# esac + if [ $OS_TYPE != "ubuntu" ] && [ $OS_TYPE != "debian" ] && [ $OS_TYPE != "raspbian" ]; then echo "This script only supports Ubuntu and Debian for now." exit @@ -41,7 +59,7 @@ echo -e "-------------" echo "Installing required packages..." apt update -y >/dev/null 2>&1 -apt install -y curl wget git jq jc >/dev/null 2>&1 +apt install -y curl wget git jq >/dev/null 2>&1 if ! [ -x "$(command -v docker)" ]; then echo "Docker is not installed. Installing Docker..." @@ -53,7 +71,7 @@ if ! [ -x "$(command -v docker)" ]; then echo "Maybe your OS is not supported." echo "Please visit https://docs.docker.com/engine/install/ and install Docker manually to continue." exit 1 - fi + fi fi echo -e "-------------" echo -e "Check Docker Configuration..." @@ -93,7 +111,6 @@ else systemctl restart docker fi - echo -e "-------------" mkdir -p /data/coolify/ssh/keys From 4c0623f022da4e91df28acfc753fe6283472a975 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 28 Nov 2023 10:54:46 +0100 Subject: [PATCH 32/46] Refactor server delete view to display defined resources as links --- resources/views/livewire/server/delete.blade.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/resources/views/livewire/server/delete.blade.php b/resources/views/livewire/server/delete.blade.php index b568306ad..02f281e3b 100644 --- a/resources/views/livewire/server/delete.blade.php +++ b/resources/views/livewire/server/delete.blade.php @@ -32,15 +32,17 @@ @empty @endforelse @else +
    @forelse ($server->definedResources() as $resource) @if ($loop->first)

    Defined resources

    @endif -
    +
    {{ str($resource->type())->headline() }}
    -     {{ $resource->name }} -
    +
    {{ $resource->name }}
    + @empty @endforelse +
    @endif
    From 636995d0e4119444ef6b7659b3facdbe8130dc01 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 28 Nov 2023 10:55:24 +0100 Subject: [PATCH 33/46] Refactor server delete view --- .../views/livewire/server/delete.blade.php | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/resources/views/livewire/server/delete.blade.php b/resources/views/livewire/server/delete.blade.php index 02f281e3b..b1855ea15 100644 --- a/resources/views/livewire/server/delete.blade.php +++ b/resources/views/livewire/server/delete.blade.php @@ -20,29 +20,30 @@ Delete @endif - @forelse ($server->definedResources() as $resource) - @if ($loop->first) -

    Defined resources

    -
    Please delete all resources before deleting this server.
    - @endif -
    -
    {{ str($resource->type())->headline() }}
    - {{ $resource->name }} -
    - @empty - @endforelse +
    + @forelse ($server->definedResources() as $resource) + @if ($loop->first) +

    Defined resources

    + @endif + +
    {{ str($resource->type())->headline() }}
    +
    {{ $resource->name }}
    +     + @empty + @endforelse +
    @else -
    - @forelse ($server->definedResources() as $resource) - @if ($loop->first) -

    Defined resources

    - @endif - -
    {{ str($resource->type())->headline() }}
    -
    {{ $resource->name }}
    -     - @empty - @endforelse +
    + @forelse ($server->definedResources() as $resource) + @if ($loop->first) +

    Defined resources

    + @endif + +
    {{ str($resource->type())->headline() }}
    +
    {{ $resource->name }}
    +     + @empty + @endforelse
    @endif
    From 0eda49b104e8eaada7ee1df33da1e5728a63633d Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 28 Nov 2023 11:10:42 +0100 Subject: [PATCH 34/46] fix: pull request build variables --- app/Jobs/ApplicationDeploymentJob.php | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index eea334c8e..7d229e879 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -657,8 +657,8 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted } $this->generate_compose_file(); // Needs separate preview variables - // $this->generate_build_env_variables(); - // $this->add_build_env_variables_to_dockerfile(); + $this->generate_build_env_variables(); + $this->add_build_env_variables_to_dockerfile(); $this->build_image(); $this->stop_running_container(); $this->execute_remote_command( @@ -810,7 +810,11 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted private function nixpacks_build_cmd() { $this->generate_env_variables(); - $nixpacks_command = "nixpacks build --cache-key '{$this->application->uuid}' -o {$this->workdir} {$this->env_args} --no-error-without-start"; + $cacheKey = $this->application->uuid; + if ($this->pull_request_id !== 0) { + $cacheKey = "{$this->application->uuid}-pr-{$this->pull_request_id}"; + } + $nixpacks_command = "nixpacks build --cache-key '{$cacheKey}' -o {$this->workdir} {$this->env_args} --no-error-without-start"; if ($this->application->build_command) { $nixpacks_command .= " --build-cmd \"{$this->application->build_command}\""; } @@ -1246,10 +1250,16 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf"); executeInDocker($this->deployment_uuid, "cat {$this->workdir}{$this->dockerfile_location}"), "hidden" => true, "save" => 'dockerfile' ]); $dockerfile = collect(Str::of($this->saved_outputs->get('dockerfile'))->trim()->explode("\n")); - - foreach ($this->application->build_environment_variables as $env) { - $dockerfile->splice(1, 0, "ARG {$env->key}={$env->value}"); + if ($this->pull_request_id === 0) { + foreach ($this->application->build_environment_variables as $env) { + $dockerfile->splice(1, 0, "ARG {$env->key}={$env->value}"); + } + } else { + foreach ($this->application->build_environment_variables_preview as $env) { + $dockerfile->splice(1, 0, "ARG {$env->key}={$env->value}"); + } } + ray($dockerfile->implode("\n")); $dockerfile_base64 = base64_encode($dockerfile->implode("\n")); $this->execute_remote_command([ executeInDocker($this->deployment_uuid, "echo '{$dockerfile_base64}' | base64 -d > {$this->workdir}{$this->dockerfile_location}"), From 7f5d7e0eb05f10763a4b63a1c1ae18081295d47b Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 28 Nov 2023 11:10:48 +0100 Subject: [PATCH 35/46] Refactor application submit method to handle dockercompose build pack --- app/Http/Livewire/Project/Application/General.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app/Http/Livewire/Project/Application/General.php b/app/Http/Livewire/Project/Application/General.php index 2917fa438..87b894bf4 100644 --- a/app/Http/Livewire/Project/Application/General.php +++ b/app/Http/Livewire/Project/Application/General.php @@ -199,7 +199,7 @@ class General extends Component public function submit($showToaster = true) { try { - if ($this->initialDockerComposeLocation !== $this->application->docker_compose_location || $this->initialDockerComposePrLocation !== $this->application->docker_compose_pr_location) { + if ($this->application->build_pack === 'dockercompose' && ($this->initialDockerComposeLocation !== $this->application->docker_compose_location || $this->initialDockerComposePrLocation !== $this->application->docker_compose_pr_location)) { $this->loadComposeFile(); } $this->validate(); @@ -234,10 +234,12 @@ class General extends Component $this->customLabels = str($this->customLabels)->replace(',', "\n"); } $this->application->custom_labels = $this->customLabels->explode("\n")->implode(','); - $this->application->docker_compose_domains = json_encode($this->parsedServiceDomains); + if ($this->application->build_pack === 'dockercompose') { + $this->application->docker_compose_domains = json_encode($this->parsedServiceDomains); + $this->parsedServices = $this->application->parseCompose(); + } $this->application->save(); $showToaster && $this->emit('success', 'Application settings updated!'); - $this->parsedServices = $this->application->parseCompose(); } catch (\Throwable $e) { return handleError($e, $this); } finally { From 39552cc42f29d70a38aadf0dbc44c256ef6f79b4 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 28 Nov 2023 12:04:21 +0100 Subject: [PATCH 36/46] fix: double default password length --- bootstrap/helpers/databases.php | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bootstrap/helpers/databases.php b/bootstrap/helpers/databases.php index 007c414bd..cef5ac7fd 100644 --- a/bootstrap/helpers/databases.php +++ b/bootstrap/helpers/databases.php @@ -24,7 +24,7 @@ function create_standalone_postgresql($environment_id, $destination_uuid): Stand } return StandalonePostgresql::create([ 'name' => generate_database_name('postgresql'), - 'postgres_password' => \Illuminate\Support\Str::password(symbols: false), + 'postgres_password' => \Illuminate\Support\Str::password(length: 64, symbols: false), 'environment_id' => $environment_id, 'destination_id' => $destination->id, 'destination_type' => $destination->getMorphClass(), @@ -39,7 +39,7 @@ function create_standalone_redis($environment_id, $destination_uuid): Standalone } return StandaloneRedis::create([ 'name' => generate_database_name('redis'), - 'redis_password' => \Illuminate\Support\Str::password(symbols: false), + 'redis_password' => \Illuminate\Support\Str::password(length: 64, symbols: false), 'environment_id' => $environment_id, 'destination_id' => $destination->id, 'destination_type' => $destination->getMorphClass(), @@ -54,7 +54,7 @@ function create_standalone_mongodb($environment_id, $destination_uuid): Standalo } return StandaloneMongodb::create([ 'name' => generate_database_name('mongodb'), - 'mongo_initdb_root_password' => \Illuminate\Support\Str::password(symbols: false), + 'mongo_initdb_root_password' => \Illuminate\Support\Str::password(length: 64, symbols: false), 'environment_id' => $environment_id, 'destination_id' => $destination->id, 'destination_type' => $destination->getMorphClass(), @@ -68,8 +68,8 @@ function create_standalone_mysql($environment_id, $destination_uuid): Standalone } return StandaloneMysql::create([ 'name' => generate_database_name('mysql'), - 'mysql_root_password' => \Illuminate\Support\Str::password(symbols: false), - 'mysql_password' => \Illuminate\Support\Str::password(symbols: false), + 'mysql_root_password' => \Illuminate\Support\Str::password(length: 64, symbols: false), + 'mysql_password' => \Illuminate\Support\Str::password(length: 64, symbols: false), 'environment_id' => $environment_id, 'destination_id' => $destination->id, 'destination_type' => $destination->getMorphClass(), @@ -83,8 +83,8 @@ function create_standalone_mariadb($environment_id, $destination_uuid): Standalo } return StandaloneMariadb::create([ 'name' => generate_database_name('mariadb'), - 'mariadb_root_password' => \Illuminate\Support\Str::password(symbols: false), - 'mariadb_password' => \Illuminate\Support\Str::password(symbols: false), + 'mariadb_root_password' => \Illuminate\Support\Str::password(length: 64, symbols: false), + 'mariadb_password' => \Illuminate\Support\Str::password(length: 64, symbols: false), 'environment_id' => $environment_id, 'destination_id' => $destination->id, 'destination_type' => $destination->getMorphClass(), From 2001be07d0a39da0c3a2c214bd4ca5480dd36237 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 28 Nov 2023 12:05:04 +0100 Subject: [PATCH 37/46] refactor: env variable generator --- app/Actions/Service/StartService.php | 4 +- app/Http/Controllers/ProjectController.php | 24 +--- bootstrap/helpers/shared.php | 143 ++++++++++----------- 3 files changed, 71 insertions(+), 100 deletions(-) diff --git a/app/Actions/Service/StartService.php b/app/Actions/Service/StartService.php index ef473e578..50f7fba82 100644 --- a/app/Actions/Service/StartService.php +++ b/app/Actions/Service/StartService.php @@ -16,13 +16,13 @@ class StartService $commands[] = "cd " . $service->workdir(); $commands[] = "echo 'Saved configuration files to {$service->workdir()}.'"; $commands[] = "echo 'Creating Docker network.'"; - $commands[] = "docker network create --attachable '{$service->uuid}' >/dev/null || true"; + $commands[] = "docker network create --attachable '{$service->uuid}' >/dev/null 2>&1 || true"; $commands[] = "echo 'Starting service {$service->name} on {$service->server->name}.'"; $commands[] = "echo 'Pulling images.'"; $commands[] = "docker compose pull"; $commands[] = "echo 'Starting containers.'"; $commands[] = "docker compose up -d --remove-orphans --force-recreate"; - $commands[] = "docker network connect $service->uuid coolify-proxy || true"; + $commands[] = "docker network connect $service->uuid coolify-proxy >/dev/null 2>&1 || true"; $compose = data_get($service,'docker_compose',[]); $serviceNames = data_get(Yaml::parse($compose),'services',[]); foreach($serviceNames as $serviceName => $serviceConfig){ diff --git a/app/Http/Controllers/ProjectController.php b/app/Http/Controllers/ProjectController.php index 58d0dbb6a..4e2fc5878 100644 --- a/app/Http/Controllers/ProjectController.php +++ b/app/Http/Controllers/ProjectController.php @@ -67,7 +67,7 @@ class ProjectController extends Controller $database = create_standalone_mongodb($environment->id, $destination_uuid); } else if ($type->value() === 'mysql') { $database = create_standalone_mysql($environment->id, $destination_uuid); - }else if ($type->value() === 'mariadb') { + } else if ($type->value() === 'mariadb') { $database = create_standalone_mariadb($environment->id, $destination_uuid); } return redirect()->route('project.database.configuration', [ @@ -104,27 +104,7 @@ class ProjectController extends Controller $generatedValue = $value; if ($value->contains('SERVICE_')) { $command = $value->after('SERVICE_')->beforeLast('_'); - // TODO: make it shared with Service.php - switch ($command->value()) { - case 'PASSWORD': - $generatedValue = Str::password(symbols: false); - break; - case 'PASSWORD_64': - $generatedValue = Str::password(length: 64, symbols: false); - break; - case 'BASE64_64': - $generatedValue = Str::random(64); - break; - case 'BASE64_128': - $generatedValue = Str::random(128); - break; - case 'BASE64': - $generatedValue = Str::random(32); - break; - case 'USER': - $generatedValue = Str::random(16); - break; - } + $generatedValue = generateEnvValue($command->value()); } EnvironmentVariable::create([ 'key' => $key, diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php index 2b2f32d33..324bd0652 100644 --- a/bootstrap/helpers/shared.php +++ b/bootstrap/helpers/shared.php @@ -579,7 +579,7 @@ function getTopLevelNetworks(Service|Application $resource) return $topLevelNetworks->keys(); } } -function parseDockerComposeFile(Service|Application $resource, bool $isNew = false, int $pull_request_id, bool $is_pr = false) +function parseDockerComposeFile(Service|Application $resource, bool $isNew = false, int $pull_request_id = 0, bool $is_pr = false) { // ray()->clearAll(); if ($resource->getMorphClass() === 'App\Models\Service') { @@ -921,22 +921,7 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal 'service_id' => $resource->id, ])->first(); if ($value->startsWith('SERVICE_')) { - // Count _ in $value - $count = substr_count($value->value(), '_'); - if ($count === 2) { - // SERVICE_FQDN_UMAMI - $command = $value->after('SERVICE_')->beforeLast('_'); - $forService = $value->afterLast('_'); - $generatedValue = null; - $port = null; - } - if ($count === 3) { - // SERVICE_FQDN_UMAMI_1000 - $command = $value->after('SERVICE_')->before('_'); - $forService = $value->after('SERVICE_')->after('_')->before('_'); - $generatedValue = null; - $port = $value->afterLast('_'); - } + ['command' => $command, 'forService' => $forService, 'generatedValue' => $generatedValue, 'port' => $port] = parseEnvVariable($value); if ($command->value() === 'FQDN' || $command->value() === 'URL') { if (Str::lower($forService) === $serviceName) { $fqdn = generateFqdn($resource->server, $containerName); @@ -967,27 +952,7 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal } } } else { - switch ($command) { - case 'PASSWORD': - $generatedValue = Str::password(symbols: false); - break; - case 'PASSWORD_64': - $generatedValue = Str::password(length: 64, symbols: false); - break; - case 'BASE64_64': - $generatedValue = Str::random(64); - break; - case 'BASE64_128': - $generatedValue = Str::random(128); - break; - case 'BASE64': - $generatedValue = Str::random(32); - break; - case 'USER': - $generatedValue = Str::random(16); - break; - } - + $generatedValue = generateEnvValue($command); if (!$foundEnv) { EnvironmentVariable::create([ 'key' => $key, @@ -1272,7 +1237,6 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal $key = Str::of($variableName); $value = Str::of($variable); } - // TODO: here is the problem if ($key->startsWith('SERVICE_FQDN')) { if ($isNew) { $name = $key->after('SERVICE_FQDN_')->beforeLast('_')->lower(); @@ -1315,22 +1279,7 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal 'application_id' => $resource->id, ])->first(); if ($value->startsWith('SERVICE_')) { - // Count _ in $value - $count = substr_count($value->value(), '_'); - if ($count === 2) { - // SERVICE_FQDN_UMAMI - $command = $value->after('SERVICE_')->beforeLast('_'); - $forService = $value->afterLast('_'); - $generatedValue = null; - $port = null; - } - if ($count === 3) { - // SERVICE_FQDN_UMAMI_1000 - $command = $value->after('SERVICE_')->before('_'); - $forService = $value->after('SERVICE_')->after('_')->before('_'); - $generatedValue = null; - $port = $value->afterLast('_'); - } + ['command' => $command, 'forService' => $forService, 'generatedValue' => $generatedValue, 'port' => $port] = parseEnvVariable($value); if ($command->value() === 'FQDN' || $command->value() === 'URL') { if (Str::lower($forService) === $serviceName) { $fqdn = generateFqdn($server, $containerName); @@ -1355,27 +1304,7 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal ]); } } else { - switch ($command) { - case 'PASSWORD': - $generatedValue = Str::password(symbols: false); - break; - case 'PASSWORD_64': - $generatedValue = Str::password(length: 64, symbols: false); - break; - case 'BASE64_64': - $generatedValue = Str::random(64); - break; - case 'BASE64_128': - $generatedValue = Str::random(128); - break; - case 'BASE64': - $generatedValue = Str::random(32); - break; - case 'USER': - $generatedValue = Str::random(16); - break; - } - + $generatedValue = generateEnvValue($command); if (!$foundEnv) { EnvironmentVariable::create([ 'key' => $key, @@ -1495,3 +1424,65 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal return collect($finalServices); } } + +function parseEnvVariable(Str|string $value) +{ + $value = str($value); + $count = substr_count($value->value(), '_'); + $command = null; + $forService = null; + $generatedValue = null; + $port = null; + + if ($count === 2) { + if ($value->startsWith('SERVICE_FQDN') || $value->startsWith('SERVICE_URL')) { + // SERVICE_FQDN_UMAMI + $command = $value->after('SERVICE_')->beforeLast('_'); + $forService = $value->afterLast('_'); + } else { + // SERVICE_BASE64_UMAMI + $command = $value->after('SERVICE_')->beforeLast('_'); + } + } + if ($count === 3) { + if ($value->startsWith('SERVICE_FQDN') || $value->startsWith('SERVICE_URL')) { + // SERVICE_FQDN_UMAMI_1000 + $command = $value->after('SERVICE_')->before('_'); + $forService = $value->after('SERVICE_')->after('_')->before('_'); + $port = $value->afterLast('_'); + } else { + // SERVICE_BASE64_64_UMAMI + $command = $value->after('SERVICE_')->beforeLast('_'); + } + } + return [ + 'command' => $command, + 'forService' => $forService, + 'generatedValue' => $generatedValue, + 'port' => $port, + ]; +} +function generateEnvValue(string $command) +{ + switch ($command) { + case 'PASSWORD': + $generatedValue = Str::password(symbols: false); + break; + case 'PASSWORD_64': + $generatedValue = Str::password(length: 64, symbols: false); + break; + case 'BASE64_64': + $generatedValue = Str::random(64); + break; + case 'BASE64_128': + $generatedValue = Str::random(128); + break; + case 'BASE64': + $generatedValue = Str::random(32); + break; + case 'USER': + $generatedValue = Str::random(16); + break; + } + return $generatedValue; +} From 066f1711630322e57fec22588d706da01372c7c4 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 28 Nov 2023 12:05:14 +0100 Subject: [PATCH 38/46] Add Docker Compose file for Formbricks service --- templates/compose/formbricks.yaml | 60 +++++++++++++++++++++++++++++++ templates/service-templates.json | 15 ++++++++ 2 files changed, 75 insertions(+) create mode 100644 templates/compose/formbricks.yaml diff --git a/templates/compose/formbricks.yaml b/templates/compose/formbricks.yaml new file mode 100644 index 000000000..7beed9de5 --- /dev/null +++ b/templates/compose/formbricks.yaml @@ -0,0 +1,60 @@ +# documentation: https://formbricks.com/docs/self-hosting/docker +# slogan: Open Source Experience Management +# tags: form, builder, forms, open source, experience, management, self-hosted, docker + +services: + formbricks: + image: formbricks/formbricks:latest + environment: + - SERVICE_FQDN_FORMBRICKS + - WEBAPP_URL=$SERVICE_FQDN_FORMBRICKS + - DATABASE_URL=postgres://$SERVICE_USER_POSTGRESQL:$SERVICE_PASSWORD_POSTGRESQL@postgresql:5432/${POSTGRESQL_DATABASE:-formbricks} + - NEXTAUTH_SECRET=$SERVICE_BASE64_64_NEXTAUTH + - NEXTAUTH_URL=$SERVICE_FQDN_FORMBRICKS + - ENCRYPTION_KEY=$SERVICE_BASE64_64_ENCRYPTION + - POSTGRES_PASSWORD=${SERVICE_PASSWORD_POSTGRESQL} + - MAIL_FROM=${MAIL_FROM:-test@example.com} + - SMTP_HOST=${SMTP_HOST:-test.example.com} + - SMTP_PORT=${SMTP_PORT:-587} + - SMTP_USER=${SMTP_USER:-test} + - SMTP_PASSWORD=${SMTP_PASSWORD:-test} + - SMTP_SECURE_ENABLED=${SMTP_SECURE_ENABLED:-0} + - SHORT_URL_BASE=${SHORT_URL_BASE} + - EMAIL_VERIFICATION_DISABLED=${EMAIL_VERIFICATION_DISABLED:-1} + - PASSWORD_RESET_DISABLED=${PASSWORD_RESET_DISABLED:-1} + - SIGNUP_DISABLED=${SIGNUP_DISABLED:-0} + - INVITE_DISABLED=${INVITE_DISABLED:-0} + - PRIVACY_URL=${PRIVACY_URL} + - TERMS_URL=${TERMS_URL} + - IMPRINT_URL=${IMPRINT_URL} + - GITHUB_AUTH_ENABLED=${GITHUB_AUTH_ENABLED:-0} + - GITHUB_ID=${GITHUB_ID} + - GITHUB_SECRET=${GITHUB_SECRET} + - GOOGLE_AUTH_ENABLED=${GOOGLE_AUTH_ENABLED:-0} + - GOOGLE_CLIENT_ID=${GOOGLE_CLIENT_ID} + - GOOGLE_CLIENT_SECRET=${GOOGLE_CLIENT_SECRET} + - ASSET_PREFIX_URL=${ASSET_PREFIX_URL} + volumes: + - formbricks-uploads:/apps/web/uploads/ + depends_on: + postgresql: + condition: service_healthy + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:3000"] + interval: 2s + timeout: 10s + retries: 15 + + postgresql: + image: postgres:16-alpine + volumes: + - formbricks-postgresql-data:/var/lib/postgresql/data + environment: + - POSTGRES_USER=${SERVICE_USER_POSTGRESQL} + - POSTGRES_PASSWORD=${SERVICE_PASSWORD_POSTGRESQL} + - POSTGRES_DB=${POSTGRESQL_DATABASE:-formbricks} + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] + interval: 5s + timeout: 20s + retries: 10 diff --git a/templates/service-templates.json b/templates/service-templates.json index c48488a90..aed78172c 100644 --- a/templates/service-templates.json +++ b/templates/service-templates.json @@ -152,6 +152,21 @@ "administration-tool" ] }, + "formbricks": { + "documentation": "https:\/\/formbricks.com\/docs\/self-hosting\/docker", + "slogan": "Open Source Experience Management", + "compose": "c2VydmljZXM6CiAgZm9ybWJyaWNrczoKICAgIGltYWdlOiAnZm9ybWJyaWNrcy9mb3JtYnJpY2tzOmxhdGVzdCcKICAgIGVudmlyb25tZW50OgogICAgICAtIFNFUlZJQ0VfRlFETl9GT1JNQlJJQ0tTCiAgICAgIC0gV0VCQVBQX1VSTD0kU0VSVklDRV9GUUROX0ZPUk1CUklDS1MKICAgICAgLSAnREFUQUJBU0VfVVJMPXBvc3RncmVzOi8vJFNFUlZJQ0VfVVNFUl9QT1NUR1JFU1FMOiRTRVJWSUNFX1BBU1NXT1JEX1BPU1RHUkVTUUxAcG9zdGdyZXNxbDo1NDMyLyR7UE9TVEdSRVNRTF9EQVRBQkFTRTotZm9ybWJyaWNrc30nCiAgICAgIC0gTkVYVEFVVEhfU0VDUkVUPSRTRVJWSUNFX0JBU0U2NF82NF9ORVhUQVVUSAogICAgICAtIE5FWFRBVVRIX1VSTD0kU0VSVklDRV9GUUROX0ZPUk1CUklDS1MKICAgICAgLSBFTkNSWVBUSU9OX0tFWT0kU0VSVklDRV9CQVNFNjRfNjRfRU5DUllQVElPTgogICAgICAtICdQT1NUR1JFU19QQVNTV09SRD0ke1NFUlZJQ0VfUEFTU1dPUkRfUE9TVEdSRVNRTH0nCiAgICAgIC0gJ01BSUxfRlJPTT0ke01BSUxfRlJPTTotdGVzdEBleGFtcGxlLmNvbX0nCiAgICAgIC0gJ1NNVFBfSE9TVD0ke1NNVFBfSE9TVDotdGVzdC5leGFtcGxlLmNvbX0nCiAgICAgIC0gJ1NNVFBfUE9SVD0ke1NNVFBfUE9SVDotNTg3fScKICAgICAgLSAnU01UUF9VU0VSPSR7U01UUF9VU0VSOi10ZXN0fScKICAgICAgLSAnU01UUF9QQVNTV09SRD0ke1NNVFBfUEFTU1dPUkQ6LXRlc3R9JwogICAgICAtICdTTVRQX1NFQ1VSRV9FTkFCTEVEPSR7U01UUF9TRUNVUkVfRU5BQkxFRDotMH0nCiAgICAgIC0gJ1NIT1JUX1VSTF9CQVNFPSR7U0hPUlRfVVJMX0JBU0V9JwogICAgICAtICdFTUFJTF9WRVJJRklDQVRJT05fRElTQUJMRUQ9JHtFTUFJTF9WRVJJRklDQVRJT05fRElTQUJMRUQ6LTF9JwogICAgICAtICdQQVNTV09SRF9SRVNFVF9ESVNBQkxFRD0ke1BBU1NXT1JEX1JFU0VUX0RJU0FCTEVEOi0xfScKICAgICAgLSAnU0lHTlVQX0RJU0FCTEVEPSR7U0lHTlVQX0RJU0FCTEVEOi0wfScKICAgICAgLSAnSU5WSVRFX0RJU0FCTEVEPSR7SU5WSVRFX0RJU0FCTEVEOi0wfScKICAgICAgLSAnUFJJVkFDWV9VUkw9JHtQUklWQUNZX1VSTH0nCiAgICAgIC0gJ1RFUk1TX1VSTD0ke1RFUk1TX1VSTH0nCiAgICAgIC0gJ0lNUFJJTlRfVVJMPSR7SU1QUklOVF9VUkx9JwogICAgICAtICdHSVRIVUJfQVVUSF9FTkFCTEVEPSR7R0lUSFVCX0FVVEhfRU5BQkxFRDotMH0nCiAgICAgIC0gJ0dJVEhVQl9JRD0ke0dJVEhVQl9JRH0nCiAgICAgIC0gJ0dJVEhVQl9TRUNSRVQ9JHtHSVRIVUJfU0VDUkVUfScKICAgICAgLSAnR09PR0xFX0FVVEhfRU5BQkxFRD0ke0dPT0dMRV9BVVRIX0VOQUJMRUQ6LTB9JwogICAgICAtICdHT09HTEVfQ0xJRU5UX0lEPSR7R09PR0xFX0NMSUVOVF9JRH0nCiAgICAgIC0gJ0dPT0dMRV9DTElFTlRfU0VDUkVUPSR7R09PR0xFX0NMSUVOVF9TRUNSRVR9JwogICAgICAtICdBU1NFVF9QUkVGSVhfVVJMPSR7QVNTRVRfUFJFRklYX1VSTH0nCiAgICB2b2x1bWVzOgogICAgICAtICdmb3JtYnJpY2tzLXVwbG9hZHM6L2FwcHMvd2ViL3VwbG9hZHMvJwogICAgZGVwZW5kc19vbjoKICAgICAgcG9zdGdyZXNxbDoKICAgICAgICBjb25kaXRpb246IHNlcnZpY2VfaGVhbHRoeQogICAgaGVhbHRoY2hlY2s6CiAgICAgIHRlc3Q6CiAgICAgICAgLSBDTUQKICAgICAgICAtIGN1cmwKICAgICAgICAtICctZicKICAgICAgICAtICdodHRwOi8vbG9jYWxob3N0OjMwMDAnCiAgICAgIGludGVydmFsOiAycwogICAgICB0aW1lb3V0OiAxMHMKICAgICAgcmV0cmllczogMTUKICBwb3N0Z3Jlc3FsOgogICAgaW1hZ2U6ICdwb3N0Z3JlczoxNi1hbHBpbmUnCiAgICB2b2x1bWVzOgogICAgICAtICdmb3JtYnJpY2tzLXBvc3RncmVzcWwtZGF0YTovdmFyL2xpYi9wb3N0Z3Jlc3FsL2RhdGEnCiAgICBlbnZpcm9ubWVudDoKICAgICAgLSAnUE9TVEdSRVNfVVNFUj0ke1NFUlZJQ0VfVVNFUl9QT1NUR1JFU1FMfScKICAgICAgLSAnUE9TVEdSRVNfUEFTU1dPUkQ9JHtTRVJWSUNFX1BBU1NXT1JEX1BPU1RHUkVTUUx9JwogICAgICAtICdQT1NUR1JFU19EQj0ke1BPU1RHUkVTUUxfREFUQUJBU0U6LWZvcm1icmlja3N9JwogICAgaGVhbHRoY2hlY2s6CiAgICAgIHRlc3Q6CiAgICAgICAgLSBDTUQtU0hFTEwKICAgICAgICAtICdwZ19pc3JlYWR5IC1VICQke1BPU1RHUkVTX1VTRVJ9IC1kICQke1BPU1RHUkVTX0RCfScKICAgICAgaW50ZXJ2YWw6IDVzCiAgICAgIHRpbWVvdXQ6IDIwcwogICAgICByZXRyaWVzOiAxMAo=", + "tags": [ + "form", + "builder", + "forms", + "open source", + "experience", + "management", + "self-hosted", + "docker" + ] + }, "ghost": { "documentation": "https:\/\/ghost.org\/docs", "slogan": "Ghost is a popular open-source content management system (CMS) and blogging platform, known for its simplicity and focus on content creation.", From d058e042139673c134a8547a45849147cdb77954 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 28 Nov 2023 12:11:03 +0100 Subject: [PATCH 39/46] Add fqdn attribute to InstanceSettings model --- app/Models/InstanceSettings.php | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/app/Models/InstanceSettings.php b/app/Models/InstanceSettings.php index 057595351..846148159 100644 --- a/app/Models/InstanceSettings.php +++ b/app/Models/InstanceSettings.php @@ -3,8 +3,10 @@ namespace App\Models; use App\Notifications\Channels\SendsEmail; +use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Database\Eloquent\Model; use Illuminate\Notifications\Notifiable; +use Spatie\Url\Url; class InstanceSettings extends Model implements SendsEmail { @@ -16,6 +18,18 @@ class InstanceSettings extends Model implements SendsEmail 'smtp_password' => 'encrypted', ]; + public function fqdn(): Attribute + { + return Attribute::make( + set: function ($value) { + if ($value) { + $url = Url::fromString($value); + $host = $url->getHost(); + return $url->getScheme() . '://' . $host; + } + } + ); + } public static function get() { return InstanceSettings::findOrFail(0); From 2788fcf4e1d0f7936f4a01b9fc67cc26b8c9897a Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 28 Nov 2023 12:48:55 +0100 Subject: [PATCH 40/46] Add Docker Compose based applications and preview deployments to proxy on restart --- app/Models/Server.php | 17 +++++++++++++++++ bootstrap/helpers/proxy.php | 21 +++++++++++++++++++-- bootstrap/helpers/shared.php | 1 - 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/app/Models/Server.php b/app/Models/Server.php index 5aa571452..650e5129a 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -228,6 +228,23 @@ class Server extends BaseModel return $standaloneDocker->applications; })->flatten(); } + public function dockerComposeBasedApplications() + { + return $this->applications()->filter(function ($application) { + return data_get($application, 'build_pack') === 'dockercompose'; + }); + } + public function dockerComposeBasedPreviewDeployments() + { + return $this->previews()->filter(function ($preview) { + $applicationId = data_get($preview, 'application_id'); + $application = Application::find($applicationId); + if (!$application) { + return false; + } + return data_get($application, 'build_pack') === 'dockercompose'; + }); + } public function services() { return $this->hasMany(Service::class); diff --git a/bootstrap/helpers/proxy.php b/bootstrap/helpers/proxy.php index 86946231b..2f595d1c6 100644 --- a/bootstrap/helpers/proxy.php +++ b/bootstrap/helpers/proxy.php @@ -1,6 +1,7 @@ standaloneDockers)->map(function ($docker) { return $docker['network']; }); // Service networks - foreach($server->services()->get() as $service) { + foreach ($server->services()->get() as $service) { $networks->push($service->networks()); } + // Docker compose based apps + $docker_compose_apps = $server->dockerComposeBasedApplications(); + foreach ($docker_compose_apps as $app) { + $networks->push($app->uuid); + } + // Docker compose based preview deployments + $docker_compose_previews = $server->dockerComposeBasedPreviewDeployments(); + foreach ($docker_compose_previews as $preview) { + $pullRequestId = $preview->pull_request_id; + $applicationId = $preview->application_id; + $application = Application::find($applicationId); + if (!$application) { + continue; + } + $network = "{$application->uuid}-{$pullRequestId}"; + $networks->push($network); + } $networks = collect($networks)->flatten()->unique(); if ($networks->count() === 0) { $networks = collect(['coolify']); diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php index 324bd0652..8eb1fd443 100644 --- a/bootstrap/helpers/shared.php +++ b/bootstrap/helpers/shared.php @@ -1072,7 +1072,6 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal throw new \Exception($e->getMessage()); } } - ray($yaml); $server = $resource->destination->server; $topLevelVolumes = collect(data_get($yaml, 'volumes', [])); if ($pull_request_id !== 0) { From 085b655d9f546eb609b6d24a6e40478527e08f2d Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 28 Nov 2023 13:02:12 +0100 Subject: [PATCH 41/46] Update version to 1.1.0 and add support for Redhat and Sles based operating systems --- scripts/install.sh | 52 ++++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index cad2599e9..6a4e1c9f3 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -1,7 +1,7 @@ #!/bin/bash ## Do not modify this file. You will lose the ability to install and auto-update! -VERSION="1.0.4" +VERSION="1.1.0" DOCKER_VERSION="24.0" CDN="https://cdn.coollabs.io/coolify" @@ -14,32 +14,14 @@ if [ $EUID != 0 ]; then echo "Please run as root" exit fi -# case "$OS_TYPE" in -# ubuntu | debian | raspbian) -# echo "Installing dependencies with APT..." -# apt update -y >/dev/null 2>&1 -# apt install -y curl wget git jq >/dev/null 2>&1 -# ;; -# centos | fedora | rhel | ol | rocky) -# echo "Installing dependencies with DNF..." -# dnf update -y >/dev/null 2>&1 -# dnf install -y curl wget git jq >/dev/null 2>&1 -# ;; -# sles | opensuse-leap | opensuse-tumbleweed) -# echo "Installing dependencies with Zypper..." -# zypper refresh >/dev/null 2>&1 -# zypper install -y curl wget git jq >/dev/null 2>&1 -# ;; -# *) -# echo "This script only supports Debian, Redhat or Sles based operating systems for now." -# exit -# ;; -# esac -if [ $OS_TYPE != "ubuntu" ] && [ $OS_TYPE != "debian" ] && [ $OS_TYPE != "raspbian" ]; then - echo "This script only supports Ubuntu and Debian for now." +case "$OS_TYPE" in +ubuntu | debian | raspbian | centos | fedora | rhel | ol | rocky | sles | opensuse-leap | opensuse-tumbleweed) ;; +*) + echo "This script only supports Debian, Redhat or Sles based operating systems for now." exit -fi + ;; +esac # Ovewrite LATEST_VERSION if user pass a version number if [ "$1" != "" ]; then @@ -58,8 +40,24 @@ echo "Coolify version: $LATEST_VERSION" echo -e "-------------" echo "Installing required packages..." -apt update -y >/dev/null 2>&1 -apt install -y curl wget git jq >/dev/null 2>&1 +case "$OS_TYPE" in +ubuntu | debian | raspbian) + apt update -y >/dev/null 2>&1 + apt install -y curl wget git jq >/dev/null 2>&1 + ;; +centos | fedora | rhel | ol | rocky) + dnf update -y >/dev/null 2>&1 + dnf install -y curl wget git jq >/dev/null 2>&1 + ;; +sles | opensuse-leap | opensuse-tumbleweed) + zypper refresh >/dev/null 2>&1 + zypper install -y curl wget git jq >/dev/null 2>&1 + ;; +*) + echo "This script only supports Debian, Redhat or Sles based operating systems for now." + exit + ;; +esac if ! [ -x "$(command -v docker)" ]; then echo "Docker is not installed. Installing Docker..." From 1c386db41d240ea7ec5aa13e84a38903b2146ff3 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 28 Nov 2023 13:12:25 +0100 Subject: [PATCH 42/46] Update Docker installation command and add support for SLES --- app/Actions/Server/InstallDocker.php | 15 +++++++++++---- app/Models/Server.php | 17 +++++++++++------ bootstrap/helpers/constants.php | 5 +++-- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/app/Actions/Server/InstallDocker.php b/app/Actions/Server/InstallDocker.php index 0713ed086..08874fc81 100644 --- a/app/Actions/Server/InstallDocker.php +++ b/app/Actions/Server/InstallDocker.php @@ -15,7 +15,7 @@ class InstallDocker if (!$supported_os_type) { throw new \Exception('Server OS type is not supported for automated installation. Please install Docker manually before continuing: documentation.'); } - ray('Installing Docker on server: ' . $server->name . ' (' . $server->ip . ')' . ' with OS: ' . $supported_os_type); + ray('Installing Docker on server: ' . $server->name . ' (' . $server->ip . ')' . ' with OS type: ' . $supported_os_type); $dockerVersion = '24.0'; $config = base64_encode('{ "log-driver": "json-file", @@ -44,18 +44,25 @@ class InstallDocker "ls -l /tmp" ]); } else { - if ($supported_os_type === 'debian') { + if ($supported_os_type->contains('debian')) { $command = $command->merge([ "echo 'Installing Prerequisites...'", - "command -v jq >/dev/null || apt-get update", + "command -v jq >/dev/null || apt-get update -y", "command -v jq >/dev/null || apt install -y jq", ]); - } else if ($supported_os_type === 'rhel') { + } else if ($supported_os_type->contains('rhel')) { $command = $command->merge([ "echo 'Installing Prerequisites...'", + "command -v jq >/dev/null || dnf update -y", "command -v jq >/dev/null || dnf install -y jq", ]); + } else if ($supported_os_type->contains('sles')) { + $command = $command->merge([ + "echo 'Installing Prerequisites...'", + "command -v jq >/dev/null || zypper update -y", + "command -v jq >/dev/null || zypper install -y jq", + ]); } else { throw new \Exception('Unsupported OS'); } diff --git a/app/Models/Server.php b/app/Models/Server.php index 650e5129a..e39bbcaa6 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -194,7 +194,8 @@ class Server extends BaseModel { return instant_remote_process(["df /| tail -1 | awk '{ print $5}' | sed 's/%//g'"], $this, false); } - public function definedResources() { + public function definedResources() + { $applications = $this->applications(); $databases = $this->databases(); $services = $this->services(); @@ -342,12 +343,16 @@ class Server extends BaseModel $collectedData->put($item->before('=')->value(), $item->after('=')->lower()->replace('"', '')->value()); } $ID = data_get($collectedData, 'ID'); - $ID_LIKE = data_get($collectedData, 'ID_LIKE'); - $VERSION_ID = data_get($collectedData, 'VERSION_ID'); - // ray($ID, $ID_LIKE, $VERSION_ID); - if (collect(SUPPORTED_OS)->contains($ID_LIKE)) { + // $ID_LIKE = data_get($collectedData, 'ID_LIKE'); + // $VERSION_ID = data_get($collectedData, 'VERSION_ID'); + $supported = collect(SUPPORTED_OS)->filter(function ($supportedOs) use ($ID) { + if (str($supportedOs)->contains($ID)) { + return str($ID); + } + }); + if ($supported->count() === 1) { ray('supported'); - return str($ID_LIKE)->explode(' ')->first(); + return $supported->first(); } else { ray('not supported'); return false; diff --git a/bootstrap/helpers/constants.php b/bootstrap/helpers/constants.php index 299d3acb9..ba659c708 100644 --- a/bootstrap/helpers/constants.php +++ b/bootstrap/helpers/constants.php @@ -29,6 +29,7 @@ const SPECIFIC_SERVICES = [ ]; const SUPPORTED_OS = [ - 'debian', - 'rhel centos fedora' + 'ubuntu debian raspbian', + 'centos fedora rhel ol rocky', + 'sles opensuse-leap opensuse-tumbleweed' ]; From e6f7e320372502adf9200cf95e02722a600374ea Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 28 Nov 2023 13:12:42 +0100 Subject: [PATCH 43/46] Add SUPPORTED_OS constant based on /etc/os-release --- bootstrap/helpers/constants.php | 1 + 1 file changed, 1 insertion(+) diff --git a/bootstrap/helpers/constants.php b/bootstrap/helpers/constants.php index ba659c708..380168005 100644 --- a/bootstrap/helpers/constants.php +++ b/bootstrap/helpers/constants.php @@ -28,6 +28,7 @@ const SPECIFIC_SERVICES = [ 'quay.io/minio/minio', ]; +// Based on /etc/os-release const SUPPORTED_OS = [ 'ubuntu debian raspbian', 'centos fedora rhel ol rocky', From b26e23e7c3e8990c5675a9c53ae79f7f293c17e2 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 28 Nov 2023 13:17:59 +0100 Subject: [PATCH 44/46] Fix validateOS() return type --- app/Models/Server.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/Models/Server.php b/app/Models/Server.php index e39bbcaa6..81810d557 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -333,7 +333,7 @@ class Server extends BaseModel { return $this->settings->is_logdrain_newrelic_enabled || $this->settings->is_logdrain_highlight_enabled || $this->settings->is_logdrain_axiom_enabled; } - public function validateOS() + public function validateOS(): bool | Str { $os_release = instant_remote_process(['cat /etc/os-release'], $this); $datas = collect(explode("\n", $os_release)); @@ -352,7 +352,7 @@ class Server extends BaseModel }); if ($supported->count() === 1) { ray('supported'); - return $supported->first(); + return str($supported->first()); } else { ray('not supported'); return false; From e35b8a0f96b3b174d0b9abec60740a500f6004fd Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 28 Nov 2023 13:21:32 +0100 Subject: [PATCH 45/46] Add Stringable interface to validateOS method --- app/Models/Server.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/Models/Server.php b/app/Models/Server.php index 81810d557..07714caa7 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -18,6 +18,7 @@ use Illuminate\Support\Sleep; use Spatie\SchemalessAttributes\Casts\SchemalessAttributes; use Spatie\SchemalessAttributes\SchemalessAttributesTrait; use Illuminate\Support\Str; +use Stringable; class Server extends BaseModel { @@ -333,7 +334,7 @@ class Server extends BaseModel { return $this->settings->is_logdrain_newrelic_enabled || $this->settings->is_logdrain_highlight_enabled || $this->settings->is_logdrain_axiom_enabled; } - public function validateOS(): bool | Str + public function validateOS(): bool | Stringable { $os_release = instant_remote_process(['cat /etc/os-release'], $this); $datas = collect(explode("\n", $os_release)); From 44f6d936390cb89655734264d4374bcdf3aabda7 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 28 Nov 2023 13:28:15 +0100 Subject: [PATCH 46/46] Update installation script to include curl and wget --- app/Actions/Server/InstallDocker.php | 7 +++---- scripts/install.sh | 1 - 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/app/Actions/Server/InstallDocker.php b/app/Actions/Server/InstallDocker.php index 08874fc81..a580c3473 100644 --- a/app/Actions/Server/InstallDocker.php +++ b/app/Actions/Server/InstallDocker.php @@ -48,20 +48,19 @@ class InstallDocker $command = $command->merge([ "echo 'Installing Prerequisites...'", "command -v jq >/dev/null || apt-get update -y", - "command -v jq >/dev/null || apt install -y jq", + "command -v jq >/dev/null || apt install -y curl wget git jq", ]); } else if ($supported_os_type->contains('rhel')) { $command = $command->merge([ "echo 'Installing Prerequisites...'", - "command -v jq >/dev/null || dnf update -y", - "command -v jq >/dev/null || dnf install -y jq", + "command -v jq >/dev/null || dnf install -y curl wget git jq", ]); } else if ($supported_os_type->contains('sles')) { $command = $command->merge([ "echo 'Installing Prerequisites...'", "command -v jq >/dev/null || zypper update -y", - "command -v jq >/dev/null || zypper install -y jq", + "command -v jq >/dev/null || zypper install -y curl wget git jq", ]); } else { throw new \Exception('Unsupported OS'); diff --git a/scripts/install.sh b/scripts/install.sh index 6a4e1c9f3..85bc58aad 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -46,7 +46,6 @@ ubuntu | debian | raspbian) apt install -y curl wget git jq >/dev/null 2>&1 ;; centos | fedora | rhel | ol | rocky) - dnf update -y >/dev/null 2>&1 dnf install -y curl wget git jq >/dev/null 2>&1 ;; sles | opensuse-leap | opensuse-tumbleweed)