From d1e10dacc05e952a5a447fefd403f428d73de8ad Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 23 Nov 2023 21:02:30 +0100 Subject: [PATCH] wip --- app/Jobs/ApplicationDeployDockerImageJob.php | 122 ++++++++++++------ app/Models/Application.php | 16 +++ app/Models/ApplicationDeploymentQueue.php | 35 +++++ app/Models/Server.php | 50 ++++--- bootstrap/helpers/applications.php | 84 ++++++------ bootstrap/helpers/remoteProcess.php | 4 +- .../application/deployment-logs.blade.php | 2 +- 7 files changed, 214 insertions(+), 99 deletions(-) diff --git a/app/Jobs/ApplicationDeployDockerImageJob.php b/app/Jobs/ApplicationDeployDockerImageJob.php index 24526937a..c21ad5680 100644 --- a/app/Jobs/ApplicationDeployDockerImageJob.php +++ b/app/Jobs/ApplicationDeployDockerImageJob.php @@ -5,8 +5,8 @@ namespace App\Jobs; use App\Enums\ApplicationDeploymentStatus; use App\Models\Application; use App\Models\ApplicationDeploymentQueue; +use App\Models\Server; use App\Traits\ExecuteRemoteCommandNew; -use Exception; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldQueue; @@ -21,77 +21,117 @@ class ApplicationDeployDockerImageJob implements ShouldQueue, ShouldBeEncrypted public $timeout = 3600; public $tries = 1; - public string $applicationDeploymentQueueId; + public $remoteCommandOutputs = []; + public Server $server; + public string $containerName; - public function __construct(string $applicationDeploymentQueueId) + public function __construct(public ApplicationDeploymentQueue $deploymentQueueEntry, public Application $application) { - $this->applicationDeploymentQueueId = $applicationDeploymentQueueId; } public function handle() { ray()->clearAll(); ray('Deploying Docker Image'); + static::$batch_counter = 0; try { - $applicationDeploymentQueue = ApplicationDeploymentQueue::find($this->applicationDeploymentQueueId); + $deploymentUuid = data_get($this->deploymentQueueEntry, 'deployment_uuid'); + $pullRequestId = data_get($this->deploymentQueueEntry, 'pull_request_id'); - $deploymentUuid = data_get($applicationDeploymentQueue, 'deployment_uuid'); - $pullRequestId = data_get($applicationDeploymentQueue, 'pull_request_id'); + $this->server = data_get($this->application->destination, 'server'); + $network = data_get($this->application->destination, 'network'); - $application = Application::find($applicationDeploymentQueue->application_id)->firstOrFail(); - $server = data_get($application->destination, 'server'); - $network = data_get($application->destination, 'network'); - - $dockerImage = data_get($application, 'docker_registry_image_name'); - $dockerImageTag = data_get($application, 'docker_registry_image_tag'); + $dockerImage = data_get($this->application, 'docker_registry_image_name'); + $dockerImageTag = data_get($this->application, 'docker_registry_image_tag'); $productionImageName = str("{$dockerImage}:{$dockerImageTag}"); - - $containerName = generateApplicationContainerName($application, $pullRequestId); - savePrivateKeyToFs($server); + $this->containerName = generateApplicationContainerName($this->application, $pullRequestId); + savePrivateKeyToFs($this->server); ray("echo 'Starting deployment of {$productionImageName}.'"); - $applicationDeploymentQueue->update([ + $this->deploymentQueueEntry->update([ 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value, ]); - $server->executeRemoteCommand( - commands: prepareHelperContainer($server, $network, $deploymentUuid), - loggingModel: $applicationDeploymentQueue + + $this->deploymentQueueEntry->addLogEntry('Starting deployment of ' . $productionImageName); + + $this->server->executeRemoteCommand( + commands: collect( + [ + [ + "name" => "ls", + "command" => 'ls -la', + "hidden" => true, + ], + [ + "name" => "pwd", + "command" => 'pwd', + "hidden" => true, + ] + ], + ), + loggingModel: $this->deploymentQueueEntry ); - $server->executeRemoteCommand( + $this->server->executeRemoteCommand( + commands: prepareHelperContainer($this->server, $network, $deploymentUuid), + loggingModel: $this->deploymentQueueEntry + ); + $this->server->executeRemoteCommand( commands: generateComposeFile( deploymentUuid: $deploymentUuid, - server: $server, + server: $this->server, network: $network, - application: $application, - containerName: $containerName, + application: $this->application, + containerName: $this->containerName, imageName: $productionImageName, pullRequestId: $pullRequestId ), - loggingModel: $applicationDeploymentQueue + loggingModel: $this->deploymentQueueEntry ); - $server->executeRemoteCommand( - commands: rollingUpdate(application: $application, deploymentUuid: $deploymentUuid), - loggingModel: $applicationDeploymentQueue - ); - $applicationDeploymentQueue->update([ + $this->deploymentQueueEntry->addLogEntry('----------------------------------------'); + + // Rolling update not possible + if (count($this->application->ports_mappings_array) > 0) { + $this->deploymentQueueEntry->addLogEntry('Application has ports mapped to the host system, rolling update is not supported.'); + $this->deploymentQueueEntry->addLogEntry('Stopping running container.'); + $this->server->stopApplicationRelatedRunningContainers($this->application->id, $this->containerName); + } else { + $this->deploymentQueueEntry->addLogEntry('Rolling update started.'); + // TODO + $this->server->executeRemoteCommand( + commands: startNewApplication(application: $this->application, deploymentUuid: $deploymentUuid, loggingModel: $this->deploymentQueueEntry), + loggingModel: $this->deploymentQueueEntry + ); + // $this->server->executeRemoteCommand( + // commands: healthCheckContainer(application: $this->application, containerName: $this->containerName , loggingModel: $this->deploymentQueueEntry), + // loggingModel: $this->deploymentQueueEntry + // ); + + } + + ray($this->remoteCommandOutputs); + $this->deploymentQueueEntry->update([ 'status' => ApplicationDeploymentStatus::FINISHED->value, ]); } catch (Throwable $e) { - $this->executeRemoteCommand( - server: $server, - logModel: $applicationDeploymentQueue, - commands: [ - "echo 'Oops something is not okay, are you okay? 😢'", - "echo '{$e->getMessage()}'", - "echo -n 'Deployment failed. Removing the new version of your application.'", - executeInDocker($deploymentUuid, "docker rm -f $containerName >/dev/null 2>&1"), - ] - ); - // $this->next(ApplicationDeploymentStatus::FAILED->value); + $this->fail($e); throw $e; } } + public function failed(Throwable $exception): void + { + $this->deploymentQueueEntry->addLogEntry('Oops something is not okay, are you okay? 😢', 'error'); + $this->deploymentQueueEntry->addLogEntry($exception->getMessage(), 'error'); + $this->deploymentQueueEntry->addLogEntry('Deployment failed. Removing the new version of your application.'); + + $this->server->executeRemoteCommand( + commands: removeOldDeployment($this->containerName), + loggingModel: $this->deploymentQueueEntry + ); + $this->deploymentQueueEntry->update([ + 'status' => ApplicationDeploymentStatus::FAILED->value, + ]); + } // private function next(string $status) // { // // If the deployment is cancelled by the user, don't update the status diff --git a/app/Models/Application.php b/app/Models/Application.php index 785ef3040..815376d18 100644 --- a/app/Models/Application.php +++ b/app/Models/Application.php @@ -342,4 +342,20 @@ class Application extends BaseModel } return false; } + public function healthCheckUrl() { + if ($this->dockerfile || $this->build_pack === 'dockerfile' || $this->build_pack === 'dockerimage') { + return null; + } + if (!$this->health_check_port) { + $health_check_port = $this->ports_exposes_array[0]; + } else { + $health_check_port = $this->health_check_port; + } + if ($this->health_check_path) { + $full_healthcheck_url = "{$this->health_check_scheme}://{$this->health_check_host}:{$health_check_port}{$this->health_check_path}"; + } else { + $full_healthcheck_url = "{$this->health_check_scheme}://{$this->health_check_host}:{$health_check_port}/"; + } + return $full_healthcheck_url; + } } diff --git a/app/Models/ApplicationDeploymentQueue.php b/app/Models/ApplicationDeploymentQueue.php index 1b7ae4781..bc9b9471a 100644 --- a/app/Models/ApplicationDeploymentQueue.php +++ b/app/Models/ApplicationDeploymentQueue.php @@ -3,8 +3,43 @@ namespace App\Models; use Illuminate\Database\Eloquent\Model; +use Illuminate\Support\Carbon; class ApplicationDeploymentQueue extends Model { protected $guarded = []; + + public function getOutput($name) { + if (!$this->logs) { + return null; + } + return collect(json_decode($this->logs))->where('name', $name)->first()?->output ?? null; + } + + public function addLogEntry(string $message, string $type = 'stdout', bool $hidden = false) + { + if ($type === 'error') { + $type = 'stderr'; + } + $newLogEntry = [ + 'command' => null, + 'output' => $message, + 'type' => $type, + 'timestamp' => Carbon::now('UTC'), + 'hidden' => $hidden, + 'batch' => 1, + ]; + if ($this->logs) { + $previousLogs = json_decode($this->logs, associative: true, flags: JSON_THROW_ON_ERROR); + $newLogEntry['order'] = count($previousLogs) + 1; + $previousLogs[] = $newLogEntry; + $this->update([ + 'logs' => json_encode($previousLogs, flags: JSON_THROW_ON_ERROR), + ]); + } else { + $this->update([ + 'logs' => json_encode([$newLogEntry], flags: JSON_THROW_ON_ERROR), + ]); + } + } } diff --git a/app/Models/Server.php b/app/Models/Server.php index 50e69f3b3..7811b1aaa 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -396,23 +396,23 @@ class Server extends BaseModel { static::$batch_counter++; foreach ($commands as $command) { - $command = data_get($command, 'command') ?? $command[0] ?? null; - if (is_null($command)) { - continue; + $realCommand = data_get($command, 'command'); + if (is_null($realCommand)) { + throw new \RuntimeException('Command is not set'); } $hidden = data_get($command, 'hidden', false); $ignoreErrors = data_get($command, 'ignoreErrors', false); $customOutputType = data_get($command, 'customOutputType'); - $saveOutput = data_get($command, 'saveOutput'); - $remoteCommand = generateSshCommand($this, $command); + $name = data_get($command, 'name'); + $remoteCommand = generateSshCommand($this, $realCommand); - $process = Process::timeout(3600)->idleTimeout(3600)->start($remoteCommand, function (string $type, string $output) use ($command, $hidden, $customOutputType, $loggingModel, $saveOutput) { + $process = Process::timeout(3600)->idleTimeout(3600)->start($remoteCommand, function (string $type, string $output) use ($realCommand, $hidden, $customOutputType, $loggingModel, $name) { $output = str($output)->trim(); if ($output->startsWith('╔')) { $output = "\n" . $output; } $newLogEntry = [ - 'command' => remove_iip($command), + 'command' => remove_iip($realCommand), 'output' => remove_iip($output), 'type' => $customOutputType ?? $type === 'err' ? 'stderr' : 'stdout', 'timestamp' => Carbon::now('UTC'), @@ -420,19 +420,21 @@ class Server extends BaseModel 'batch' => static::$batch_counter, ]; if (!$loggingModel->logs) { - $new_log_entry['order'] = 1; + $newLogEntry['order'] = 1; } else { - $previous_logs = json_decode($this->log_model->logs, associative: true, flags: JSON_THROW_ON_ERROR); - $new_log_entry['order'] = count($previous_logs) + 1; + $previousLogs = json_decode($loggingModel->logs, associative: true, flags: JSON_THROW_ON_ERROR); + $newLogEntry['order'] = count($previousLogs) + 1; } - $previousLogs = json_decode($loggingModel->logs, associative: true, flags: JSON_THROW_ON_ERROR); - $newLogEntry['order'] = count($previousLogs) + 1; + if ($name) { + $newLogEntry['name'] = $name; + } + $previousLogs[] = $newLogEntry; $loggingModel->logs = json_encode($previousLogs, flags: JSON_THROW_ON_ERROR); $loggingModel->save(); - if ($saveOutput) { - $this->remoteCommandOutputs[$saveOutput] = str($output)->trim(); - } + // if ($name) { + // $loggingModel['savedOutputs'][$name] = str($output)->trim(); + // } }); $loggingModel->update([ 'current_process_id' => $process->id(), @@ -449,4 +451,22 @@ class Server extends BaseModel } } } + public function stopApplicationRelatedRunningContainers(string $applicationId, string $containerName) + { + $containers = getCurrentApplicationContainerStatus($this, $applicationId, 0); + $containers = $containers->filter(function ($container) use ($containerName) { + return data_get($container, 'Names') !== $containerName; + }); + $containers->each(function ($container) { + $removableContainer = data_get($container, 'Names'); + $this->server->executeRemoteCommand( + commands: collect([ + 'command' => "docker rm -f $removableContainer >/dev/null 2>&1", + 'hidden' => true, + 'ignoreErrors' => true + ]), + loggingModel: $this->deploymentQueueEntry + ); + }); + } } diff --git a/bootstrap/helpers/applications.php b/bootstrap/helpers/applications.php index 1cead8a54..17c3aa486 100644 --- a/bootstrap/helpers/applications.php +++ b/bootstrap/helpers/applications.php @@ -37,7 +37,9 @@ function queue_application_deployment(int $application_id, string $deployment_uu return; } // New deployment - // dispatchDeploymentJob($deployment->id); + // dispatchDeploymentJob($deployment); + + // Old deployment dispatch(new ApplicationDeploymentJob( application_deployment_queue_id: $deployment->id, ))->onConnection('long-running')->onQueue('long-running'); @@ -48,29 +50,34 @@ function queue_next_deployment(Application $application) { $next_found = ApplicationDeploymentQueue::where('application_id', $application->id)->where('status', 'queued')->first(); if ($next_found) { - // New deployment + // New deployment // dispatchDeploymentJob($next_found->id); + + // Old deployment dispatch(new ApplicationDeploymentJob( application_deployment_queue_id: $next_found->id, ))->onConnection('long-running')->onQueue('long-running'); - } } -function dispatchDeploymentJob($id) +function dispatchDeploymentJob(ApplicationDeploymentQueue $deploymentQueueEntry) { - $applicationQueue = ApplicationDeploymentQueue::find($id); - $application = Application::find($applicationQueue->application_id); + $application = Application::find($deploymentQueueEntry->application_id); - $isRestartOnly = data_get($applicationQueue, 'restart_only'); + $isRestartOnly = data_get($deploymentQueueEntry, 'restart_only'); $isSimpleDockerFile = data_get($application, 'dockerfile'); $isDockerImage = data_get($application, 'build_pack') === 'dockerimage'; - if ($isRestartOnly) { - ApplicationRestartJob::dispatch(applicationDeploymentQueueId: $id)->onConnection('long-running')->onQueue('long-running'); - } else if ($isSimpleDockerFile) { - ApplicationDeploySimpleDockerfileJob::dispatch(applicationDeploymentQueueId: $id)->onConnection('long-running')->onQueue('long-running'); - } else if ($isDockerImage) { - ApplicationDeployDockerImageJob::dispatch(applicationDeploymentQueueId: $id)->onConnection('long-running')->onQueue('long-running'); + // if ($isRestartOnly) { + // ApplicationRestartJob::dispatch(queue: $deploymentQueueEntry, application: $application)->onConnection('long-running')->onQueue('long-running'); + // } else if ($isSimpleDockerFile) { + // ApplicationDeploySimpleDockerfileJob::dispatch(applicationDeploymentQueueId: $id)->onConnection('long-running')->onQueue('long-running'); + // } else + + if ($isDockerImage) { + ApplicationDeployDockerImageJob::dispatch( + deploymentQueueEntry: $deploymentQueueEntry, + application: $application + )->onConnection('long-running')->onQueue('long-running'); } else { throw new Exception('Unknown build pack'); } @@ -201,7 +208,6 @@ function generateComposeFile(string $deploymentUuid, Server $server, string $net ]; } if ($application->settings->is_gpu_enabled) { - ray('asd'); $docker_compose['services'][$containerName]['deploy']['resources']['reservations']['devices'] = [ [ 'driver' => data_get($application, 'settings.gpu_driver', 'nvidia'), @@ -302,39 +308,37 @@ function generateEnvironmentVariables(Application $application, $ports, int $pul return $environment_variables->all(); } -function rollingUpdate(Application $application, string $deploymentUuid) +function startNewApplication(Application $application, string $deploymentUuid, ApplicationDeploymentQueue $loggingModel) { $commands = collect([]); $workDir = generateWorkdir($deploymentUuid, $application); - if (count($application->ports_mappings_array) > 0) { - // $this->execute_remote_command( - // [ - // "echo '\n----------------------------------------'", - // ], - // ["echo -n 'Application has ports mapped to the host system, rolling update is not supported.'"], - // ); - // $this->stop_running_container(force: true); - // $this->start_by_compose_file(); + if ($application->build_pack === 'dockerimage') { + $loggingModel->addLogEntry('Pulling latest images from the registry.'); + $commands->push( + [ + "command" => executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} pull"), + "hidden" => true + ], + [ + "command" => executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} up --build -d"), + "hidden" => true + ], + ); } else { $commands->push( [ - "command" => "echo '\n----------------------------------------'" + "command" => executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} up --build -d"), + "hidden" => true ], - [ - "command" => "echo -n 'Rolling update started.'" - ] ); - if ($application->build_pack === 'dockerimage') { - $commands->push( - ["echo -n 'Pulling latest images from the registry.'"], - [executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} pull"), "hidden" => true], - [executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} up --build -d"), "hidden" => true], - ); - } else { - $commands->push( - [executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} up --build -d"), "hidden" => true], - ); - } - return $commands; } + return $commands; +} +function removeOldDeployment(string $containerName) +{ + $commands = collect([]); + $commands->push( + ["docker rm -f $containerName >/dev/null 2>&1"], + ); + return $commands; } diff --git a/bootstrap/helpers/remoteProcess.php b/bootstrap/helpers/remoteProcess.php index 948e47329..4f755c89e 100644 --- a/bootstrap/helpers/remoteProcess.php +++ b/bootstrap/helpers/remoteProcess.php @@ -165,9 +165,9 @@ function decode_remote_command_output(?ApplicationDeploymentQueue $application_d $formatted = $formatted->filter(fn ($i) => $i['hidden'] === false ?? false); } $formatted = $formatted - ->sortBy(fn ($i) => $i['order']) + ->sortBy(fn ($i) => data_get($i, 'order')) ->map(function ($i) { - $i['timestamp'] = Carbon::parse($i['timestamp'])->format('Y-M-d H:i:s.u'); + data_set($i, 'timestamp', Carbon::parse(data_get($i, 'timestamp'))->format('Y-M-d H:i:s.u')); return $i; }); return $formatted; diff --git a/resources/views/livewire/project/application/deployment-logs.blade.php b/resources/views/livewire/project/application/deployment-logs.blade.php index 09cb8232e..6afabbe13 100644 --- a/resources/views/livewire/project/application/deployment-logs.blade.php +++ b/resources/views/livewire/project/application/deployment-logs.blade.php @@ -49,7 +49,7 @@
$line['hidden'], - 'text-error' => $line['type'] == 'stderr', + 'text-red-500' => $line['type'] == 'stderr', ])>[{{ $line['timestamp'] }}] @if ($line['hidden'])
COMMAND:
{{ $line['command'] }}

OUTPUT: @endif{{ $line['output'] }}@if ($line['hidden'])