wip

app/Jobs/ApplicationDeployDockerImageJob.php

@@ -5,8 +5,8 @@ namespace App\Jobs;
 use App\Enums\ApplicationDeploymentStatus;
 use App\Models\Application;
 use App\Models\ApplicationDeploymentQueue;
+use App\Models\Server;
 use App\Traits\ExecuteRemoteCommandNew;
-use Exception;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldBeEncrypted;
 use Illuminate\Contracts\Queue\ShouldQueue;
@@ -21,77 +21,117 @@ class ApplicationDeployDockerImageJob implements ShouldQueue, ShouldBeEncrypted
 
     public $timeout = 3600;
     public $tries = 1;
-    public string $applicationDeploymentQueueId;
+    public $remoteCommandOutputs = [];
+    public Server $server;
+    public string $containerName;
 
-    public function __construct(string $applicationDeploymentQueueId)
+    public function __construct(public ApplicationDeploymentQueue $deploymentQueueEntry, public Application $application)
     {
-        $this->applicationDeploymentQueueId = $applicationDeploymentQueueId;
     }
     public function handle()
     {
         ray()->clearAll();
         ray('Deploying Docker Image');
+        static::$batch_counter = 0;
         try {
-            $applicationDeploymentQueue = ApplicationDeploymentQueue::find($this->applicationDeploymentQueueId);
+            $deploymentUuid = data_get($this->deploymentQueueEntry, 'deployment_uuid');
+            $pullRequestId = data_get($this->deploymentQueueEntry, 'pull_request_id');
 
-            $deploymentUuid = data_get($applicationDeploymentQueue, 'deployment_uuid');
+            $this->server = data_get($this->application->destination, 'server');
-            $pullRequestId = data_get($applicationDeploymentQueue, 'pull_request_id');
+            $network = data_get($this->application->destination, 'network');
 
-            $application = Application::find($applicationDeploymentQueue->application_id)->firstOrFail();
+            $dockerImage = data_get($this->application, 'docker_registry_image_name');
-            $server = data_get($application->destination, 'server');
+            $dockerImageTag = data_get($this->application, 'docker_registry_image_tag');
-            $network = data_get($application->destination, 'network');
-
-            $dockerImage = data_get($application, 'docker_registry_image_name');
-            $dockerImageTag = data_get($application, 'docker_registry_image_tag');
 
             $productionImageName = str("{$dockerImage}:{$dockerImageTag}");
-
+            $this->containerName = generateApplicationContainerName($this->application, $pullRequestId);
-            $containerName = generateApplicationContainerName($application, $pullRequestId);
+            savePrivateKeyToFs($this->server);
-            savePrivateKeyToFs($server);
 
             ray("echo 'Starting deployment of {$productionImageName}.'");
 
-            $applicationDeploymentQueue->update([
+            $this->deploymentQueueEntry->update([
                 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value,
             ]);
-            $server->executeRemoteCommand(
+
-                commands: prepareHelperContainer($server, $network, $deploymentUuid),
+            $this->deploymentQueueEntry->addLogEntry('Starting deployment of ' . $productionImageName);
-                loggingModel: $applicationDeploymentQueue
+
+            $this->server->executeRemoteCommand(
+                commands: collect(
+                    [
+                        [
+                            "name" => "ls",
+                            "command" => 'ls -la',
+                            "hidden" => true,
+                        ],
+                        [
+                            "name" => "pwd",
+                            "command" => 'pwd',
+                            "hidden" => true,
+                        ]
+                    ],
+                ),
+                loggingModel: $this->deploymentQueueEntry
             );
-            $server->executeRemoteCommand(
+            $this->server->executeRemoteCommand(
+                commands: prepareHelperContainer($this->server, $network, $deploymentUuid),
+                loggingModel: $this->deploymentQueueEntry
+            );
+            $this->server->executeRemoteCommand(
                 commands: generateComposeFile(
                     deploymentUuid: $deploymentUuid,
-                    server: $server,
+                    server: $this->server,
                     network: $network,
-                    application: $application,
+                    application: $this->application,
-                    containerName: $containerName,
+                    containerName: $this->containerName,
                     imageName: $productionImageName,
                     pullRequestId: $pullRequestId
                 ),
-                loggingModel: $applicationDeploymentQueue
+                loggingModel: $this->deploymentQueueEntry
             );
-            $server->executeRemoteCommand(
+            $this->deploymentQueueEntry->addLogEntry('----------------------------------------');
-                commands: rollingUpdate(application: $application, deploymentUuid: $deploymentUuid),
+
-                loggingModel: $applicationDeploymentQueue
+            // Rolling update not possible
-            );
+            if (count($this->application->ports_mappings_array) > 0) {
-            $applicationDeploymentQueue->update([
+                $this->deploymentQueueEntry->addLogEntry('Application has ports mapped to the host system, rolling update is not supported.');
+                $this->deploymentQueueEntry->addLogEntry('Stopping running container.');
+                $this->server->stopApplicationRelatedRunningContainers($this->application->id, $this->containerName);
+            } else {
+                $this->deploymentQueueEntry->addLogEntry('Rolling update started.');
+                // TODO
+                $this->server->executeRemoteCommand(
+                    commands: startNewApplication(application: $this->application, deploymentUuid: $deploymentUuid, loggingModel: $this->deploymentQueueEntry),
+                    loggingModel: $this->deploymentQueueEntry
+                );
+                // $this->server->executeRemoteCommand(
+                //     commands: healthCheckContainer(application: $this->application, containerName: $this->containerName , loggingModel: $this->deploymentQueueEntry),
+                //     loggingModel: $this->deploymentQueueEntry
+                // );
+
+            }
+
+            ray($this->remoteCommandOutputs);
+            $this->deploymentQueueEntry->update([
                 'status' => ApplicationDeploymentStatus::FINISHED->value,
             ]);
         } catch (Throwable $e) {
-            $this->executeRemoteCommand(
+            $this->fail($e);
-                server: $server,
-                logModel: $applicationDeploymentQueue,
-                commands: [
-                    "echo 'Oops something is not okay, are you okay? 😢'",
-                    "echo '{$e->getMessage()}'",
-                    "echo -n 'Deployment failed. Removing the new version of your application.'",
-                    executeInDocker($deploymentUuid, "docker rm -f $containerName >/dev/null 2>&1"),
-                ]
-            );
-            // $this->next(ApplicationDeploymentStatus::FAILED->value);
             throw $e;
         }
     }
+    public function failed(Throwable $exception): void
+    {
+        $this->deploymentQueueEntry->addLogEntry('Oops something is not okay, are you okay? 😢', 'error');
+        $this->deploymentQueueEntry->addLogEntry($exception->getMessage(), 'error');
+        $this->deploymentQueueEntry->addLogEntry('Deployment failed. Removing the new version of your application.');
+
+        $this->server->executeRemoteCommand(
+            commands: removeOldDeployment($this->containerName),
+            loggingModel: $this->deploymentQueueEntry
+        );
+        $this->deploymentQueueEntry->update([
+            'status' => ApplicationDeploymentStatus::FAILED->value,
+        ]);
+    }
     // private function next(string $status)
     // {
     //     // If the deployment is cancelled by the user, don't update the status

app/Models/Application.php

@@ -342,4 +342,20 @@ class Application extends BaseModel
         }
         return false;
     }
+    public function healthCheckUrl() {
+        if ($this->dockerfile || $this->build_pack === 'dockerfile' || $this->build_pack === 'dockerimage') {
+            return null;
+        }
+        if (!$this->health_check_port) {
+            $health_check_port = $this->ports_exposes_array[0];
+        } else {
+            $health_check_port = $this->health_check_port;
+        }
+        if ($this->health_check_path) {
+            $full_healthcheck_url = "{$this->health_check_scheme}://{$this->health_check_host}:{$health_check_port}{$this->health_check_path}";
+        } else {
+            $full_healthcheck_url = "{$this->health_check_scheme}://{$this->health_check_host}:{$health_check_port}/";
+        }
+        return $full_healthcheck_url;
+    }
 }

app/Models/ApplicationDeploymentQueue.php

@@ -3,8 +3,43 @@
 namespace App\Models;
 
 use Illuminate\Database\Eloquent\Model;
+use Illuminate\Support\Carbon;
 
 class ApplicationDeploymentQueue extends Model
 {
     protected $guarded = [];
+
+    public function getOutput($name) {
+        if (!$this->logs) {
+            return null;
+        }
+        return collect(json_decode($this->logs))->where('name', $name)->first()?->output ?? null;
+    }
+
+    public function addLogEntry(string $message, string $type = 'stdout', bool $hidden = false)
+    {
+        if ($type === 'error') {
+            $type = 'stderr';
+        }
+        $newLogEntry = [
+            'command' => null,
+            'output' => $message,
+            'type' => $type,
+            'timestamp' => Carbon::now('UTC'),
+            'hidden' => $hidden,
+            'batch' => 1,
+        ];
+        if ($this->logs) {
+            $previousLogs = json_decode($this->logs, associative: true, flags: JSON_THROW_ON_ERROR);
+            $newLogEntry['order'] = count($previousLogs) + 1;
+            $previousLogs[] = $newLogEntry;
+            $this->update([
+                'logs' => json_encode($previousLogs, flags: JSON_THROW_ON_ERROR),
+            ]);
+        } else {
+            $this->update([
+                'logs' => json_encode([$newLogEntry], flags: JSON_THROW_ON_ERROR),
+            ]);
+        }
+    }
 }

app/Models/Server.php

@@ -396,23 +396,23 @@ class Server extends BaseModel
     {
         static::$batch_counter++;
         foreach ($commands as $command) {
-            $command = data_get($command, 'command') ?? $command[0] ?? null;
+            $realCommand = data_get($command, 'command');
-            if (is_null($command)) {
+            if (is_null($realCommand)) {
-                continue;
+                throw new \RuntimeException('Command is not set');
             }
             $hidden = data_get($command, 'hidden', false);
             $ignoreErrors = data_get($command, 'ignoreErrors', false);
             $customOutputType = data_get($command, 'customOutputType');
-            $saveOutput = data_get($command, 'saveOutput');
+            $name = data_get($command, 'name');
-            $remoteCommand = generateSshCommand($this, $command);
+            $remoteCommand = generateSshCommand($this, $realCommand);
 
-            $process = Process::timeout(3600)->idleTimeout(3600)->start($remoteCommand, function (string $type, string $output) use ($command, $hidden, $customOutputType, $loggingModel, $saveOutput) {
+            $process = Process::timeout(3600)->idleTimeout(3600)->start($remoteCommand, function (string $type, string $output) use ($realCommand, $hidden, $customOutputType, $loggingModel, $name) {
                 $output = str($output)->trim();
                 if ($output->startsWith('╔')) {
                     $output = "\n" . $output;
                 }
                 $newLogEntry = [
-                    'command' => remove_iip($command),
+                    'command' => remove_iip($realCommand),
                     'output' => remove_iip($output),
                     'type' => $customOutputType ?? $type === 'err' ? 'stderr' : 'stdout',
                     'timestamp' => Carbon::now('UTC'),
@@ -420,19 +420,21 @@ class Server extends BaseModel
                     'batch' => static::$batch_counter,
                 ];
                 if (!$loggingModel->logs) {
-                    $new_log_entry['order'] = 1;
+                    $newLogEntry['order'] = 1;
                 } else {
-                    $previous_logs = json_decode($this->log_model->logs, associative: true, flags: JSON_THROW_ON_ERROR);
+                    $previousLogs = json_decode($loggingModel->logs, associative: true, flags: JSON_THROW_ON_ERROR);
-                    $new_log_entry['order'] = count($previous_logs) + 1;
+                    $newLogEntry['order'] = count($previousLogs) + 1;
                 }
-                $previousLogs = json_decode($loggingModel->logs, associative: true, flags: JSON_THROW_ON_ERROR);
+                if ($name) {
-                $newLogEntry['order'] = count($previousLogs) + 1;
+                    $newLogEntry['name'] = $name;
+                }
+
                 $previousLogs[] = $newLogEntry;
                 $loggingModel->logs = json_encode($previousLogs, flags: JSON_THROW_ON_ERROR);
                 $loggingModel->save();
-                if ($saveOutput) {
+                // if ($name) {
-                    $this->remoteCommandOutputs[$saveOutput] = str($output)->trim();
+                //     $loggingModel['savedOutputs'][$name] = str($output)->trim();
-                }
+                // }
             });
             $loggingModel->update([
                 'current_process_id' => $process->id(),
@@ -449,4 +451,22 @@ class Server extends BaseModel
             }
         }
     }
+    public function stopApplicationRelatedRunningContainers(string $applicationId, string $containerName)
+    {
+        $containers = getCurrentApplicationContainerStatus($this, $applicationId, 0);
+        $containers = $containers->filter(function ($container) use ($containerName) {
+            return data_get($container, 'Names') !== $containerName;
+        });
+        $containers->each(function ($container) {
+            $removableContainer = data_get($container, 'Names');
+            $this->server->executeRemoteCommand(
+                commands: collect([
+                    'command' => "docker rm -f $removableContainer >/dev/null 2>&1",
+                    'hidden' => true,
+                    'ignoreErrors' => true
+                ]),
+                loggingModel: $this->deploymentQueueEntry
+            );
+        });
+    }
 }

bootstrap/helpers/applications.php

@@ -37,7 +37,9 @@ function queue_application_deployment(int $application_id, string $deployment_uu
         return;
     }
     // New deployment
-    // dispatchDeploymentJob($deployment->id);
+    // dispatchDeploymentJob($deployment);
+
+    // Old deployment
     dispatch(new ApplicationDeploymentJob(
         application_deployment_queue_id: $deployment->id,
     ))->onConnection('long-running')->onQueue('long-running');
@@ -48,29 +50,34 @@ function queue_next_deployment(Application $application)
 {
     $next_found = ApplicationDeploymentQueue::where('application_id', $application->id)->where('status', 'queued')->first();
     if ($next_found) {
-         // New deployment
+        // New deployment
         // dispatchDeploymentJob($next_found->id);
+
+        // Old deployment
         dispatch(new ApplicationDeploymentJob(
             application_deployment_queue_id: $next_found->id,
         ))->onConnection('long-running')->onQueue('long-running');
-
     }
 }
-function dispatchDeploymentJob($id)
+function dispatchDeploymentJob(ApplicationDeploymentQueue $deploymentQueueEntry)
 {
-    $applicationQueue = ApplicationDeploymentQueue::find($id);
+    $application = Application::find($deploymentQueueEntry->application_id);
-    $application = Application::find($applicationQueue->application_id);
 
-    $isRestartOnly = data_get($applicationQueue, 'restart_only');
+    $isRestartOnly = data_get($deploymentQueueEntry, 'restart_only');
     $isSimpleDockerFile = data_get($application, 'dockerfile');
     $isDockerImage = data_get($application, 'build_pack') === 'dockerimage';
 
-    if ($isRestartOnly) {
+    // if ($isRestartOnly) {
-        ApplicationRestartJob::dispatch(applicationDeploymentQueueId: $id)->onConnection('long-running')->onQueue('long-running');
+    //     ApplicationRestartJob::dispatch(queue: $deploymentQueueEntry, application: $application)->onConnection('long-running')->onQueue('long-running');
-    } else if ($isSimpleDockerFile) {
+    // } else if ($isSimpleDockerFile) {
-        ApplicationDeploySimpleDockerfileJob::dispatch(applicationDeploymentQueueId: $id)->onConnection('long-running')->onQueue('long-running');
+    //     ApplicationDeploySimpleDockerfileJob::dispatch(applicationDeploymentQueueId: $id)->onConnection('long-running')->onQueue('long-running');
-    } else if ($isDockerImage) {
+    // } else
-        ApplicationDeployDockerImageJob::dispatch(applicationDeploymentQueueId: $id)->onConnection('long-running')->onQueue('long-running');
+
+    if ($isDockerImage) {
+        ApplicationDeployDockerImageJob::dispatch(
+            deploymentQueueEntry: $deploymentQueueEntry,
+            application: $application
+        )->onConnection('long-running')->onQueue('long-running');
     } else {
         throw new Exception('Unknown build pack');
     }
@@ -201,7 +208,6 @@ function generateComposeFile(string $deploymentUuid, Server $server, string $net
         ];
     }
     if ($application->settings->is_gpu_enabled) {
-        ray('asd');
         $docker_compose['services'][$containerName]['deploy']['resources']['reservations']['devices'] = [
             [
                 'driver' => data_get($application, 'settings.gpu_driver', 'nvidia'),
@@ -302,39 +308,37 @@ function generateEnvironmentVariables(Application $application, $ports, int $pul
     return $environment_variables->all();
 }
 
-function rollingUpdate(Application $application, string $deploymentUuid)
+function startNewApplication(Application $application, string $deploymentUuid, ApplicationDeploymentQueue $loggingModel)
 {
     $commands = collect([]);
     $workDir = generateWorkdir($deploymentUuid, $application);
-    if (count($application->ports_mappings_array) > 0) {
+    if ($application->build_pack === 'dockerimage') {
-        // $this->execute_remote_command(
+        $loggingModel->addLogEntry('Pulling latest images from the registry.');
-        //     [
+        $commands->push(
-        //         "echo '\n----------------------------------------'",
+            [
-        //     ],
+                "command" => executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} pull"),
-        //     ["echo -n 'Application has ports mapped to the host system, rolling update is not supported.'"],
+                "hidden" => true
-        // );
+            ],
-        // $this->stop_running_container(force: true);
+            [
-        // $this->start_by_compose_file();
+                "command" => executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} up --build -d"),
+                "hidden" => true
+            ],
+        );
     } else {
         $commands->push(
             [
-                "command" =>  "echo '\n----------------------------------------'"
+                "command" => executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} up --build -d"),
+                "hidden" => true
             ],
-            [
-                "command" =>  "echo -n 'Rolling update started.'"
-            ]
         );
-        if ($application->build_pack === 'dockerimage') {
-            $commands->push(
-                ["echo -n 'Pulling latest images from the registry.'"],
-                [executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} pull"), "hidden" => true],
-                [executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} up --build -d"), "hidden" => true],
-            );
-        } else {
-            $commands->push(
-                [executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} up --build -d"), "hidden" => true],
-            );
-        }
-        return $commands;
     }
+    return $commands;
+}
+function removeOldDeployment(string $containerName)
+{
+    $commands = collect([]);
+    $commands->push(
+        ["docker rm -f $containerName >/dev/null 2>&1"],
+    );
+    return $commands;
 }

bootstrap/helpers/remoteProcess.php

@@ -165,9 +165,9 @@ function decode_remote_command_output(?ApplicationDeploymentQueue $application_d
         $formatted = $formatted->filter(fn ($i) => $i['hidden'] === false ?? false);
     }
     $formatted = $formatted
-        ->sortBy(fn ($i) => $i['order'])
+        ->sortBy(fn ($i) => data_get($i, 'order'))
         ->map(function ($i) {
-            $i['timestamp'] = Carbon::parse($i['timestamp'])->format('Y-M-d H:i:s.u');
+            data_set($i, 'timestamp', Carbon::parse(data_get($i, 'timestamp'))->format('Y-M-d H:i:s.u'));
             return $i;
         });
     return $formatted;

resources/views/livewire/project/application/deployment-logs.blade.php

@@ -49,7 +49,7 @@
                         <div @class([
                             'font-mono whitespace-pre-line',
                             'text-warning' => $line['hidden'],
-                            'text-error' => $line['type'] == 'stderr',
+                            'text-red-500' => $line['type'] == 'stderr',
                         ])>[{{ $line['timestamp'] }}] @if ($line['hidden'])
                                 <br>COMMAND: <br>{{ $line['command'] }} <br><br>OUTPUT:
                                 @endif{{ $line['output'] }}@if ($line['hidden'])