Shiloh/lasthourcloud
Shiloh/lasthourcloud
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 4 Wiki Activity

wip

Browse Source
This commit is contained in:
Andras Bacsai 2023-11-23 21:02:30 +01:00
parent 96327af838
commit d1e10dacc0
7 changed files with 214 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

122
app/Jobs/ApplicationDeployDockerImageJob.php
View File

@ -5,8 +5,8 @@ namespace App\Jobs;
use App\Enums\ApplicationDeploymentStatus;
use App\Models\Application;
use App\Models\ApplicationDeploymentQueue;
use App\Models\Server;
use App\Traits\ExecuteRemoteCommandNew;
use Exception;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
@ -21,77 +21,117 @@ class ApplicationDeployDockerImageJob implements ShouldQueue, ShouldBeEncrypted
public $timeout = 3600;
public $tries = 1;
public string $applicationDeploymentQueueId;
public $remoteCommandOutputs = [];
public Server $server;
public string $containerName;
public function __construct(string $applicationDeploymentQueueId)
public function __construct(public ApplicationDeploymentQueue $deploymentQueueEntry, public Application $application)
{
$this->applicationDeploymentQueueId = $applicationDeploymentQueueId;
}
public function handle()
{
ray()->clearAll();
ray('Deploying Docker Image');
static::$batch_counter = 0;
try {
$applicationDeploymentQueue = ApplicationDeploymentQueue::find($this->applicationDeploymentQueueId);
$deploymentUuid = data_get($this->deploymentQueueEntry, 'deployment_uuid');
$pullRequestId = data_get($this->deploymentQueueEntry, 'pull_request_id');
$deploymentUuid = data_get($applicationDeploymentQueue, 'deployment_uuid');
$pullRequestId = data_get($applicationDeploymentQueue, 'pull_request_id');
$this->server = data_get($this->application->destination, 'server');
$network = data_get($this->application->destination, 'network');
$application = Application::find($applicationDeploymentQueue->application_id)->firstOrFail();
$server = data_get($application->destination, 'server');
$network = data_get($application->destination, 'network');
$dockerImage = data_get($application, 'docker_registry_image_name');
$dockerImageTag = data_get($application, 'docker_registry_image_tag');
$dockerImage = data_get($this->application, 'docker_registry_image_name');
$dockerImageTag = data_get($this->application, 'docker_registry_image_tag');
$productionImageName = str("{$dockerImage}:{$dockerImageTag}");
$containerName = generateApplicationContainerName($application, $pullRequestId);
savePrivateKeyToFs($server);
$this->containerName = generateApplicationContainerName($this->application, $pullRequestId);
savePrivateKeyToFs($this->server);
ray("echo 'Starting deployment of {$productionImageName}.'");
$applicationDeploymentQueue->update([
$this->deploymentQueueEntry->update([
'status' => ApplicationDeploymentStatus::IN_PROGRESS->value,
]);
$server->executeRemoteCommand(
commands: prepareHelperContainer($server, $network, $deploymentUuid),
loggingModel: $applicationDeploymentQueue
$this->deploymentQueueEntry->addLogEntry('Starting deployment of ' . $productionImageName);
$this->server->executeRemoteCommand(
commands: collect(
[
[
"name" => "ls",
"command" => 'ls -la',
"hidden" => true,
],
[
"name" => "pwd",
"command" => 'pwd',
"hidden" => true,
]
],
),
loggingModel: $this->deploymentQueueEntry
);
$server->executeRemoteCommand(
$this->server->executeRemoteCommand(
commands: prepareHelperContainer($this->server, $network, $deploymentUuid),
loggingModel: $this->deploymentQueueEntry
);
$this->server->executeRemoteCommand(
commands: generateComposeFile(
deploymentUuid: $deploymentUuid,
server: $server,
server: $this->server,
network: $network,
application: $application,
containerName: $containerName,
application: $this->application,
containerName: $this->containerName,
imageName: $productionImageName,
pullRequestId: $pullRequestId
),
loggingModel: $applicationDeploymentQueue
loggingModel: $this->deploymentQueueEntry
);
$server->executeRemoteCommand(
commands: rollingUpdate(application: $application, deploymentUuid: $deploymentUuid),
loggingModel: $applicationDeploymentQueue
);
$applicationDeploymentQueue->update([
$this->deploymentQueueEntry->addLogEntry('----------------------------------------');
// Rolling update not possible
if (count($this->application->ports_mappings_array) > 0) {
$this->deploymentQueueEntry->addLogEntry('Application has ports mapped to the host system, rolling update is not supported.');
$this->deploymentQueueEntry->addLogEntry('Stopping running container.');
$this->server->stopApplicationRelatedRunningContainers($this->application->id, $this->containerName);
} else {
$this->deploymentQueueEntry->addLogEntry('Rolling update started.');
// TODO
$this->server->executeRemoteCommand(
commands: startNewApplication(application: $this->application, deploymentUuid: $deploymentUuid, loggingModel: $this->deploymentQueueEntry),
loggingModel: $this->deploymentQueueEntry
);
// $this->server->executeRemoteCommand(
// commands: healthCheckContainer(application: $this->application, containerName: $this->containerName , loggingModel: $this->deploymentQueueEntry),
// loggingModel: $this->deploymentQueueEntry
// );
}
ray($this->remoteCommandOutputs);
$this->deploymentQueueEntry->update([
'status' => ApplicationDeploymentStatus::FINISHED->value,
]);
} catch (Throwable $e) {
$this->executeRemoteCommand(
server: $server,
logModel: $applicationDeploymentQueue,
commands: [
"echo 'Oops something is not okay, are you okay? 😢'",
"echo '{$e->getMessage()}'",
"echo -n 'Deployment failed. Removing the new version of your application.'",
executeInDocker($deploymentUuid, "docker rm -f $containerName >/dev/null 2>&1"),
]
);
// $this->next(ApplicationDeploymentStatus::FAILED->value);
$this->fail($e);
throw $e;
}
}
public function failed(Throwable $exception): void
{
$this->deploymentQueueEntry->addLogEntry('Oops something is not okay, are you okay? 😢', 'error');
$this->deploymentQueueEntry->addLogEntry($exception->getMessage(), 'error');
$this->deploymentQueueEntry->addLogEntry('Deployment failed. Removing the new version of your application.');
$this->server->executeRemoteCommand(
commands: removeOldDeployment($this->containerName),
loggingModel: $this->deploymentQueueEntry
);
$this->deploymentQueueEntry->update([
'status' => ApplicationDeploymentStatus::FAILED->value,
]);
}
// private function next(string $status)
// {
// // If the deployment is cancelled by the user, don't update the status

16
app/Models/Application.php
View File

@ -342,4 +342,20 @@ class Application extends BaseModel
}
return false;
}
public function healthCheckUrl() {
if ($this->dockerfile || $this->build_pack === 'dockerfile' || $this->build_pack === 'dockerimage') {
return null;
}
if (!$this->health_check_port) {
$health_check_port = $this->ports_exposes_array[0];
} else {
$health_check_port = $this->health_check_port;
}
if ($this->health_check_path) {
$full_healthcheck_url = "{$this->health_check_scheme}://{$this->health_check_host}:{$health_check_port}{$this->health_check_path}";
} else {
$full_healthcheck_url = "{$this->health_check_scheme}://{$this->health_check_host}:{$health_check_port}/";
}
return $full_healthcheck_url;
}
}

35
app/Models/ApplicationDeploymentQueue.php
View File

@ -3,8 +3,43 @@
namespace App\Models;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Support\Carbon;
class ApplicationDeploymentQueue extends Model
{
protected $guarded = [];
public function getOutput($name) {
if (!$this->logs) {
return null;
}
return collect(json_decode($this->logs))->where('name', $name)->first()?->output ?? null;
}
public function addLogEntry(string $message, string $type = 'stdout', bool $hidden = false)
{
if ($type === 'error') {
$type = 'stderr';
}
$newLogEntry = [
'command' => null,
'output' => $message,
'type' => $type,
'timestamp' => Carbon::now('UTC'),
'hidden' => $hidden,
'batch' => 1,
];
if ($this->logs) {
$previousLogs = json_decode($this->logs, associative: true, flags: JSON_THROW_ON_ERROR);
$newLogEntry['order'] = count($previousLogs) + 1;
$previousLogs[] = $newLogEntry;
$this->update([
'logs' => json_encode($previousLogs, flags: JSON_THROW_ON_ERROR),
]);
} else {
$this->update([
'logs' => json_encode([$newLogEntry], flags: JSON_THROW_ON_ERROR),
]);
}
}
}

50
app/Models/Server.php
View File

@ -396,23 +396,23 @@ class Server extends BaseModel
{
static::$batch_counter++;
foreach ($commands as $command) {
$command = data_get($command, 'command') ?? $command[0] ?? null;
if (is_null($command)) {
continue;
$realCommand = data_get($command, 'command');
if (is_null($realCommand)) {
throw new \RuntimeException('Command is not set');
}
$hidden = data_get($command, 'hidden', false);
$ignoreErrors = data_get($command, 'ignoreErrors', false);
$customOutputType = data_get($command, 'customOutputType');
$saveOutput = data_get($command, 'saveOutput');
$remoteCommand = generateSshCommand($this, $command);
$name = data_get($command, 'name');
$remoteCommand = generateSshCommand($this, $realCommand);
$process = Process::timeout(3600)->idleTimeout(3600)->start($remoteCommand, function (string $type, string $output) use ($command, $hidden, $customOutputType, $loggingModel, $saveOutput) {
$process = Process::timeout(3600)->idleTimeout(3600)->start($remoteCommand, function (string $type, string $output) use ($realCommand, $hidden, $customOutputType, $loggingModel, $name) {
$output = str($output)->trim();
if ($output->startsWith('╔')) {
$output = "\n" . $output;
}
$newLogEntry = [
'command' => remove_iip($command),
'command' => remove_iip($realCommand),
'output' => remove_iip($output),
'type' => $customOutputType ?? $type === 'err' ? 'stderr' : 'stdout',
'timestamp' => Carbon::now('UTC'),
@ -420,19 +420,21 @@ class Server extends BaseModel
'batch' => static::$batch_counter,
];
if (!$loggingModel->logs) {
$new_log_entry['order'] = 1;
$newLogEntry['order'] = 1;
} else {
$previous_logs = json_decode($this->log_model->logs, associative: true, flags: JSON_THROW_ON_ERROR);
$new_log_entry['order'] = count($previous_logs) + 1;
$previousLogs = json_decode($loggingModel->logs, associative: true, flags: JSON_THROW_ON_ERROR);
$newLogEntry['order'] = count($previousLogs) + 1;
}
$previousLogs = json_decode($loggingModel->logs, associative: true, flags: JSON_THROW_ON_ERROR);
$newLogEntry['order'] = count($previousLogs) + 1;
if ($name) {
$newLogEntry['name'] = $name;
}
$previousLogs[] = $newLogEntry;
$loggingModel->logs = json_encode($previousLogs, flags: JSON_THROW_ON_ERROR);
$loggingModel->save();
if ($saveOutput) {
$this->remoteCommandOutputs[$saveOutput] = str($output)->trim();
}
// if ($name) {
// $loggingModel['savedOutputs'][$name] = str($output)->trim();
// }
});
$loggingModel->update([
'current_process_id' => $process->id(),
@ -449,4 +451,22 @@ class Server extends BaseModel
}
}
}
public function stopApplicationRelatedRunningContainers(string $applicationId, string $containerName)
{
$containers = getCurrentApplicationContainerStatus($this, $applicationId, 0);
$containers = $containers->filter(function ($container) use ($containerName) {
return data_get($container, 'Names') !== $containerName;
});
$containers->each(function ($container) {
$removableContainer = data_get($container, 'Names');
$this->server->executeRemoteCommand(
commands: collect([
'command' => "docker rm -f $removableContainer >/dev/null 2>&1",
'hidden' => true,
'ignoreErrors' => true
]),
loggingModel: $this->deploymentQueueEntry
);
});
}
}

84
bootstrap/helpers/applications.php
View File

@ -37,7 +37,9 @@ function queue_application_deployment(int $application_id, string $deployment_uu
return;
}
// New deployment
// dispatchDeploymentJob($deployment->id);
// dispatchDeploymentJob($deployment);
// Old deployment
dispatch(new ApplicationDeploymentJob(
application_deployment_queue_id: $deployment->id,
))->onConnection('long-running')->onQueue('long-running');
@ -48,29 +50,34 @@ function queue_next_deployment(Application $application)
{
$next_found = ApplicationDeploymentQueue::where('application_id', $application->id)->where('status', 'queued')->first();
if ($next_found) {
// New deployment
// New deployment
// dispatchDeploymentJob($next_found->id);
// Old deployment
dispatch(new ApplicationDeploymentJob(
application_deployment_queue_id: $next_found->id,
))->onConnection('long-running')->onQueue('long-running');
}
}
function dispatchDeploymentJob($id)
function dispatchDeploymentJob(ApplicationDeploymentQueue $deploymentQueueEntry)
{
$applicationQueue = ApplicationDeploymentQueue::find($id);
$application = Application::find($applicationQueue->application_id);
$application = Application::find($deploymentQueueEntry->application_id);
$isRestartOnly = data_get($applicationQueue, 'restart_only');
$isRestartOnly = data_get($deploymentQueueEntry, 'restart_only');
$isSimpleDockerFile = data_get($application, 'dockerfile');
$isDockerImage = data_get($application, 'build_pack') === 'dockerimage';
if ($isRestartOnly) {
ApplicationRestartJob::dispatch(applicationDeploymentQueueId: $id)->onConnection('long-running')->onQueue('long-running');
} else if ($isSimpleDockerFile) {
ApplicationDeploySimpleDockerfileJob::dispatch(applicationDeploymentQueueId: $id)->onConnection('long-running')->onQueue('long-running');
} else if ($isDockerImage) {
ApplicationDeployDockerImageJob::dispatch(applicationDeploymentQueueId: $id)->onConnection('long-running')->onQueue('long-running');
// if ($isRestartOnly) {
// ApplicationRestartJob::dispatch(queue: $deploymentQueueEntry, application: $application)->onConnection('long-running')->onQueue('long-running');
// } else if ($isSimpleDockerFile) {
// ApplicationDeploySimpleDockerfileJob::dispatch(applicationDeploymentQueueId: $id)->onConnection('long-running')->onQueue('long-running');
// } else
if ($isDockerImage) {
ApplicationDeployDockerImageJob::dispatch(
deploymentQueueEntry: $deploymentQueueEntry,
application: $application
)->onConnection('long-running')->onQueue('long-running');
} else {
throw new Exception('Unknown build pack');
}
@ -201,7 +208,6 @@ function generateComposeFile(string $deploymentUuid, Server $server, string $net
];
}
if ($application->settings->is_gpu_enabled) {
ray('asd');
$docker_compose['services'][$containerName]['deploy']['resources']['reservations']['devices'] = [
[
'driver' => data_get($application, 'settings.gpu_driver', 'nvidia'),
@ -302,39 +308,37 @@ function generateEnvironmentVariables(Application $application, $ports, int $pul
return $environment_variables->all();
}
function rollingUpdate(Application $application, string $deploymentUuid)
function startNewApplication(Application $application, string $deploymentUuid, ApplicationDeploymentQueue $loggingModel)
{
$commands = collect([]);
$workDir = generateWorkdir($deploymentUuid, $application);
if (count($application->ports_mappings_array) > 0) {
// $this->execute_remote_command(
// [
// "echo '\n----------------------------------------'",
// ],
// ["echo -n 'Application has ports mapped to the host system, rolling update is not supported.'"],
// );
// $this->stop_running_container(force: true);
// $this->start_by_compose_file();
if ($application->build_pack === 'dockerimage') {
$loggingModel->addLogEntry('Pulling latest images from the registry.');
$commands->push(
[
"command" => executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} pull"),
"hidden" => true
],
[
"command" => executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} up --build -d"),
"hidden" => true
],
);
} else {
$commands->push(
[
"command" => "echo '\n----------------------------------------'"
"command" => executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} up --build -d"),
"hidden" => true
],
[
"command" => "echo -n 'Rolling update started.'"
]
);
if ($application->build_pack === 'dockerimage') {
$commands->push(
["echo -n 'Pulling latest images from the registry.'"],
[executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} pull"), "hidden" => true],
[executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} up --build -d"), "hidden" => true],
);
} else {
$commands->push(
[executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} up --build -d"), "hidden" => true],
);
}
return $commands;
}
return $commands;
}
function removeOldDeployment(string $containerName)
{
$commands = collect([]);
$commands->push(
["docker rm -f $containerName >/dev/null 2>&1"],
);
return $commands;
}

4
bootstrap/helpers/remoteProcess.php
View File

@ -165,9 +165,9 @@ function decode_remote_command_output(?ApplicationDeploymentQueue $application_d
$formatted = $formatted->filter(fn ($i) => $i['hidden'] === false ?? false);
}
$formatted = $formatted
->sortBy(fn ($i) => $i['order'])
->sortBy(fn ($i) => data_get($i, 'order'))
->map(function ($i) {
$i['timestamp'] = Carbon::parse($i['timestamp'])->format('Y-M-d H:i:s.u');
data_set($i, 'timestamp', Carbon::parse(data_get($i, 'timestamp'))->format('Y-M-d H:i:s.u'));
return $i;
});
return $formatted;

2
resources/views/livewire/project/application/deployment-logs.blade.php
View File

@ -49,7 +49,7 @@
<div @class([
'font-mono whitespace-pre-line',
'text-warning' => $line['hidden'],
'text-error' => $line['type'] == 'stderr',
'text-red-500' => $line['type'] == 'stderr',
])>[{{ $line['timestamp'] }}] @if ($line['hidden'])
<br>COMMAND: <br>{{ $line['command'] }} <br><br>OUTPUT:
@endif{{ $line['output'] }}@if ($line['hidden'])