Shiloh/lasthourcloud
Shiloh/lasthourcloud
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 4 Wiki Activity

wip: new deployment jobs

Browse Source
This commit is contained in:
Andras Bacsai 2023-11-21 22:17:35 +01:00
parent ef7fc1b260
commit ce26127705
9 changed files with 1716 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

111
app/Jobs/ApplicationDeployDockerImageJob.php Normal file
View File

@ -0,0 +1,111 @@
<?php
namespace App\Jobs;
use App\Enums\ApplicationDeploymentStatus;
use App\Models\Application;
use App\Models\ApplicationDeploymentQueue;
use App\Traits\ExecuteRemoteCommandNew;
use Exception;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Throwable;
class ApplicationDeployDockerImageJob implements ShouldQueue, ShouldBeEncrypted
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, ExecuteRemoteCommandNew;
public $timeout = 3600;
public $tries = 1;
public string $applicationDeploymentQueueId;
public function __construct(string $applicationDeploymentQueueId)
{
$this->applicationDeploymentQueueId = $applicationDeploymentQueueId;
}
public function handle()
{
ray()->clearAll();
ray('Deploying Docker Image');
try {
$applicationDeploymentQueue = ApplicationDeploymentQueue::find($this->applicationDeploymentQueueId);
$application = Application::find($applicationDeploymentQueue->application_id);
$deploymentUuid = data_get($applicationDeploymentQueue, 'deployment_uuid');
$dockerImage = data_get($application, 'docker_registry_image_name');
$dockerImageTag = data_get($application, 'docker_registry_image_tag');
$productionImageName = str("{$dockerImage}:{$dockerImageTag}");
$destination = $application->destination->getMorphClass()::where('id', $application->destination->id)->first();
$pullRequestId = data_get($applicationDeploymentQueue, 'pull_request_id');
$server = data_get($destination, 'server');
$network = data_get($destination, 'network');
$containerName = generateApplicationContainerName($application, $pullRequestId);
savePrivateKeyToFs($server);
ray("echo 'Starting deployment of {$productionImageName}.'");
$applicationDeploymentQueue->update([
'status' => ApplicationDeploymentStatus::IN_PROGRESS->value,
]);
$this->executeRemoteCommand(
server: $server,
logModel: $applicationDeploymentQueue,
commands: prepareHelperContainer($server, $network, $deploymentUuid)
);
$this->executeRemoteCommand(
server: $server,
logModel: $applicationDeploymentQueue,
commands: generateComposeFile(
deploymentUuid: $deploymentUuid,
server: $server,
network: $network,
application: $application,
containerName: $containerName,
imageName: $productionImageName,
pullRequestId: $pullRequestId
)
);
$this->executeRemoteCommand(
server: $server,
logModel: $applicationDeploymentQueue,
commands: rollingUpdate(application: $application, deploymentUuid: $deploymentUuid)
);
} catch (Throwable $e) {
$this->executeRemoteCommand(
server: $server,
logModel: $applicationDeploymentQueue,
commands: [
"echo 'Oops something is not okay, are you okay? 😢'",
"echo '{$e->getMessage()}'",
"echo -n 'Deployment failed. Removing the new version of your application.'",
executeInDocker($deploymentUuid, "docker rm -f $containerName >/dev/null 2>&1"),
]
);
// $this->next(ApplicationDeploymentStatus::FAILED->value);
throw $e;
}
}
// private function next(string $status)
// {
// // If the deployment is cancelled by the user, don't update the status
// if ($this->application_deployment_queue->status !== ApplicationDeploymentStatus::CANCELLED_BY_USER->value) {
// $this->application_deployment_queue->update([
// 'status' => $status,
// ]);
// }
// queue_next_deployment($this->application);
// if ($status === ApplicationDeploymentStatus::FINISHED->value) {
// $this->application->environment->project->team->notify(new DeploymentSuccess($this->application, $this->deployment_uuid, $this->preview));
// }
// if ($status === ApplicationDeploymentStatus::FAILED->value) {
// $this->application->environment->project->team->notify(new DeploymentFailed($this->application, $this->deployment_uuid, $this->preview));
// }
// }
}

29
app/Jobs/ApplicationDeploySimpleDockerfileJob.php Normal file
View File

@ -0,0 +1,29 @@
<?php
namespace App\Jobs;
use App\Traits\ExecuteRemoteCommand;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
class ApplicationDeploySimpleDockerfileJob implements ShouldQueue, ShouldBeEncrypted
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, ExecuteRemoteCommand;
public $timeout = 3600;
public $tries = 1;
public string $applicationDeploymentQueueId;
public function __construct(string $applicationDeploymentQueueId)
{
$this->applicationDeploymentQueueId = $applicationDeploymentQueueId;
}
public function handle() {
ray('Deploying Simple Dockerfile');
}
}

28
app/Jobs/ApplicationRestartJob.php Normal file
View File

@ -0,0 +1,28 @@
<?php
namespace App\Jobs;
use App\Traits\ExecuteRemoteCommand;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
class ApplicationRestartJob implements ShouldQueue, ShouldBeEncrypted
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, ExecuteRemoteCommand;
public $timeout = 3600;
public $tries = 1;
public string $applicationDeploymentQueueId;
public function __construct(string $applicationDeploymentQueueId)
{
$this->applicationDeploymentQueueId = $applicationDeploymentQueueId;
}
public function handle() {
ray('Restarting application');
}
}

1165
app/Jobs/MultipleApplicationDeploymentJob.php Normal file
View File

File diff suppressed because it is too large Load Diff

10
app/Models/Application.php
View File

@ -332,4 +332,14 @@ class Application extends BaseModel
return true;
}
}
public function isMultipleServerDeployment()
{
if (isDev()) {
return true;
}
if (data_get($this, 'additional_destinations') && data_get($this, 'docker_registry_image_name')) {
return true;
}
return false;
}
}

2
app/Models/Server.php
View File

@ -171,7 +171,7 @@ class Server extends BaseModel
break;
}
$result = $this->validateConnection();
ray('validateConnection: ' . $result);
// ray('validateConnection: ' . $result);
if (!$result) {
$serverUptimeCheckNumber++;
$this->update([

1
app/Traits/ExecuteRemoteCommand.php
View File

@ -12,7 +12,6 @@ use Illuminate\Support\Str;
trait ExecuteRemoteCommand
{
public ?string $save = null;
public function execute_remote_command(...$commands)
{
static::$batch_counter++;

77
app/Traits/ExecuteRemoteCommandNew.php Normal file
View File

@ -0,0 +1,77 @@
<?php
namespace App\Traits;
use App\Enums\ApplicationDeploymentStatus;
use App\Models\Server;
use Carbon\Carbon;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Process;
trait ExecuteRemoteCommandNew
{
public static $batch_counter = 0;
public function executeRemoteCommand(Server $server, $logModel, $commands)
{
static::$batch_counter++;
if ($commands instanceof Collection) {
$commandsText = $commands;
} else {
$commandsText = collect($commands);
}
$commandsText->each(function ($singleCommand) use ($server, $logModel) {
$command = data_get($singleCommand, 'command') ?? $singleCommand[0] ?? null;
if ($command === null) {
throw new \RuntimeException('Command is not set');
}
$hidden = data_get($singleCommand, 'hidden', false);
$customType = data_get($singleCommand, 'type');
$ignoreErrors = data_get($singleCommand, 'ignore_errors', false);
$save = data_get($singleCommand, 'save');
$remote_command = generateSshCommand($server, $command);
$process = Process::timeout(3600)->idleTimeout(3600)->start($remote_command, function (string $type, string $output) use ($command, $hidden, $customType, $logModel, $save) {
$output = str($output)->trim();
if ($output->startsWith('╔')) {
$output = "\n" . $output;
}
$newLogEntry = [
'command' => remove_iip($command),
'output' => remove_iip($output),
'type' => $customType ?? $type === 'err' ? 'stderr' : 'stdout',
'timestamp' => Carbon::now('UTC'),
'hidden' => $hidden,
'batch' => static::$batch_counter,
];
if (!$logModel->logs) {
$newLogEntry['order'] = 1;
} else {
$previousLogs = json_decode($logModel->logs, associative: true, flags: JSON_THROW_ON_ERROR);
$newLogEntry['order'] = count($previousLogs) + 1;
}
$previousLogs[] = $newLogEntry;
$logModel->logs = json_encode($previousLogs, flags: JSON_THROW_ON_ERROR);
$logModel->save();
if ($save) {
$this->remoteCommandOutputs[$save] = str($output)->trim();
}
});
$logModel->update([
'current_process_id' => $process->id(),
]);
$processResult = $process->wait();
if ($processResult->exitCode() !== 0) {
if (!$ignoreErrors) {
$status = ApplicationDeploymentStatus::FAILED->value;
$logModel->status = $status;
$logModel->save();
throw new \RuntimeException($processResult->errorOutput());
}
}
});
}
}

295
bootstrap/helpers/applications.php
View File

@ -1,8 +1,15 @@
<?php
use App\Jobs\ApplicationDeployDockerImageJob;
use App\Jobs\ApplicationDeploymentJob;
use App\Jobs\ApplicationDeploySimpleDockerfileJob;
use App\Jobs\ApplicationRestartJob;
use App\Jobs\MultipleApplicationDeploymentJob;
use App\Models\Application;
use App\Models\ApplicationDeploymentQueue;
use App\Models\ApplicationPreview;
use App\Models\Server;
use Symfony\Component\Yaml\Yaml;
function queue_application_deployment(int $application_id, string $deployment_uuid, int | null $pull_request_id = 0, string $commit = 'HEAD', bool $force_rebuild = false, bool $is_webhook = false, bool $restart_only = false, ?string $git_type = null)
{
@ -29,17 +36,305 @@ function queue_application_deployment(int $application_id, string $deployment_uu
if ($running_deployments->count() > 0) {
return;
}
// New deployment
// dispatchDeploymentJob($deployment->id);
dispatch(new ApplicationDeploymentJob(
application_deployment_queue_id: $deployment->id,
))->onConnection('long-running')->onQueue('long-running');
}
function queue_next_deployment(Application $application)
{
$next_found = ApplicationDeploymentQueue::where('application_id', $application->id)->where('status', 'queued')->first();
if ($next_found) {
// New deployment
// dispatchDeploymentJob($next_found->id);
dispatch(new ApplicationDeploymentJob(
application_deployment_queue_id: $next_found->id,
))->onConnection('long-running')->onQueue('long-running');
}
}
function dispatchDeploymentJob($id)
{
$applicationQueue = ApplicationDeploymentQueue::find($id);
$application = Application::find($applicationQueue->application_id);
$isRestartOnly = data_get($applicationQueue, 'restart_only');
$isSimpleDockerFile = data_get($application, 'dockerfile');
$isDockerImage = data_get($application, 'build_pack') === 'dockerimage';
if ($isRestartOnly) {
ApplicationRestartJob::dispatch(applicationDeploymentQueueId: $id)->onConnection('long-running')->onQueue('long-running');
} else if ($isSimpleDockerFile) {
ApplicationDeploySimpleDockerfileJob::dispatch(applicationDeploymentQueueId: $id)->onConnection('long-running')->onQueue('long-running');
} else if ($isDockerImage) {
ApplicationDeployDockerImageJob::dispatch(applicationDeploymentQueueId: $id)->onConnection('long-running')->onQueue('long-running');
} else {
throw new Exception('Unknown build pack');
}
}
// Deployment things
function generateHostIpMapping(Server $server, string $network)
{
// Generate custom host<->ip hostnames
$allContainers = instant_remote_process(["docker network inspect {$network} -f '{{json .Containers}}' "], $server);
$allContainers = format_docker_command_output_to_json($allContainers);
$ips = collect([]);
if (count($allContainers) > 0) {
$allContainers = $allContainers[0];
foreach ($allContainers as $container) {
$containerName = data_get($container, 'Name');
if ($containerName === 'coolify-proxy') {
continue;
}
$containerIp = data_get($container, 'IPv4Address');
if ($containerName && $containerIp) {
$containerIp = str($containerIp)->before('/');
$ips->put($containerName, $containerIp->value());
}
}
}
return $ips->map(function ($ip, $name) {
return "--add-host $name:$ip";
})->implode(' ');
}
function generateBaseDir(string $deplyomentUuid)
{
return "/artifacts/$deplyomentUuid";
}
function generateWorkdir(string $deplyomentUuid, Application $application)
{
return generateBaseDir($deplyomentUuid) . rtrim($application->base_directory, '/');
}
function prepareHelperContainer(Server $server, string $network, string $deploymentUuid)
{
$basedir = generateBaseDir($deploymentUuid);
$helperImage = config('coolify.helper_image');
$serverUserHomeDir = instant_remote_process(["echo \$HOME"], $server);
$dockerConfigFileExists = instant_remote_process(["test -f {$serverUserHomeDir}/.docker/config.json && echo 'OK' || echo 'NOK'"], $server);
$commands = collect([]);
if ($dockerConfigFileExists === 'OK') {
$commands->push([
"command" => "docker run -d --network $network -v /:/host --name $deploymentUuid --rm -v {$serverUserHomeDir}/.docker/config.json:/root/.docker/config.json:ro -v /var/run/docker.sock:/var/run/docker.sock $helperImage",
"hidden" => true,
]);
} else {
$commands->push([
"command" => "docker run -d --network {$network} -v /:/host --name {$deploymentUuid} --rm -v /var/run/docker.sock:/var/run/docker.sock {$helperImage}",
"hidden" => true,
]);
}
$commands->push([
"command" => executeInDocker($deploymentUuid, "mkdir -p {$basedir}"),
"hidden" => true,
]);
return $commands;
}
function generateComposeFile(string $deploymentUuid, Server $server, string $network, Application $application, string $containerName, string $imageName, ?ApplicationPreview $preview = null, int $pullRequestId = 0)
{
$ports = $application->settings->is_static ? [80] : $application->ports_exposes_array;
$workDir = generateWorkdir($deploymentUuid, $application);
$persistent_storages = generateLocalPersistentVolumes($application, $pullRequestId);
$volume_names = generateLocalPersistentVolumesOnlyVolumeNames($application, $pullRequestId);
$environment_variables = generateEnvironmentVariables($application, $ports, $pullRequestId);
if (data_get($application, 'custom_labels')) {
$labels = collect(str($application->custom_labels)->explode(','));
$labels = $labels->filter(function ($value, $key) {
return !str($value)->startsWith('coolify.');
});
$application->custom_labels = $labels->implode(',');
$application->save();
} else {
$labels = collect(generateLabelsApplication($application, $preview));
}
if ($pullRequestId !== 0) {
$labels = collect(generateLabelsApplication($application, $preview));
}
$labels = $labels->merge(defaultLabels($application->id, $application->uuid, 0))->toArray();
$docker_compose = [
'version' => '3.8',
'services' => [
$containerName => [
'image' => $imageName,
'container_name' => $containerName,
'restart' => RESTART_MODE,
'environment' => $environment_variables,
'labels' => $labels,
'expose' => $ports,
'networks' => [
$network,
],
'mem_limit' => $application->limits_memory,
'memswap_limit' => $application->limits_memory_swap,
'mem_swappiness' => $application->limits_memory_swappiness,
'mem_reservation' => $application->limits_memory_reservation,
'cpus' => (int) $application->limits_cpus,
'cpuset' => $application->limits_cpuset,
'cpu_shares' => $application->limits_cpu_shares,
]
],
'networks' => [
$network => [
'external' => true,
'name' => $network,
'attachable' => true
]
]
];
if ($server->isLogDrainEnabled() && $application->isLogDrainEnabled()) {
$docker_compose['services'][$containerName]['logging'] = [
'driver' => 'fluentd',
'options' => [
'fluentd-address' => "tcp://127.0.0.1:24224",
'fluentd-async' => "true",
'fluentd-sub-second-precision' => "true",
]
];
}
if ($application->settings->is_gpu_enabled) {
ray('asd');
$docker_compose['services'][$containerName]['deploy']['resources']['reservations']['devices'] = [
[
'driver' => data_get($application, 'settings.gpu_driver', 'nvidia'),
'capabilities' => ['gpu'],
'options' => data_get($application, 'settings.gpu_options', [])
]
];
if (data_get($application, 'settings.gpu_count')) {
$count = data_get($application, 'settings.gpu_count');
if ($count === 'all') {
$docker_compose['services'][$containerName]['deploy']['resources']['reservations']['devices'][0]['count'] = $count;
} else {
$docker_compose['services'][$containerName]['deploy']['resources']['reservations']['devices'][0]['count'] = (int) $count;
}
} else if (data_get($application, 'settings.gpu_device_ids')) {
$docker_compose['services'][$containerName]['deploy']['resources']['reservations']['devices'][0]['ids'] = data_get($application, 'settings.gpu_device_ids');
}
}
if ($application->isHealthcheckDisabled()) {
data_forget($docker_compose, 'services.' . $containerName . '.healthcheck');
}
if (count($application->ports_mappings_array) > 0 && $pullRequestId === 0) {
$docker_compose['services'][$containerName]['ports'] = $application->ports_mappings_array;
}
if (count($persistent_storages) > 0) {
$docker_compose['services'][$containerName]['volumes'] = $persistent_storages;
}
if (count($volume_names) > 0) {
$docker_compose['volumes'] = $volume_names;
}
$docker_compose = Yaml::dump($docker_compose, 10);
$docker_compose_base64 = base64_encode($docker_compose);
$commands = collect([]);
$commands->push([
"command" => executeInDocker($deploymentUuid, "echo '{$docker_compose_base64}' | base64 -d > {$workDir}/docker-compose.yml"),
"hidden" => true,
]);
return $commands;
}
function generateLocalPersistentVolumes(Application $application, int $pullRequestId = 0)
{
$local_persistent_volumes = [];
foreach ($application->persistentStorages as $persistentStorage) {
$volume_name = $persistentStorage->host_path ?? $persistentStorage->name;
if ($pullRequestId !== 0) {
$volume_name = $volume_name . '-pr-' . $pullRequestId;
}
$local_persistent_volumes[] = $volume_name . ':' . $persistentStorage->mount_path;
}
return $local_persistent_volumes;
}
function generateLocalPersistentVolumesOnlyVolumeNames(Application $application, int $pullRequestId = 0)
{
$local_persistent_volumes_names = [];
foreach ($application->persistentStorages as $persistentStorage) {
if ($persistentStorage->host_path) {
continue;
}
$name = $persistentStorage->name;
if ($pullRequestId !== 0) {
$name = $name . '-pr-' . $pullRequestId;
}
$local_persistent_volumes_names[$name] = [
'name' => $name,
'external' => false,
];
}
return $local_persistent_volumes_names;
}
function generateEnvironmentVariables(Application $application, $ports, int $pullRequestId = 0)
{
$environment_variables = collect();
// ray('Generate Environment Variables')->green();
if ($pullRequestId === 0) {
// ray($this->application->runtime_environment_variables)->green();
foreach ($application->runtime_environment_variables as $env) {
$environment_variables->push("$env->key=$env->value");
}
foreach ($application->nixpacks_environment_variables as $env) {
$environment_variables->push("$env->key=$env->value");
}
} else {
// ray($this->application->runtime_environment_variables_preview)->green();
foreach ($application->runtime_environment_variables_preview as $env) {
$environment_variables->push("$env->key=$env->value");
}
foreach ($application->nixpacks_environment_variables_preview as $env) {
$environment_variables->push("$env->key=$env->value");
}
}
// Add PORT if not exists, use the first port as default
if ($environment_variables->filter(fn ($env) => str($env)->contains('PORT'))->isEmpty()) {
$environment_variables->push("PORT={$ports[0]}");
}
return $environment_variables->all();
}
function rollingUpdate(Application $application, string $deploymentUuid)
{
$commands = collect([]);
$workDir = generateWorkdir($deploymentUuid, $application);
if (count($application->ports_mappings_array) > 0) {
// $this->execute_remote_command(
// [
// "echo '\n----------------------------------------'",
// ],
// ["echo -n 'Application has ports mapped to the host system, rolling update is not supported.'"],
// );
// $this->stop_running_container(force: true);
// $this->start_by_compose_file();
} else {
$commands->push(
[
"command" => "echo '\n----------------------------------------'"
],
[
"command" => "echo -n 'Rolling update started.'"
]
);
if ($application->build_pack === 'dockerimage') {
$commands->push(
["echo -n 'Pulling latest images from the registry.'"],
[executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} pull"), "hidden" => true],
[executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} up --build -d"), "hidden" => true],
);
} else {
$commands->push(
[executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} up --build -d"), "hidden" => true],
);
}
return $commands;
}
}