diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index 34f358367..457cc04ad 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -348,9 +348,9 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted ], ); } - $this->application_deployment_queue->addLogEntry("Image pushed to docker registry.'"); + $this->application_deployment_queue->addLogEntry("Image pushed to docker registry."); } catch (Exception $e) { - $this->application_deployment_queue->addLogEntry("Failed to push image to docker registry. Please check debug logs for more information.'"); + $this->application_deployment_queue->addLogEntry("Failed to push image to docker registry. Please check debug logs for more information."); if ($forceFail) { throw $e; } @@ -488,10 +488,8 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted } else { $this->application_deployment_queue->addLogEntry("Starting pull request (#{$this->pull_request_id}) deployment of {$this->customRepository}:{$this->application->git_branch}."); } - $this->server->executeRemoteCommand( - commands: $this->application->prepareHelperImage($this->deployment_uuid), - loggingModel: $this->application_deployment_queue - ); + ray('asddf'); + $this->prepare_builder_image(); $this->check_git_if_build_needed(); $this->clone_repository(); $this->generate_image_names(); diff --git a/app/Jobs/ApplicationDeploymentNewJob.php b/app/Jobs/ApplicationDeploymentNewJob.php deleted file mode 100644 index 94b9a4e27..000000000 --- a/app/Jobs/ApplicationDeploymentNewJob.php +++ /dev/null @@ -1,180 +0,0 @@ -mainServer = data_get($this->application, 'destination.server'); - $this->deploymentUuid = data_get($this->deployment, 'deployment_uuid'); - $this->pullRequestId = data_get($this->deployment, 'pull_request_id', 0); - $this->gitType = data_get($this->deployment, 'git_type'); - - $this->basedir = $this->application->generateBaseDir($this->deploymentUuid); - $this->workdir = $this->basedir . rtrim($this->application->base_directory, '/'); - } - public function handle() - { - try { - ray()->clearAll(); - $this->deployment->setStatus(ApplicationDeploymentStatus::IN_PROGRESS->value); - - $hostIpMappings = $this->mainServer->getHostIPMappings($this->application->destination->network); - if ($this->application->dockerfile_target_build) { - $buildTarget = " --target {$this->application->dockerfile_target_build} "; - } - - // Get the git repository and port (custom port or default port) - [ - 'repository' => $this->gitRepository, - 'port' => $this->gitPort - ] = $this->application->customRepository(); - - // Get the git branch and git import commands - [ - 'commands' => $this->gitImportCommands, - 'branch' => $this->gitBranch, - 'fullRepoUrl' => $this->gitFullRepoUrl - ] = $this->application->generateGitImportCommands($this->deploymentUuid, $this->pullRequestId, $this->gitType); - - $this->servers = $this->application->servers(); - - if ($this->deployment->restart_only) { - if ($this->application->build_pack === 'dockerimage') { - throw new \Exception('Restart only is not supported for docker image based deployments'); - } - $this->deployment->addLogEntry("Starting deployment of {$this->application->name}."); - $this->servers->each(function ($server) { - $this->deployment->addLogEntry("Restarting {$this->application->name} on {$server->name}."); - $this->restartOnly($server); - }); - } - $this->next(ApplicationDeploymentStatus::FINISHED->value); - } catch (Throwable $exception) { - $this->fail($exception); - } finally { - $this->servers->each(function ($server) { - $this->deployment->addLogEntry("Cleaning up temporary containers on {$server->name}."); - $server->executeRemoteCommand( - commands: collect([])->push([ - "command" => "docker rm -f {$this->deploymentUuid}", - "hidden" => true, - "ignoreErrors" => true, - ]), - loggingModel: $this->deployment - ); - }); - } - } - public function restartOnly(Server $server) - { - $server->executeRemoteCommand( - commands: $this->application->prepareHelperImage($this->deploymentUuid), - loggingModel: $this->deployment - ); - - $privateKey = data_get($this->application, 'private_key.private_key', null); - $gitLsRemoteCommand = collect([]); - if ($privateKey) { - $privateKey = base64_decode($privateKey); - $gitLsRemoteCommand - ->push([ - "command" => executeInDocker($this->deploymentUuid, "mkdir -p /root/.ssh") - ]) - ->push([ - "command" => executeInDocker($this->deploymentUuid, "echo '{$privateKey}' | base64 -d > /root/.ssh/id_rsa") - ]) - ->push([ - "command" => executeInDocker($this->deploymentUuid, "chmod 600 /root/.ssh/id_rsa") - ]) - ->push([ - "name" => "git_commit_sha", - "command" => executeInDocker($this->deploymentUuid, "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$this->gitPort} -o Port={$this->gitPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa\" git ls-remote {$this->gitFullRepoUrl} {$this->gitBranch}"), - "hidden" => true, - ]); - } else { - $gitLsRemoteCommand->push([ - "name" => "git_commit_sha", - "command" => executeInDocker($this->deploymentUuid, "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$this->gitPort} -o Port={$this->gitPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null\" git ls-remote {$this->gitFullRepoUrl} {$this->gitBranch}"), - "hidden" => true, - ]); - } - $this->deployment->addLogEntry("Checking if there is any new commit on {$this->gitBranch} branch."); - - $server->executeRemoteCommand( - commands: $gitLsRemoteCommand, - loggingModel: $this->deployment - ); - $commit = str($this->deployment->getOutput('git_commit_sha'))->before("\t"); - - [ - 'productionImageName' => $productionImageName - ] = $this->application->generateImageNames($commit, $this->pullRequestId); - - $this->deployment->addLogEntry("Checking if the image {$productionImageName} already exists."); - $server->checkIfDockerImageExists($productionImageName, $this->deployment); - - if (str($this->deployment->getOutput('local_image_found'))->isNotEmpty()) { - $this->deployment->addLogEntry("Image {$productionImageName} already exists. Skipping the build."); - - $server->createWorkDirForDeployment($this->workdir, $this->deployment); - - $this->application->generateDockerComposeFile($server, $this->deployment, $this->workdir); - $this->application->rollingUpdateApplication($server, $this->deployment, $this->workdir); - return; - } - throw new RuntimeException('Cannot find image anywhere. Please redeploy the application.'); - } - public function failed(Throwable $exception): void - { - ray($exception); - $this->next(ApplicationDeploymentStatus::FAILED->value); - } - private function next(string $status) - { - // If the deployment is cancelled by the user, don't update the status - if ($this->deployment->status !== ApplicationDeploymentStatus::CANCELLED_BY_USER->value) { - $this->deployment->update([ - 'status' => $status, - ]); - } - queue_next_deployment($this->application, isNew: true); - } -} diff --git a/app/Models/Application.php b/app/Models/Application.php index 05d078bb2..1f9614baa 100644 --- a/app/Models/Application.php +++ b/app/Models/Application.php @@ -6,11 +6,9 @@ use App\Enums\ApplicationDeploymentStatus; use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Database\Eloquent\Relations\HasMany; use Illuminate\Database\Eloquent\SoftDeletes; -use Illuminate\Support\Collection; use Spatie\Activitylog\Models\Activity; use Illuminate\Support\Str; use RuntimeException; -use Symfony\Component\Yaml\Yaml; use Visus\Cuid2\Cuid2; class Application extends BaseModel @@ -54,62 +52,6 @@ class Application extends BaseModel }); } - public function servers(): Collection - { - $mainServer = data_get($this, 'destination.server'); - $additionalDestinations = data_get($this, 'additional_destinations', null); - $additionalServers = collect([]); - if ($this->isMultipleServerDeployment()) { - ray('asd'); - if (str($additionalDestinations)->isNotEmpty()) { - $additionalDestinations = str($additionalDestinations)->explode(','); - foreach ($additionalDestinations as $destinationId) { - $destination = StandaloneDocker::find($destinationId)->whereNot('id', $mainServer->id)->first(); - $server = data_get($destination, 'server'); - $additionalServers->push($server); - } - } - } - return collect([$mainServer])->merge($additionalServers); - } - - public function generateImageNames(string $commit, int $pullRequestId) - { - if ($this->dockerfile) { - if ($this->docker_registry_image_name) { - $buildImageName = Str::lower("{$this->docker_registry_image_name}:build"); - $productionImageName = Str::lower("{$this->docker_registry_image_name}:latest"); - } else { - $buildImageName = Str::lower("{$this->uuid}:build"); - $productionImageName = Str::lower("{$this->uuid}:latest"); - } - } else if ($this->build_pack === 'dockerimage') { - $productionImageName = Str::lower("{$this->docker_registry_image_name}:{$this->docker_registry_image_tag}"); - } else if ($pullRequestId === 0) { - $dockerImageTag = str($commit)->substr(0, 128); - if ($this->docker_registry_image_name) { - $buildImageName = Str::lower("{$this->docker_registry_image_name}:{$dockerImageTag}-build"); - $productionImageName = Str::lower("{$this->docker_registry_image_name}:{$dockerImageTag}"); - } else { - $buildImageName = Str::lower("{$this->uuid}:{$dockerImageTag}-build"); - $productionImageName = Str::lower("{$this->uuid}:{$dockerImageTag}"); - } - } else if ($pullRequestId !== 0) { - if ($this->docker_registry_image_name) { - $buildImageName = Str::lower("{$this->docker_registry_image_name}:pr-{$pullRequestId}-build"); - $productionImageName = Str::lower("{$this->docker_registry_image_name}:pr-{$pullRequestId}"); - } else { - $buildImageName = Str::lower("{$this->uuid}:pr-{$pullRequestId}-build"); - $productionImageName = Str::lower("{$this->uuid}:pr-{$pullRequestId}"); - } - } - return [ - 'buildImageName' => $buildImageName, - 'productionImageName' => $productionImageName, - ]; - } - // End of build packs / deployment types - public function is_github_based(): bool { if (data_get($this, 'source')) { @@ -464,31 +406,6 @@ class Application extends BaseModel return true; } } - public function isMultipleServerDeployment() - { - return false; - if (data_get($this, 'additional_destinations') && data_get($this, 'docker_registry_image_name')) { - return true; - } - return false; - } - public function healthCheckUrl() - { - if ($this->dockerfile || $this->build_pack === 'dockerfile' || $this->build_pack === 'dockerimage') { - return null; - } - if (!$this->health_check_port) { - $health_check_port = $this->ports_exposes_array[0]; - } else { - $health_check_port = $this->health_check_port; - } - if ($this->health_check_path) { - $full_healthcheck_url = "{$this->health_check_scheme}://{$this->health_check_host}:{$health_check_port}{$this->health_check_path}"; - } else { - $full_healthcheck_url = "{$this->health_check_scheme}://{$this->health_check_host}:{$health_check_port}/"; - } - return $full_healthcheck_url; - } function customRepository() { preg_match('/(?<=:)\d+(?=\/)/', $this->git_repository, $matches); @@ -510,296 +427,7 @@ class Application extends BaseModel { return "/artifacts/{$uuid}"; } - function generateHealthCheckCommands() - { - if ($this->dockerfile || $this->build_pack === 'dockerfile' || $this->build_pack === 'dockerimage') { - // TODO: disabled HC because there are several ways to hc a simple docker image, hard to figure out a good way. Like some docker images (pocketbase) does not have curl. - return 'exit 0'; - } - if (!$this->health_check_port) { - $health_check_port = $this->ports_exposes_array[0]; - } else { - $health_check_port = $this->health_check_port; - } - if ($this->health_check_path) { - $this->full_healthcheck_url = "{$this->health_check_method}: {$this->health_check_scheme}://{$this->health_check_host}:{$health_check_port}{$this->health_check_path}"; - $generated_healthchecks_commands = [ - "curl -s -X {$this->health_check_method} -f {$this->health_check_scheme}://{$this->health_check_host}:{$health_check_port}{$this->health_check_path} > /dev/null" - ]; - } else { - $this->full_healthcheck_url = "{$this->health_check_method}: {$this->health_check_scheme}://{$this->health_check_host}:{$health_check_port}/"; - $generated_healthchecks_commands = [ - "curl -s -X {$this->health_check_method} -f {$this->health_check_scheme}://{$this->health_check_host}:{$health_check_port}/" - ]; - } - return implode(' ', $generated_healthchecks_commands); - } - function generateLocalPersistentVolumes(int $pullRequestId) - { - $persistentStorages = []; - $volumeNames = []; - foreach ($this->persistentStorages as $persistentStorage) { - $volume_name = $persistentStorage->host_path ?? $persistentStorage->name; - if ($pullRequestId !== 0) { - $volume_name = $volume_name . '-pr-' . $pullRequestId; - } - $persistentStorages[] = $volume_name . ':' . $persistentStorage->mount_path; - - if ($persistentStorage->host_path) { - continue; - } - - $name = $persistentStorage->name; - - if ($pullRequestId !== 0) { - $name = $name . '-pr-' . $pullRequestId; - } - - $volumeNames[$name] = [ - 'name' => $name, - 'external' => false, - ]; - } - - return [ - 'persistentStorages' => $persistentStorages, - 'volumeNames' => $volumeNames, - ]; - } - public function generateEnvironmentVariables($ports) - { - $environmentVariables = collect(); - // ray('Generate Environment Variables')->green(); - if ($this->pull_request_id === 0) { - // ray($this->runtime_environment_variables)->green(); - foreach ($this->runtime_environment_variables as $env) { - $environmentVariables->push("$env->key=$env->value"); - } - foreach ($this->nixpacks_environment_variables as $env) { - $environmentVariables->push("$env->key=$env->value"); - } - } else { - // ray($this->runtime_environment_variables_preview)->green(); - foreach ($this->runtime_environment_variables_preview as $env) { - $environmentVariables->push("$env->key=$env->value"); - } - foreach ($this->nixpacks_environment_variables_preview as $env) { - $environmentVariables->push("$env->key=$env->value"); - } - } - // Add PORT if not exists, use the first port as default - if ($environmentVariables->filter(fn ($env) => Str::of($env)->contains('PORT'))->isEmpty()) { - $environmentVariables->push("PORT={$ports[0]}"); - } - return $environmentVariables->all(); - } - function generateDockerComposeFile(Server $server, ApplicationDeploymentQueue $deployment, string $workdir) - { - $pullRequestId = $deployment->pull_request_id; - $ports = $this->settings->is_static ? [80] : $this->ports_exposes_array; - $container_name = generateApplicationContainerName($this, $this->pull_request_id); - $commit = str($deployment->getOutput('git_commit_sha'))->before("\t"); - - [ - 'productionImageName' => $productionImageName - ] = $this->generateImageNames($commit, $pullRequestId); - - [ - 'persistentStorages' => $persistentStorages, - 'volumeNames' => $volumeNames - ] = $this->generateLocalPersistentVolumes($pullRequestId); - - $environmentVariables = $this->generateEnvironmentVariables($ports); - - if (data_get($this, 'custom_labels')) { - $labels = collect(str($this->custom_labels)->explode(',')); - $labels = $labels->filter(function ($value, $key) { - return !Str::startsWith($value, 'coolify.'); - }); - $this->custom_labels = $labels->implode(','); - $this->save(); - } else { - $labels = collect(generateLabelsApplication($this, $this->preview)); - } - if ($this->pull_request_id !== 0) { - $labels = collect(generateLabelsApplication($this, $this->preview)); - } - $labels = $labels->merge(defaultLabels($this->id, $this->uuid, $this->pull_request_id))->toArray(); - $docker_compose = [ - 'version' => '3.8', - 'services' => [ - $container_name => [ - 'image' => $productionImageName, - 'container_name' => $container_name, - 'restart' => RESTART_MODE, - 'environment' => $environmentVariables, - 'expose' => $ports, - 'networks' => [ - $this->destination->network, - ], - 'healthcheck' => [ - 'test' => [ - 'CMD-SHELL', - $this->generateHealthCheckCommands() - ], - 'interval' => $this->health_check_interval . 's', - 'timeout' => $this->health_check_timeout . 's', - 'retries' => $this->health_check_retries, - 'start_period' => $this->health_check_start_period . 's' - ], - 'mem_limit' => $this->limits_memory, - 'memswap_limit' => $this->limits_memory_swap, - 'mem_swappiness' => $this->limits_memory_swappiness, - 'mem_reservation' => $this->limits_memory_reservation, - 'cpus' => (float) $this->limits_cpus, - 'cpu_shares' => $this->limits_cpu_shares, - ] - ], - 'networks' => [ - $this->destination->network => [ - 'external' => true, - 'name' => $this->destination->network, - 'attachable' => true - ] - ] - ]; - if (!is_null($this->limits_cpuset)) { - data_set($docker_compose, "services.{$container_name}.cpuset", $this->limits_cpuset); - } - if ($server->isSwarm()) { - data_forget($docker_compose, 'services.' . $container_name . '.container_name'); - data_forget($docker_compose, 'services.' . $container_name . '.expose'); - data_forget($docker_compose, 'services.' . $container_name . '.restart'); - - data_forget($docker_compose, 'services.' . $container_name . '.mem_limit'); - data_forget($docker_compose, 'services.' . $container_name . '.memswap_limit'); - data_forget($docker_compose, 'services.' . $container_name . '.mem_swappiness'); - data_forget($docker_compose, 'services.' . $container_name . '.mem_reservation'); - data_forget($docker_compose, 'services.' . $container_name . '.cpus'); - data_forget($docker_compose, 'services.' . $container_name . '.cpuset'); - data_forget($docker_compose, 'services.' . $container_name . '.cpu_shares'); - - $docker_compose['services'][$container_name]['deploy'] = [ - 'placement' => [ - 'constraints' => [ - 'node.role == worker' - ] - ], - 'mode' => 'replicated', - 'replicas' => 1, - 'update_config' => [ - 'order' => 'start-first' - ], - 'rollback_config' => [ - 'order' => 'start-first' - ], - 'labels' => $labels, - 'resources' => [ - 'limits' => [ - 'cpus' => $this->limits_cpus, - 'memory' => $this->limits_memory, - ], - 'reservations' => [ - 'cpus' => $this->limits_cpus, - 'memory' => $this->limits_memory, - ] - ] - ]; - } else { - $docker_compose['services'][$container_name]['labels'] = $labels; - } - if ($server->isLogDrainEnabled() && $this->isLogDrainEnabled()) { - $docker_compose['services'][$container_name]['logging'] = [ - 'driver' => 'fluentd', - 'options' => [ - 'fluentd-address' => "tcp://127.0.0.1:24224", - 'fluentd-async' => "true", - 'fluentd-sub-second-precision' => "true", - ] - ]; - } - if ($this->settings->is_gpu_enabled) { - $docker_compose['services'][$container_name]['deploy']['resources']['reservations']['devices'] = [ - [ - 'driver' => data_get($this, 'settings.gpu_driver', 'nvidia'), - 'capabilities' => ['gpu'], - 'options' => data_get($this, 'settings.gpu_options', []) - ] - ]; - if (data_get($this, 'settings.gpu_count')) { - $count = data_get($this, 'settings.gpu_count'); - if ($count === 'all') { - $docker_compose['services'][$container_name]['deploy']['resources']['reservations']['devices'][0]['count'] = $count; - } else { - $docker_compose['services'][$container_name]['deploy']['resources']['reservations']['devices'][0]['count'] = (int) $count; - } - } else if (data_get($this, 'settings.gpu_device_ids')) { - $docker_compose['services'][$container_name]['deploy']['resources']['reservations']['devices'][0]['ids'] = data_get($this, 'settings.gpu_device_ids'); - } - } - if ($this->isHealthcheckDisabled()) { - data_forget($docker_compose, 'services.' . $container_name . '.healthcheck'); - } - if (count($this->ports_mappings_array) > 0 && $this->pull_request_id === 0) { - $docker_compose['services'][$container_name]['ports'] = $this->ports_mappings_array; - } - if (count($persistentStorages) > 0) { - $docker_compose['services'][$container_name]['volumes'] = $persistentStorages; - } - if (count($volumeNames) > 0) { - $docker_compose['volumes'] = $volumeNames; - } - - $docker_compose['services'][$this->uuid] = $docker_compose['services'][$container_name]; - - data_forget($docker_compose, 'services.' . $container_name); - - $docker_compose = Yaml::dump($docker_compose, 10); - $docker_compose_base64 = base64_encode($docker_compose); - $server->executeRemoteCommand( - commands: collect([])->push([ - 'command' => executeInDocker($deployment->deployment_uuid, "echo '{$docker_compose_base64}' | base64 -d > {$workdir}/docker-compose.yml"), - 'hidden' => true, - 'ignoreErrors' => true - ]), - loggingModel: $deployment - ); - } - function rollingUpdateApplication(Server $server, ApplicationDeploymentQueue $deployment, string $workdir) - { - $pullRequestId = $deployment->pull_request_id; - $containerName = generateApplicationContainerName($this, $pullRequestId); - // if (count($this->ports_mappings_array) > 0) { - // $deployment->addLogEntry('Application has ports mapped to the host system, rolling update is not supported.'); - $containers = getCurrentApplicationContainerStatus($server, $this->id, $pullRequestId); - // if ($pullRequestId === 0) { - // $containers = $containers->filter(function ($container) use ($containerName) { - // return data_get($container, 'Names') !== $containerName; - // }); - // } - $containers->each(function ($container) use ($server, $deployment) { - $removingContainerName = data_get($container, 'Names'); - $server->executeRemoteCommand( - commands: collect([])->push([ - 'command' => "docker rm -f $removingContainerName", - 'hidden' => true, - 'ignoreErrors' => true - ]), - loggingModel: $deployment - ); - }); - // } - $server->executeRemoteCommand( - commands: collect([])->push([ - 'command' => executeInDocker($deployment->deployment_uuid, "docker compose --project-directory {$workdir} up --build -d"), - 'hidden' => true, - 'ignoreErrors' => true - ]), - loggingModel: $deployment - ); - $deployment->addLogEntry("New container started."); - } - function setGitImportSettings(string $deployment_uuid, string $git_clone_command) + function setGitImportSettings(string $deployment_uuid, string $git_clone_command) { $baseDir = $this->generateBaseDir($deployment_uuid); if ($this->git_commit_sha !== 'HEAD') { @@ -977,34 +605,6 @@ class Application extends BaseModel ]; } } - public function prepareHelperImage(string $deploymentUuid) - { - $basedir = $this->generateBaseDir($deploymentUuid); - $helperImage = config('coolify.helper_image'); - $server = data_get($this, 'destination.server'); - $network = data_get($this, 'destination.network'); - - $serverUserHomeDir = instant_remote_process(["echo \$HOME"], $server); - $dockerConfigFileExists = instant_remote_process(["test -f {$serverUserHomeDir}/.docker/config.json && echo 'OK' || echo 'NOK'"], $server); - - $commands = collect([]); - if ($dockerConfigFileExists === 'OK') { - $commands->push([ - "command" => "docker run -d --network $network --name $deploymentUuid --rm -v {$serverUserHomeDir}/.docker/config.json:/root/.docker/config.json:ro -v /var/run/docker.sock:/var/run/docker.sock $helperImage", - "hidden" => true, - ]); - } else { - $commands->push([ - "command" => "docker run -d --network {$network} --name {$deploymentUuid} --rm -v /var/run/docker.sock:/var/run/docker.sock {$helperImage}", - "hidden" => true, - ]); - } - $commands->push([ - "command" => executeInDocker($deploymentUuid, "mkdir -p {$basedir}"), - "hidden" => true, - ]); - return $commands; - } function parseCompose(int $pull_request_id = 0) { if ($this->docker_compose_raw) { diff --git a/app/Models/Server.php b/app/Models/Server.php index 432a296e9..972751083 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -2,16 +2,12 @@ namespace App\Models; -use App\Enums\ApplicationDeploymentStatus; use App\Enums\ProxyStatus; use App\Enums\ProxyTypes; use App\Notifications\Server\Revived; use App\Notifications\Server\Unreachable; use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Casts\Attribute; -use Illuminate\Support\Carbon; -use Illuminate\Support\Collection; -use Illuminate\Support\Facades\Process; use Spatie\SchemalessAttributes\Casts\SchemalessAttributes; use Spatie\SchemalessAttributes\SchemalessAttributesTrait; use Illuminate\Support\Str; @@ -335,20 +331,6 @@ class Server extends BaseModel if ($this->proxyType() === ProxyTypes::NONE->value || $this->settings->is_build_server) { return false; } - // foreach ($this->applications() as $application) { - // if (data_get($application, 'fqdn')) { - // $shouldRun = true; - // break; - // } - // } - // ray($this->services()->get()); - - // if ($this->id === 0) { - // $settings = InstanceSettings::get(); - // if (data_get($settings, 'fqdn')) { - // $shouldRun = true; - // } - // } return true; } public function isFunctional() @@ -483,153 +465,4 @@ class Server extends BaseModel return instant_remote_process(["docker network create coolify --attachable >/dev/null 2>&1 || true"], $this, false); } } - public function executeRemoteCommand(Collection $commands, ?ApplicationDeploymentQueue $loggingModel = null) - { - static::$batch_counter++; - foreach ($commands as $command) { - $realCommand = data_get($command, 'command'); - if (is_null($realCommand)) { - throw new \RuntimeException('Command is not set'); - } - $hidden = data_get($command, 'hidden', false); - $ignoreErrors = data_get($command, 'ignoreErrors', false); - $customOutputType = data_get($command, 'customOutputType'); - $name = data_get($command, 'name'); - $remoteCommand = generateSshCommand($this, $realCommand); - - $process = Process::timeout(3600)->idleTimeout(3600)->start($remoteCommand, function (string $type, string $output) use ($realCommand, $hidden, $customOutputType, $loggingModel, $name) { - $output = str($output)->trim(); - if ($output->startsWith('╔')) { - $output = "\n" . $output; - } - $newLogEntry = [ - 'command' => remove_iip($realCommand), - 'output' => remove_iip($output), - 'type' => $customOutputType ?? $type === 'err' ? 'stderr' : 'stdout', - 'timestamp' => Carbon::now('UTC'), - 'hidden' => $hidden, - 'batch' => static::$batch_counter, - ]; - if ($loggingModel) { - if (!$loggingModel->logs) { - $newLogEntry['order'] = 1; - } else { - $previousLogs = json_decode($loggingModel->logs, associative: true, flags: JSON_THROW_ON_ERROR); - $newLogEntry['order'] = count($previousLogs) + 1; - } - if ($name) { - $newLogEntry['name'] = $name; - } - - $previousLogs[] = $newLogEntry; - $loggingModel->logs = json_encode($previousLogs, flags: JSON_THROW_ON_ERROR); - $loggingModel->save(); - } - }); - if ($loggingModel) { - $loggingModel->update([ - 'current_process_id' => $process->id(), - ]); - } - $processResult = $process->wait(); - if ($processResult->exitCode() !== 0) { - if (!$ignoreErrors) { - if ($loggingModel) { - $status = ApplicationDeploymentStatus::FAILED->value; - $loggingModel->status = $status; - $loggingModel->save(); - } - throw new \RuntimeException($processResult->errorOutput()); - } - } - } - } - public function stopApplicationRelatedRunningContainers(string $applicationId, string $containerName) - { - $containers = getCurrentApplicationContainerStatus($this, $applicationId, 0); - $containers = $containers->filter(function ($container) use ($containerName) { - return data_get($container, 'Names') !== $containerName; - }); - $containers->each(function ($container) { - $removableContainer = data_get($container, 'Names'); - $this->server->executeRemoteCommand( - commands: collect([ - 'command' => "docker rm -f $removableContainer >/dev/null 2>&1", - 'hidden' => true, - 'ignoreErrors' => true - ]), - loggingModel: $this->deploymentQueueEntry - ); - }); - } - public function getHostIPMappings($network) - { - $addHosts = null; - $allContainers = instant_remote_process(["docker network inspect {$network} -f '{{json .Containers}}' "], $this); - if (!is_null($allContainers)) { - $allContainers = format_docker_command_output_to_json($allContainers); - $ips = collect([]); - if (count($allContainers) > 0) { - $allContainers = $allContainers[0]; - foreach ($allContainers as $container) { - $containerName = data_get($container, 'Name'); - if ($containerName === 'coolify-proxy') { - continue; - } - $containerIp = data_get($container, 'IPv4Address'); - if ($containerName && $containerIp) { - $containerIp = str($containerIp)->before('/'); - $ips->put($containerName, $containerIp->value()); - } - } - } - $addHosts = $ips->map(function ($ip, $name) { - return "--add-host $name:$ip"; - })->implode(' '); - } - return $addHosts; - } - public function checkIfDockerImageExists(string $imageName, ApplicationDeploymentQueue $deployment) - { - $this->executeRemoteCommand( - commands: collect([ - [ - "name" => "local_image_found", - "command" => "docker images -q {$imageName} 2>/dev/null", - "hidden" => true, - ] - ]), - loggingModel: $deployment - ); - if (str($deployment->getOutput('local_image_found'))->isEmpty()) { - $this->executeRemoteCommand( - commands: collect([ - [ - "command" => "docker pull {$imageName} 2>/dev/null", - "ignoreErrors" => true, - "hidden" => true - ], - [ - "name" => "local_image_found", - "command" => "docker images -q {$imageName} 2>/dev/null", - "hidden" => true, - ] - ]), - loggingModel: $deployment - ); - } - } - public function createWorkDirForDeployment(string $workdir, ApplicationDeploymentQueue $deployment) - { - $this->executeRemoteCommand( - commands: collect([ - [ - "command" => executeInDocker($deployment->deployment_uuid, "mkdir -p {$workdir}"), - "ignoreErrors" => true, - "hidden" => true - ], - ]), - loggingModel: $deployment - ); - } } diff --git a/app/Traits/ExecuteRemoteCommandNew.php b/app/Traits/ExecuteRemoteCommandNew.php deleted file mode 100644 index ca56d9e50..000000000 --- a/app/Traits/ExecuteRemoteCommandNew.php +++ /dev/null @@ -1,77 +0,0 @@ -each(function ($singleCommand) use ($server, $logModel) { - $command = data_get($singleCommand, 'command') ?? $singleCommand[0] ?? null; - if ($command === null) { - throw new \RuntimeException('Command is not set'); - } - $hidden = data_get($singleCommand, 'hidden', false); - $customType = data_get($singleCommand, 'type'); - $ignoreErrors = data_get($singleCommand, 'ignore_errors', false); - $save = data_get($singleCommand, 'save'); - - $remote_command = generateSshCommand($server, $command); - $process = Process::timeout(3600)->idleTimeout(3600)->start($remote_command, function (string $type, string $output) use ($command, $hidden, $customType, $logModel, $save) { - $output = str($output)->trim(); - if ($output->startsWith('╔')) { - $output = "\n" . $output; - } - $newLogEntry = [ - 'command' => remove_iip($command), - 'output' => remove_iip($output), - 'type' => $customType ?? $type === 'err' ? 'stderr' : 'stdout', - 'timestamp' => Carbon::now('UTC'), - 'hidden' => $hidden, - 'batch' => static::$batch_counter, - ]; - - if (!$logModel->logs) { - $newLogEntry['order'] = 1; - } else { - $previousLogs = json_decode($logModel->logs, associative: true, flags: JSON_THROW_ON_ERROR); - $newLogEntry['order'] = count($previousLogs) + 1; - } - - $previousLogs[] = $newLogEntry; - $logModel->logs = json_encode($previousLogs, flags: JSON_THROW_ON_ERROR); - $logModel->save(); - - if ($save) { - $this->remoteCommandOutputs[$save] = str($output)->trim(); - } - }); - $logModel->update([ - 'current_process_id' => $process->id(), - ]); - - $processResult = $process->wait(); - if ($processResult->exitCode() !== 0) { - if (!$ignoreErrors) { - $status = ApplicationDeploymentStatus::FAILED->value; - $logModel->status = $status; - $logModel->save(); - throw new \RuntimeException($processResult->errorOutput()); - } - } - }); - } -} diff --git a/bootstrap/helpers/applications.php b/bootstrap/helpers/applications.php index 079c1add8..de381212f 100644 --- a/bootstrap/helpers/applications.php +++ b/bootstrap/helpers/applications.php @@ -1,15 +1,12 @@ id; $deployment_link = Url::fromString($application->link() . "/deployment/{$deployment_uuid}"); @@ -31,13 +28,33 @@ function queue_application_deployment(Application $application, string $deployme 'git_type' => $git_type ]); + if (next_queuable($server_id, $application_id)) { + dispatch(new ApplicationDeploymentJob( + application_deployment_queue_id: $deployment->id, + )); + } +} + +function queue_next_deployment(Application $application) +{ + $server_id = $application->destination->server_id; + $next_found = ApplicationDeploymentQueue::where('server_id', $server_id)->where('status', 'queued')->get()->sortBy('created_at')->first(); + if ($next_found) { + dispatch(new ApplicationDeploymentJob( + application_deployment_queue_id: $next_found->id, + )); + } +} + +function next_queuable(string $server_id, string $application_id): bool +{ $deployments = ApplicationDeploymentQueue::where('server_id', $server_id)->whereIn('status', ['in_progress', 'queued'])->get()->sortByDesc('created_at'); $same_application_deployments = $deployments->where('application_id', $application_id); $in_progress = $same_application_deployments->filter(function ($value, $key) { return $value->status === 'in_progress'; }); if ($in_progress->count() > 0) { - return; + return false; } $server = Server::find($server_id); $concurrent_builds = $server->settings->concurrent_builds; @@ -45,296 +62,7 @@ function queue_application_deployment(Application $application, string $deployme ray("serverId:{$server->id}", "concurrentBuilds:{$concurrent_builds}", "deployments:{$deployments->count()}", "sameApplicationDeployments:{$same_application_deployments->count()}"); if ($deployments->count() > $concurrent_builds) { - return; - } - if ($is_new_deployment) { - dispatch(new ApplicationDeploymentNewJob( - deployment: $deployment, - application: Application::find($application_id) - )); - } else { - dispatch(new ApplicationDeploymentJob( - application_deployment_queue_id: $deployment->id, - )); + return false; } -} - -function queue_next_deployment(Application $application, bool $isNew = false) -{ - $server_id = $application->destination->server_id; - $next_found = ApplicationDeploymentQueue::where('server_id', $server_id)->where('status', 'queued')->get()->sortBy('created_at')->first();; - // ray($next_found, $server_id); - if ($next_found) { - if ($isNew) { - dispatch(new ApplicationDeploymentNewJob( - deployment: $next_found, - application: $application - )); - } else { - dispatch(new ApplicationDeploymentJob( - application_deployment_queue_id: $next_found->id, - )); - } - } -} -// Deployment things -function generateHostIpMapping(Server $server, string $network) -{ - // Generate custom host<->ip hostnames - $allContainers = instant_remote_process(["docker network inspect {$network} -f '{{json .Containers}}' "], $server); - $allContainers = format_docker_command_output_to_json($allContainers); - $ips = collect([]); - if (count($allContainers) > 0) { - $allContainers = $allContainers[0]; - foreach ($allContainers as $container) { - $containerName = data_get($container, 'Name'); - if ($containerName === 'coolify-proxy') { - continue; - } - $containerIp = data_get($container, 'IPv4Address'); - if ($containerName && $containerIp) { - $containerIp = str($containerIp)->before('/'); - $ips->put($containerName, $containerIp->value()); - } - } - } - return $ips->map(function ($ip, $name) { - return "--add-host $name:$ip"; - })->implode(' '); -} - -function generateBaseDir(string $deplyomentUuid) -{ - return "/artifacts/$deplyomentUuid"; -} -function generateWorkdir(string $deplyomentUuid, Application $application) -{ - return generateBaseDir($deplyomentUuid) . rtrim($application->base_directory, '/'); -} - -function prepareHelperContainer(Server $server, string $network, string $deploymentUuid) -{ - $basedir = generateBaseDir($deploymentUuid); - $helperImage = config('coolify.helper_image'); - - $serverUserHomeDir = instant_remote_process(["echo \$HOME"], $server); - $dockerConfigFileExists = instant_remote_process(["test -f {$serverUserHomeDir}/.docker/config.json && echo 'OK' || echo 'NOK'"], $server); - - $commands = collect([]); - if ($dockerConfigFileExists === 'OK') { - $commands->push([ - "command" => "docker run -d --network $network --name $deploymentUuid --rm -v {$serverUserHomeDir}/.docker/config.json:/root/.docker/config.json:ro -v /var/run/docker.sock:/var/run/docker.sock $helperImage", - "hidden" => true, - ]); - } else { - $commands->push([ - "command" => "docker run -d --network {$network} --name {$deploymentUuid} --rm -v /var/run/docker.sock:/var/run/docker.sock {$helperImage}", - "hidden" => true, - ]); - } - $commands->push([ - "command" => executeInDocker($deploymentUuid, "mkdir -p {$basedir}"), - "hidden" => true, - ]); - return $commands; -} - -function generateComposeFile(string $deploymentUuid, Server $server, string $network, Application $application, string $containerName, string $imageName, ?ApplicationPreview $preview = null, int $pullRequestId = 0) -{ - $ports = $application->settings->is_static ? [80] : $application->ports_exposes_array; - $workDir = generateWorkdir($deploymentUuid, $application); - $persistent_storages = generateLocalPersistentVolumes($application, $pullRequestId); - $volume_names = generateLocalPersistentVolumesOnlyVolumeNames($application, $pullRequestId); - $environment_variables = generateEnvironmentVariables($application, $ports, $pullRequestId); - - if (data_get($application, 'custom_labels')) { - $labels = collect(str($application->custom_labels)->explode(',')); - $labels = $labels->filter(function ($value, $key) { - return !str($value)->startsWith('coolify.'); - }); - $application->custom_labels = $labels->implode(','); - $application->save(); - } else { - $labels = collect(generateLabelsApplication($application, $preview)); - } - if ($pullRequestId !== 0) { - $labels = collect(generateLabelsApplication($application, $preview)); - } - $labels = $labels->merge(defaultLabels($application->id, $application->uuid, 0))->toArray(); - $docker_compose = [ - 'version' => '3.8', - 'services' => [ - $containerName => [ - 'image' => $imageName, - 'container_name' => $containerName, - 'restart' => RESTART_MODE, - 'environment' => $environment_variables, - 'labels' => $labels, - 'expose' => $ports, - 'networks' => [ - $network, - ], - 'mem_limit' => $application->limits_memory, - 'memswap_limit' => $application->limits_memory_swap, - 'mem_swappiness' => $application->limits_memory_swappiness, - 'mem_reservation' => $application->limits_memory_reservation, - 'cpus' => (int) $application->limits_cpus, - 'cpu_shares' => $application->limits_cpu_shares, - ] - ], - 'networks' => [ - $network => [ - 'external' => true, - 'name' => $network, - 'attachable' => true - ] - ] - ]; - if (!is_null($application->limits_cpuset)) { - data_set($docker_compose, "services.{$containerName}.cpuset", $application->limits_cpuset); - } - if ($server->isLogDrainEnabled() && $application->isLogDrainEnabled()) { - $docker_compose['services'][$containerName]['logging'] = [ - 'driver' => 'fluentd', - 'options' => [ - 'fluentd-address' => "tcp://127.0.0.1:24224", - 'fluentd-async' => "true", - 'fluentd-sub-second-precision' => "true", - ] - ]; - } - if ($application->settings->is_gpu_enabled) { - $docker_compose['services'][$containerName]['deploy']['resources']['reservations']['devices'] = [ - [ - 'driver' => data_get($application, 'settings.gpu_driver', 'nvidia'), - 'capabilities' => ['gpu'], - 'options' => data_get($application, 'settings.gpu_options', []) - ] - ]; - if (data_get($application, 'settings.gpu_count')) { - $count = data_get($application, 'settings.gpu_count'); - if ($count === 'all') { - $docker_compose['services'][$containerName]['deploy']['resources']['reservations']['devices'][0]['count'] = $count; - } else { - $docker_compose['services'][$containerName]['deploy']['resources']['reservations']['devices'][0]['count'] = (int) $count; - } - } else if (data_get($application, 'settings.gpu_device_ids')) { - $docker_compose['services'][$containerName]['deploy']['resources']['reservations']['devices'][0]['ids'] = data_get($application, 'settings.gpu_device_ids'); - } - } - if ($application->isHealthcheckDisabled()) { - data_forget($docker_compose, 'services.' . $containerName . '.healthcheck'); - } - if (count($application->ports_mappings_array) > 0 && $pullRequestId === 0) { - $docker_compose['services'][$containerName]['ports'] = $application->ports_mappings_array; - } - if (count($persistent_storages) > 0) { - $docker_compose['services'][$containerName]['volumes'] = $persistent_storages; - } - if (count($volume_names) > 0) { - $docker_compose['volumes'] = $volume_names; - } - $docker_compose = Yaml::dump($docker_compose, 10); - $docker_compose_base64 = base64_encode($docker_compose); - $commands = collect([]); - $commands->push([ - "command" => executeInDocker($deploymentUuid, "echo '{$docker_compose_base64}' | base64 -d > {$workDir}/docker-compose.yml"), - "hidden" => true, - ]); - return $commands; -} -function generateLocalPersistentVolumes(Application $application, int $pullRequestId = 0) -{ - $local_persistent_volumes = []; - foreach ($application->persistentStorages as $persistentStorage) { - $volume_name = $persistentStorage->host_path ?? $persistentStorage->name; - if ($pullRequestId !== 0) { - $volume_name = $volume_name . '-pr-' . $pullRequestId; - } - $local_persistent_volumes[] = $volume_name . ':' . $persistentStorage->mount_path; - } - return $local_persistent_volumes; -} - -function generateLocalPersistentVolumesOnlyVolumeNames(Application $application, int $pullRequestId = 0) -{ - $local_persistent_volumes_names = []; - foreach ($application->persistentStorages as $persistentStorage) { - if ($persistentStorage->host_path) { - continue; - } - $name = $persistentStorage->name; - - if ($pullRequestId !== 0) { - $name = $name . '-pr-' . $pullRequestId; - } - - $local_persistent_volumes_names[$name] = [ - 'name' => $name, - 'external' => false, - ]; - } - return $local_persistent_volumes_names; -} -function generateEnvironmentVariables(Application $application, $ports, int $pullRequestId = 0) -{ - $environment_variables = collect(); - // ray('Generate Environment Variables')->green(); - if ($pullRequestId === 0) { - // ray($this->application->runtime_environment_variables)->green(); - foreach ($application->runtime_environment_variables as $env) { - $environment_variables->push("$env->key=$env->value"); - } - foreach ($application->nixpacks_environment_variables as $env) { - $environment_variables->push("$env->key=$env->value"); - } - } else { - // ray($this->application->runtime_environment_variables_preview)->green(); - foreach ($application->runtime_environment_variables_preview as $env) { - $environment_variables->push("$env->key=$env->value"); - } - foreach ($application->nixpacks_environment_variables_preview as $env) { - $environment_variables->push("$env->key=$env->value"); - } - } - // Add PORT if not exists, use the first port as default - if ($environment_variables->filter(fn ($env) => str($env)->contains('PORT'))->isEmpty()) { - $environment_variables->push("PORT={$ports[0]}"); - } - return $environment_variables->all(); -} - -function startNewApplication(Application $application, string $deploymentUuid, ApplicationDeploymentQueue $loggingModel) -{ - $commands = collect([]); - $workDir = generateWorkdir($deploymentUuid, $application); - if ($application->build_pack === 'dockerimage') { - $loggingModel->addLogEntry('Pulling latest images from the registry.'); - $commands->push( - [ - "command" => executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} pull"), - "hidden" => true - ], - [ - "command" => executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} up --build -d"), - "hidden" => true - ], - ); - } else { - $commands->push( - [ - "command" => executeInDocker($deploymentUuid, "docker compose --project-directory {$workDir} up --build -d"), - "hidden" => true - ], - ); - } - return $commands; -} -function removeOldDeployment(string $containerName) -{ - $commands = collect([]); - $commands->push( - ["docker rm -f $containerName >/dev/null 2>&1"], - ); - return $commands; + return true; }