diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index 4f3911353..f38cf8463 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -17,12 +17,12 @@ use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; use Illuminate\Support\Collection; -use Illuminate\Support\Facades\Notification; use Illuminate\Support\Facades\Storage; use Spatie\Activitylog\Models\Activity; use Symfony\Component\Yaml\Yaml; use Illuminate\Support\Str; use Spatie\Url\Url; +use Throwable; use Visus\Cuid2\Cuid2; class ApplicationDeploymentJob implements ShouldQueue @@ -46,7 +46,7 @@ class ApplicationDeploymentJob implements ShouldQueue private ApplicationPreview|null $preview = null; public static int $batch_counter = 0; - public $timeout = 10200; + public $failOnTimeout = true; public function __construct( public int $application_deployment_queue_id, @@ -128,7 +128,8 @@ public function handle(): void "echo '\nOops something is not okay, are you okay? 😢'", "echo '\n\n{$e->getMessage()}'", ]); - $this->fail(); + ray($e); + $this->fail($e->getMessage()); } finally { if (isset($this->docker_compose)) { Storage::disk('deployments')->put(Str::kebab($this->application->name) . '/docker-compose.yml', $this->docker_compose); @@ -142,9 +143,15 @@ private function start_builder_image() $this->execute_now([ "echo -n 'Pulling latest version of the builder image (ghcr.io/coollabsio/coolify-builder)... '", ]); - $this->execute_now([ - "docker run --pull=always -d --name {$this->deployment_uuid} --rm -v /var/run/docker.sock:/var/run/docker.sock ghcr.io/coollabsio/coolify-builder", - ], isDebuggable: true); + if (isDev()) { + $this->execute_now([ + "docker run -d --name {$this->deployment_uuid} --rm -v /var/run/docker.sock:/var/run/docker.sock coolify-builder", + ], isDebuggable: true); + } else { + $this->execute_now([ + "docker run --pull=always -d --name {$this->deployment_uuid} --rm -v /var/run/docker.sock:/var/run/docker.sock ghcr.io/coollabsio/coolify-builder", + ], isDebuggable: true); + } $this->execute_now([ "echo 'Done.'" ]); @@ -291,8 +298,9 @@ private function deploy() $this->next(ProcessStatus::FINISHED->value); } - public function failed(): void + public function failed(Throwable $exception): void { + ray($exception); $this->next(ProcessStatus::ERROR->value); } @@ -329,7 +337,7 @@ private function next(string $status) } private function execute_in_builder(string $command) { - return "docker exec {$this->deployment_uuid} bash -c '{$command}'"; + return "docker exec {$this->deployment_uuid} bash -c '{$command} |& tee -a /proc/1/fd/1'"; } private function generate_environment_variables($ports) { diff --git a/bootstrap/helpers/applications.php b/bootstrap/helpers/applications.php index bbf24ff00..0f69d42ca 100644 --- a/bootstrap/helpers/applications.php +++ b/bootstrap/helpers/applications.php @@ -34,7 +34,7 @@ function queue_application_deployment(int $application_id, string $deployment_uu force_rebuild: $force_rebuild, rollback_commit: $commit, pull_request_id: $pull_request_id, - )); + ))->onConnection('long-running')->onQueue('long-running'); } function queue_next_deployment(Application $application) @@ -47,6 +47,6 @@ function queue_next_deployment(Application $application) deployment_uuid: $next_found->deployment_uuid, force_rebuild: $next_found->force_rebuild, pull_request_id: $next_found->pull_request_id - )); + ))->onConnection('long-running')->onQueue('long-running'); } } diff --git a/bootstrap/helpers/github.php b/bootstrap/helpers/github.php index 53a0f52fb..4e4fdd7fc 100644 --- a/bootstrap/helpers/github.php +++ b/bootstrap/helpers/github.php @@ -64,7 +64,7 @@ function git_api(GithubApp|GitlabApp $source, string $endpoint, string $method = } $json = $response->json(); if ($response->failed() && $throwError) { - throw new \Exception("Failed to get data from {$source->name} with error: " . $json['message']); + throw new \Exception("Failed to get data from {$source->name} with error:

" . $json['message']); } return [ 'rate_limit_remaining' => $response->header('X-RateLimit-Remaining'), diff --git a/config/horizon.php b/config/horizon.php index d9f841aa6..f35cbd731 100644 --- a/config/horizon.php +++ b/config/horizon.php @@ -190,7 +190,18 @@ 'maxJobs' => 0, 'memory' => 128, 'tries' => 1, - 'timeout' => 3600, + 'timeout' => 300, + 'nice' => 0, + ], + 'long-running' => [ + 'connection' => 'redis', + 'queue' => ['long-running'], + 'balance' => 'auto', + 'maxTime' => 0, + 'maxJobs' => 0, + 'memory' => 128, + 'tries' => 1, + 'timeout' => 3560, 'nice' => 0, ], ], @@ -203,6 +214,12 @@ 'balanceMaxShift' => env('HORIZON_BALANCE_MAX_SHIFT', 1), 'balanceCooldown' => env('HORIZON_BALANCE_COOLDOWN', 1), ], + 'long-running' => [ + 'autoScalingStrategy' => 'size', + 'maxProcesses' => env('HORIZON_MAX_PROCESSES', 10), + 'balanceMaxShift' => env('HORIZON_BALANCE_MAX_SHIFT', 1), + 'balanceCooldown' => env('HORIZON_BALANCE_COOLDOWN', 1), + ], ], 'local' => [ @@ -212,6 +229,12 @@ 'balanceMaxShift' => env('HORIZON_BALANCE_MAX_SHIFT', 1), 'balanceCooldown' => env('HORIZON_BALANCE_COOLDOWN', 1), ], + 'long-running' => [ + 'autoScalingStrategy' => 'size', + 'maxProcesses' => env('HORIZON_MAX_PROCESSES', 10), + 'balanceMaxShift' => env('HORIZON_BALANCE_MAX_SHIFT', 1), + 'balanceCooldown' => env('HORIZON_BALANCE_COOLDOWN', 1), + ], ], ], ]; diff --git a/config/queue.php b/config/queue.php index a7a3d46f8..7bb782605 100644 --- a/config/queue.php +++ b/config/queue.php @@ -33,7 +33,14 @@ 'sync' => [ 'driver' => 'sync', ], - + 'long-running' => [ + 'driver' => 'redis', + 'connection' => 'default', + 'queue' => 'long-running', + 'retry_after' => 3600, + 'block_for' => null, + 'after_commit' => false, + ], 'database' => [ 'driver' => 'database', 'table' => 'jobs', @@ -66,7 +73,7 @@ 'driver' => 'redis', 'connection' => 'default', 'queue' => env('REDIS_QUEUE', 'default'), - 'retry_after' => 90, + 'retry_after' => 300, 'block_for' => null, 'after_commit' => false, ], diff --git a/docker/coolify-builder/Dockerfile b/docker/coolify-builder/Dockerfile index d03e0196c..390ea38f9 100644 --- a/docker/coolify-builder/Dockerfile +++ b/docker/coolify-builder/Dockerfile @@ -35,5 +35,5 @@ RUN if [[ ${TARGETPLATFORM} == 'linux/arm64' ]]; then \ ;fi ENTRYPOINT ["/sbin/tini", "--"] -CMD ["sh", "-c", "while true; do sleep 3600 && exit 0; done"] +CMD ["sh", "-c", "while true; do sleep 1; done"] diff --git a/resources/views/livewire/project/application/deployment-navbar.blade.php b/resources/views/livewire/project/application/deployment-navbar.blade.php index c5f09d175..87654251e 100644 --- a/resources/views/livewire/project/application/deployment-navbar.blade.php +++ b/resources/views/livewire/project/application/deployment-navbar.blade.php @@ -1,6 +1,6 @@

Logs

- @if (data_get($activity, 'properties.status') === 'in_progress') + @if (data_get($activity, 'properties.status') === 'in_progress' || data_get($activity, 'properties.status') === 'queued') Cancel deployment @endif
diff --git a/resources/views/livewire/project/application/deployments.blade.php b/resources/views/livewire/project/application/deployments.blade.php index ff1099028..1385025af 100644 --- a/resources/views/livewire/project/application/deployments.blade.php +++ b/resources/views/livewire/project/application/deployments.blade.php @@ -67,8 +67,8 @@ class="hover:no-underline"> dayjs.extend(window.dayjs_plugin_relativeTime); Alpine.data('elapsedTime', (uuid, status, created_at, updated_at) => ({ - finished_time: '0s', - started_time: '0s', + finished_time: 'calculating...', + started_time: 'calculating...', init() { if (timers[uuid]) { clearInterval(timers[uuid]); diff --git a/resources/views/vendor/toaster/hub.blade.php b/resources/views/vendor/toaster/hub.blade.php index 9bdcd47d6..5aff1b506 100644 --- a/resources/views/vendor/toaster/hub.blade.php +++ b/resources/views/vendor/toaster/hub.blade.php @@ -71,7 +71,7 @@ class="relative flex duration-300 transform transition ease-in-out max-w-md w-fu - + @if ($closeable)