feat: long running queue with 1 hour of timeout

app/Jobs/ApplicationDeploymentJob.php

@@ -17,12 +17,12 @@
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Collection;
-use Illuminate\Support\Facades\Notification;
 use Illuminate\Support\Facades\Storage;
 use Spatie\Activitylog\Models\Activity;
 use Symfony\Component\Yaml\Yaml;
 use Illuminate\Support\Str;
 use Spatie\Url\Url;
+use Throwable;
 use Visus\Cuid2\Cuid2;
 
 class ApplicationDeploymentJob implements ShouldQueue
@@ -46,7 +46,7 @@ class ApplicationDeploymentJob implements ShouldQueue
     private ApplicationPreview|null $preview = null;
 
     public static int $batch_counter = 0;
-    public $timeout = 10200;
+    public $failOnTimeout = true;
 
     public function __construct(
         public int $application_deployment_queue_id,
@@ -128,7 +128,8 @@ public function handle(): void
                 "echo '\nOops something is not okay, are you okay? 😢'",
                 "echo '\n\n{$e->getMessage()}'",
             ]);
-            $this->fail();
+            ray($e);
+            $this->fail($e->getMessage());
         } finally {
             if (isset($this->docker_compose)) {
                 Storage::disk('deployments')->put(Str::kebab($this->application->name) . '/docker-compose.yml', $this->docker_compose);
@@ -142,9 +143,15 @@ private function start_builder_image()
         $this->execute_now([
             "echo -n 'Pulling latest version of the builder image (ghcr.io/coollabsio/coolify-builder)... '",
         ]);
+        if (isDev()) {
+            $this->execute_now([
+                "docker run -d --name {$this->deployment_uuid} --rm -v /var/run/docker.sock:/var/run/docker.sock coolify-builder",
+            ], isDebuggable: true);
+        } else {
             $this->execute_now([
                 "docker run --pull=always -d --name {$this->deployment_uuid} --rm -v /var/run/docker.sock:/var/run/docker.sock ghcr.io/coollabsio/coolify-builder",
             ], isDebuggable: true);
+        }
         $this->execute_now([
             "echo 'Done.'"
         ]);
@@ -291,8 +298,9 @@ private function deploy()
         $this->next(ProcessStatus::FINISHED->value);
     }
 
-    public function failed(): void
+    public function failed(Throwable $exception): void
     {
+        ray($exception);
         $this->next(ProcessStatus::ERROR->value);
     }
 
@@ -329,7 +337,7 @@ private function next(string $status)
     }
     private function execute_in_builder(string $command)
     {
-        return "docker exec {$this->deployment_uuid} bash -c '{$command}'";
+        return "docker exec {$this->deployment_uuid} bash -c '{$command} |& tee -a /proc/1/fd/1'";
     }
     private function generate_environment_variables($ports)
     {

bootstrap/helpers/applications.php

@@ -34,7 +34,7 @@ function queue_application_deployment(int $application_id, string $deployment_uu
         force_rebuild: $force_rebuild,
         rollback_commit: $commit,
         pull_request_id: $pull_request_id,
-    ));
+    ))->onConnection('long-running')->onQueue('long-running');
 }
 
 function queue_next_deployment(Application $application)
@@ -47,6 +47,6 @@ function queue_next_deployment(Application $application)
             deployment_uuid: $next_found->deployment_uuid,
             force_rebuild: $next_found->force_rebuild,
             pull_request_id: $next_found->pull_request_id
-        ));
+        ))->onConnection('long-running')->onQueue('long-running');
     }
 }

bootstrap/helpers/github.php

@@ -64,7 +64,7 @@ function git_api(GithubApp|GitlabApp $source, string $endpoint, string $method =
     }
     $json = $response->json();
     if ($response->failed() && $throwError) {
-        throw new \Exception("Failed to get data from {$source->name} with error: " . $json['message']);
+        throw new \Exception("Failed to get data from {$source->name} with error:<br><br>" . $json['message']);
     }
     return [
         'rate_limit_remaining' => $response->header('X-RateLimit-Remaining'),

config/horizon.php

@@ -190,7 +190,18 @@
             'maxJobs' => 0,
             'memory' => 128,
             'tries' => 1,
-            'timeout' => 3600,
+            'timeout' => 300,
+            'nice' => 0,
+        ],
+        'long-running' => [
+            'connection' => 'redis',
+            'queue' => ['long-running'],
+            'balance' => 'auto',
+            'maxTime' => 0,
+            'maxJobs' => 0,
+            'memory' => 128,
+            'tries' => 1,
+            'timeout' => 3560,
             'nice' => 0,
         ],
     ],
@@ -203,6 +214,12 @@
                 'balanceMaxShift' => env('HORIZON_BALANCE_MAX_SHIFT', 1),
                 'balanceCooldown' => env('HORIZON_BALANCE_COOLDOWN', 1),
             ],
+            'long-running' => [
+                'autoScalingStrategy' => 'size',
+                'maxProcesses' => env('HORIZON_MAX_PROCESSES', 10),
+                'balanceMaxShift' => env('HORIZON_BALANCE_MAX_SHIFT', 1),
+                'balanceCooldown' => env('HORIZON_BALANCE_COOLDOWN', 1),
+            ],
         ],
 
         'local' => [
@@ -212,6 +229,12 @@
                 'balanceMaxShift' => env('HORIZON_BALANCE_MAX_SHIFT', 1),
                 'balanceCooldown' => env('HORIZON_BALANCE_COOLDOWN', 1),
             ],
+            'long-running' => [
+                'autoScalingStrategy' => 'size',
+                'maxProcesses' => env('HORIZON_MAX_PROCESSES', 10),
+                'balanceMaxShift' => env('HORIZON_BALANCE_MAX_SHIFT', 1),
+                'balanceCooldown' => env('HORIZON_BALANCE_COOLDOWN', 1),
+            ],
         ],
     ],
 ];

config/queue.php

@@ -33,7 +33,14 @@
         'sync' => [
             'driver' => 'sync',
         ],
-
+        'long-running' => [
+            'driver' => 'redis',
+            'connection' => 'default',
+            'queue' => 'long-running',
+            'retry_after' => 3600,
+            'block_for' => null,
+            'after_commit' => false,
+        ],
         'database' => [
             'driver' => 'database',
             'table' => 'jobs',
@@ -66,7 +73,7 @@
             'driver' => 'redis',
             'connection' => 'default',
             'queue' => env('REDIS_QUEUE', 'default'),
-            'retry_after' => 90,
+            'retry_after' => 300,
             'block_for' => null,
             'after_commit' => false,
         ],

docker/coolify-builder/Dockerfile

@@ -35,5 +35,5 @@ RUN if [[ ${TARGETPLATFORM} == 'linux/arm64' ]]; then \
     ;fi
 
 ENTRYPOINT ["/sbin/tini", "--"]
-CMD ["sh", "-c", "while true; do sleep 3600 && exit 0; done"]
+CMD ["sh", "-c", "while true; do sleep 1; done"]
 

resources/views/livewire/project/application/deployment-navbar.blade.php

@@ -1,6 +1,6 @@
 <div class="flex items-center gap-2 pb-4">
     <h2>Logs</h2>
-    @if (data_get($activity, 'properties.status') === 'in_progress')
+    @if (data_get($activity, 'properties.status') === 'in_progress' || data_get($activity, 'properties.status') === 'queued')
         <x-forms.button wire:click.prevent="cancel">Cancel deployment</x-forms.button>
     @endif
 </div>

resources/views/livewire/project/application/deployments.blade.php

@@ -67,8 +67,8 @@ class="hover:no-underline">
              dayjs.extend(window.dayjs_plugin_relativeTime);
 
              Alpine.data('elapsedTime', (uuid, status, created_at, updated_at) => ({
-                 finished_time: '0s',
-                 started_time: '0s',
+                 finished_time: 'calculating...',
+                 started_time: 'calculating...',
                  init() {
                      if (timers[uuid]) {
                          clearInterval(timers[uuid]);

resources/views/vendor/toaster/hub.blade.php

@@ -71,7 +71,7 @@ class="relative flex duration-300 transform transition ease-in-out max-w-md w-fu
                         </svg>
                     </div>
                 </template>
-                <span x-text="toast.message" />
+                <span x-html="toast.message" />
             </i>
 
             @if ($closeable)