feat: long running queue with 1 hour of timeout
This commit is contained in:
Andras Bacsai
parent
ba18c589f0
commit
80af200c9f
@ -17,12 +17,12 @@
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Collection;
|
||||
use Illuminate\Support\Facades\Notification;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use Spatie\Activitylog\Models\Activity;
|
||||
use Symfony\Component\Yaml\Yaml;
|
||||
use Illuminate\Support\Str;
|
||||
use Spatie\Url\Url;
|
||||
use Throwable;
|
||||
use Visus\Cuid2\Cuid2;
|
||||
|
||||
class ApplicationDeploymentJob implements ShouldQueue
|
||||
@ -46,7 +46,7 @@ class ApplicationDeploymentJob implements ShouldQueue
|
||||
private ApplicationPreview|null $preview = null;
|
||||
|
||||
public static int $batch_counter = 0;
|
||||
public $timeout = 10200;
|
||||
public $failOnTimeout = true;
|
||||
|
||||
public function __construct(
|
||||
public int $application_deployment_queue_id,
|
||||
@ -128,7 +128,8 @@ public function handle(): void
|
||||
"echo '\nOops something is not okay, are you okay? 😢'",
|
||||
"echo '\n\n{$e->getMessage()}'",
|
||||
]);
|
||||
$this->fail();
|
||||
ray($e);
|
||||
$this->fail($e->getMessage());
|
||||
} finally {
|
||||
if (isset($this->docker_compose)) {
|
||||
Storage::disk('deployments')->put(Str::kebab($this->application->name) . '/docker-compose.yml', $this->docker_compose);
|
||||
@ -142,9 +143,15 @@ private function start_builder_image()
|
||||
$this->execute_now([
|
||||
"echo -n 'Pulling latest version of the builder image (ghcr.io/coollabsio/coolify-builder)... '",
|
||||
]);
|
||||
$this->execute_now([
|
||||
"docker run --pull=always -d --name {$this->deployment_uuid} --rm -v /var/run/docker.sock:/var/run/docker.sock ghcr.io/coollabsio/coolify-builder",
|
||||
], isDebuggable: true);
|
||||
if (isDev()) {
|
||||
$this->execute_now([
|
||||
"docker run -d --name {$this->deployment_uuid} --rm -v /var/run/docker.sock:/var/run/docker.sock coolify-builder",
|
||||
], isDebuggable: true);
|
||||
} else {
|
||||
$this->execute_now([
|
||||
"docker run --pull=always -d --name {$this->deployment_uuid} --rm -v /var/run/docker.sock:/var/run/docker.sock ghcr.io/coollabsio/coolify-builder",
|
||||
], isDebuggable: true);
|
||||
}
|
||||
$this->execute_now([
|
||||
"echo 'Done.'"
|
||||
]);
|
||||
@ -291,8 +298,9 @@ private function deploy()
|
||||
$this->next(ProcessStatus::FINISHED->value);
|
||||
}
|
||||
|
||||
public function failed(): void
|
||||
public function failed(Throwable $exception): void
|
||||
{
|
||||
ray($exception);
|
||||
$this->next(ProcessStatus::ERROR->value);
|
||||
}
|
||||
|
||||
@ -329,7 +337,7 @@ private function next(string $status)
|
||||
}
|
||||
private function execute_in_builder(string $command)
|
||||
{
|
||||
return "docker exec {$this->deployment_uuid} bash -c '{$command}'";
|
||||
return "docker exec {$this->deployment_uuid} bash -c '{$command} |& tee -a /proc/1/fd/1'";
|
||||
}
|
||||
private function generate_environment_variables($ports)
|
||||
{
|
||||
|
||||
@ -34,7 +34,7 @@ function queue_application_deployment(int $application_id, string $deployment_uu
|
||||
force_rebuild: $force_rebuild,
|
||||
rollback_commit: $commit,
|
||||
pull_request_id: $pull_request_id,
|
||||
));
|
||||
))->onConnection('long-running')->onQueue('long-running');
|
||||
}
|
||||
|
||||
function queue_next_deployment(Application $application)
|
||||
@ -47,6 +47,6 @@ function queue_next_deployment(Application $application)
|
||||
deployment_uuid: $next_found->deployment_uuid,
|
||||
force_rebuild: $next_found->force_rebuild,
|
||||
pull_request_id: $next_found->pull_request_id
|
||||
));
|
||||
))->onConnection('long-running')->onQueue('long-running');
|
||||
}
|
||||
}
|
||||
|
||||
@ -64,7 +64,7 @@ function git_api(GithubApp|GitlabApp $source, string $endpoint, string $method =
|
||||
}
|
||||
$json = $response->json();
|
||||
if ($response->failed() && $throwError) {
|
||||
throw new \Exception("Failed to get data from {$source->name} with error: " . $json['message']);
|
||||
throw new \Exception("Failed to get data from {$source->name} with error:<br><br>" . $json['message']);
|
||||
}
|
||||
return [
|
||||
'rate_limit_remaining' => $response->header('X-RateLimit-Remaining'),
|
||||
|
||||
@ -190,7 +190,18 @@
|
||||
'maxJobs' => 0,
|
||||
'memory' => 128,
|
||||
'tries' => 1,
|
||||
'timeout' => 3600,
|
||||
'timeout' => 300,
|
||||
'nice' => 0,
|
||||
],
|
||||
'long-running' => [
|
||||
'connection' => 'redis',
|
||||
'queue' => ['long-running'],
|
||||
'balance' => 'auto',
|
||||
'maxTime' => 0,
|
||||
'maxJobs' => 0,
|
||||
'memory' => 128,
|
||||
'tries' => 1,
|
||||
'timeout' => 3560,
|
||||
'nice' => 0,
|
||||
],
|
||||
],
|
||||
@ -203,6 +214,12 @@
|
||||
'balanceMaxShift' => env('HORIZON_BALANCE_MAX_SHIFT', 1),
|
||||
'balanceCooldown' => env('HORIZON_BALANCE_COOLDOWN', 1),
|
||||
],
|
||||
'long-running' => [
|
||||
'autoScalingStrategy' => 'size',
|
||||
'maxProcesses' => env('HORIZON_MAX_PROCESSES', 10),
|
||||
'balanceMaxShift' => env('HORIZON_BALANCE_MAX_SHIFT', 1),
|
||||
'balanceCooldown' => env('HORIZON_BALANCE_COOLDOWN', 1),
|
||||
],
|
||||
],
|
||||
|
||||
'local' => [
|
||||
@ -212,6 +229,12 @@
|
||||
'balanceMaxShift' => env('HORIZON_BALANCE_MAX_SHIFT', 1),
|
||||
'balanceCooldown' => env('HORIZON_BALANCE_COOLDOWN', 1),
|
||||
],
|
||||
'long-running' => [
|
||||
'autoScalingStrategy' => 'size',
|
||||
'maxProcesses' => env('HORIZON_MAX_PROCESSES', 10),
|
||||
'balanceMaxShift' => env('HORIZON_BALANCE_MAX_SHIFT', 1),
|
||||
'balanceCooldown' => env('HORIZON_BALANCE_COOLDOWN', 1),
|
||||
],
|
||||
],
|
||||
],
|
||||
];
|
||||
|
||||
@ -33,7 +33,14 @@
|
||||
'sync' => [
|
||||
'driver' => 'sync',
|
||||
],
|
||||
|
||||
'long-running' => [
|
||||
'driver' => 'redis',
|
||||
'connection' => 'default',
|
||||
'queue' => 'long-running',
|
||||
'retry_after' => 3600,
|
||||
'block_for' => null,
|
||||
'after_commit' => false,
|
||||
],
|
||||
'database' => [
|
||||
'driver' => 'database',
|
||||
'table' => 'jobs',
|
||||
@ -66,7 +73,7 @@
|
||||
'driver' => 'redis',
|
||||
'connection' => 'default',
|
||||
'queue' => env('REDIS_QUEUE', 'default'),
|
||||
'retry_after' => 90,
|
||||
'retry_after' => 300,
|
||||
'block_for' => null,
|
||||
'after_commit' => false,
|
||||
],
|
||||
|
||||
@ -35,5 +35,5 @@ RUN if [[ ${TARGETPLATFORM} == 'linux/arm64' ]]; then \
|
||||
;fi
|
||||
|
||||
ENTRYPOINT ["/sbin/tini", "--"]
|
||||
CMD ["sh", "-c", "while true; do sleep 3600 && exit 0; done"]
|
||||
CMD ["sh", "-c", "while true; do sleep 1; done"]
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
<div class="flex items-center gap-2 pb-4">
|
||||
<h2>Logs</h2>
|
||||
@if (data_get($activity, 'properties.status') === 'in_progress')
|
||||
@if (data_get($activity, 'properties.status') === 'in_progress' || data_get($activity, 'properties.status') === 'queued')
|
||||
<x-forms.button wire:click.prevent="cancel">Cancel deployment</x-forms.button>
|
||||
@endif
|
||||
</div>
|
||||
|
||||
@ -67,8 +67,8 @@ class="hover:no-underline">
|
||||
dayjs.extend(window.dayjs_plugin_relativeTime);
|
||||
|
||||
Alpine.data('elapsedTime', (uuid, status, created_at, updated_at) => ({
|
||||
finished_time: '0s',
|
||||
started_time: '0s',
|
||||
finished_time: 'calculating...',
|
||||
started_time: 'calculating...',
|
||||
init() {
|
||||
if (timers[uuid]) {
|
||||
clearInterval(timers[uuid]);
|
||||
|
||||
2
resources/views/vendor/toaster/hub.blade.php
vendored
2
resources/views/vendor/toaster/hub.blade.php
vendored
@ -71,7 +71,7 @@ class="relative flex duration-300 transform transition ease-in-out max-w-md w-fu
|
||||
</svg>
|
||||
</div>
|
||||
</template>
|
||||
<span x-text="toast.message" />
|
||||
<span x-html="toast.message" />
|
||||
</i>
|
||||
|
||||
@if ($closeable)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user