diff --git a/app/Actions/CoolifyTask/RunRemoteProcessNew.php b/app/Actions/CoolifyTask/RunRemoteProcessNew.php new file mode 100644 index 000000000..912522cfc --- /dev/null +++ b/app/Actions/CoolifyTask/RunRemoteProcessNew.php @@ -0,0 +1,163 @@ +application = Application::find($application_deployment_queue->application_id)->get(); + } + + public function __invoke(): ProcessResult + { + $this->time_start = hrtime(true); + + $status = ProcessStatus::IN_PROGRESS; + + $processResult = Process::timeout(TIMEOUT)->idleTimeout(IDLE_TIMEOUT)->run($this->getCommand(), $this->handleOutput(...)); + + if ($this->application_deployment_queue->properties->get('status') === ProcessStatus::ERROR->value) { + $status = ProcessStatus::ERROR; + } else { + if (($processResult->exitCode() == 0 && $this->is_finished) || $this->application_deployment_queue->properties->get('status') === ProcessStatus::FINISHED->value) { + $status = ProcessStatus::FINISHED; + } + if ($processResult->exitCode() != 0 && !$this->ignore_errors) { + $status = ProcessStatus::ERROR; + } + } + + $this->application_deployment_queue->properties = $this->application_deployment_queue->properties->merge([ + 'exitCode' => $processResult->exitCode(), + 'stdout' => $processResult->output(), + 'stderr' => $processResult->errorOutput(), + 'status' => $status->value, + ]); + $this->application_deployment_queue->save(); + + if ($processResult->exitCode() != 0 && !$this->ignore_errors) { + throw new \RuntimeException($processResult->errorOutput()); + } + + return $processResult; + } + + protected function getLatestCounter(): int + { + $description = json_decode($this->application_deployment_queue->description, associative: true, flags: JSON_THROW_ON_ERROR); + if ($description === null || count($description) === 0) { + return 1; + } + return end($description)['order'] + 1; + } + + protected function getCommand(): string + { + $user = data_get($this->application_deployment_queue, 'properties.user'); + $server_ip = data_get($this->application_deployment_queue, 'properties.server_ip'); + $private_key_location = data_get($this->application_deployment_queue, 'properties.private_key_location'); + $port = data_get($this->application_deployment_queue, 'properties.port'); + $command = data_get($this->application_deployment_queue, 'properties.command'); + + return generate_ssh_command($private_key_location, $server_ip, $user, $port, $command); + } + + protected function handleOutput(string $type, string $output) + { + if ($this->hide_from_output) { + return; + } + $this->current_time = $this->elapsedTime(); + $this->application_deployment_queue->log = $this->encodeOutput($type, $output); + + if ($this->isAfterLastThrottle()) { + // Let's write to database. + DB::transaction(function () { + $this->application_deployment_queue->save(); + $this->last_write_at = $this->current_time; + }); + } + } + + public function encodeOutput($type, $output) + { + $outputStack = json_decode($this->application_deployment_queue->description, associative: true, flags: JSON_THROW_ON_ERROR); + + $outputStack[] = [ + 'type' => $type, + 'output' => $output, + 'timestamp' => hrtime(true), + 'batch' => ApplicationDeploymentJob::$batch_counter, + 'order' => $this->getLatestCounter(), + ]; + + return json_encode($outputStack, flags: JSON_THROW_ON_ERROR); + } + + public static function decodeOutput(?ApplicationDeploymentQueue $application_deployment_queue = null): string + { + if (is_null($application_deployment_queue)) { + return ''; + } + + try { + $decoded = json_decode( + data_get($application_deployment_queue, 'description'), + associative: true, + flags: JSON_THROW_ON_ERROR + ); + } catch (\JsonException $exception) { + return ''; + } + + return collect($decoded) + ->sortBy(fn ($i) => $i['order']) + ->map(fn ($i) => $i['output']) + ->implode(""); + } + + /** + * Determines if it's time to write again to database. + * + * @return bool + */ + protected function isAfterLastThrottle() + { + // If DB was never written, then we immediately decide we have to write. + if ($this->last_write_at === 0) { + return true; + } + + return ($this->current_time - $this->throttle_interval_ms) > $this->last_write_at; + } + + protected function elapsedTime(): int + { + $timeMs = (hrtime(true) - $this->time_start) / 1_000_000; + + return intval($timeMs); + } +} diff --git a/app/Enums/ApplicationDeploymentStatus.php b/app/Enums/ApplicationDeploymentStatus.php new file mode 100644 index 000000000..6a934c75d --- /dev/null +++ b/app/Enums/ApplicationDeploymentStatus.php @@ -0,0 +1,12 @@ +route('dashboard'); } - $activity = Activity::where('properties->type_uuid', '=', $deploymentUuid)->first(); - if (!$activity) { - return redirect()->route('project.application.deployments', [ - 'project_uuid' => $project->uuid, - 'environment_name' => $environment->name, - 'application_uuid' => $application->uuid, - ]); - } - $deployment = ApplicationDeploymentQueue::where('deployment_uuid', $deploymentUuid)->first(); - if (!$deployment) { + // $activity = Activity::where('properties->type_uuid', '=', $deploymentUuid)->first(); + // if (!$activity) { + // return redirect()->route('project.application.deployments', [ + // 'project_uuid' => $project->uuid, + // 'environment_name' => $environment->name, + // 'application_uuid' => $application->uuid, + // ]); + // } + $application_deployment_queue = ApplicationDeploymentQueue::where('deployment_uuid', $deploymentUuid)->first(); + if (!$application_deployment_queue) { return redirect()->route('project.application.deployments', [ 'project_uuid' => $project->uuid, 'environment_name' => $environment->name, @@ -79,8 +79,8 @@ public function deployment() } return view('project.application.deployment', [ 'application' => $application, - 'activity' => $activity, - 'deployment' => $deployment, + // 'activity' => $activity, + 'application_deployment_queue' => $application_deployment_queue, 'deployment_uuid' => $deploymentUuid, ]); } diff --git a/app/Http/Livewire/Project/Application/DeploymentLogs.php b/app/Http/Livewire/Project/Application/DeploymentLogs.php index 3780608da..b39cc47f7 100644 --- a/app/Http/Livewire/Project/Application/DeploymentLogs.php +++ b/app/Http/Livewire/Project/Application/DeploymentLogs.php @@ -2,31 +2,23 @@ namespace App\Http\Livewire\Project\Application; -use App\Enums\ActivityTypes; -use App\Models\Application; -use Illuminate\Support\Facades\Redis; +use App\Models\ApplicationDeploymentQueue; use Livewire\Component; -use Spatie\Activitylog\Models\Activity; class DeploymentLogs extends Component { - public Application $application; - public $activity; + public ApplicationDeploymentQueue $application_deployment_queue; public $isKeepAliveOn = true; - public $deployment_uuid; + protected $listeners = ['refreshQueue']; + public function refreshQueue() + { + $this->application_deployment_queue->refresh(); + } public function polling() { $this->emit('deploymentFinished'); - if (is_null($this->activity) && isset($this->deployment_uuid)) { - $this->activity = Activity::query() - ->where('properties->type', '=', ActivityTypes::DEPLOYMENT->value) - ->where('properties->type_uuid', '=', $this->deployment_uuid) - ->first(); - } else { - $this->activity?->refresh(); - } - - if (data_get($this->activity, 'properties.status') == 'finished' || data_get($this->activity, 'properties.status') == 'failed') { + $this->application_deployment_queue->refresh(); + if (data_get($this->application_deployment_queue, 'status') == 'finished' || data_get($this->application_deployment_queue, 'status') == 'failed') { $this->isKeepAliveOn = false; } } diff --git a/app/Http/Livewire/Project/Application/DeploymentNavbar.php b/app/Http/Livewire/Project/Application/DeploymentNavbar.php index 8f39ff762..fb4692158 100644 --- a/app/Http/Livewire/Project/Application/DeploymentNavbar.php +++ b/app/Http/Livewire/Project/Application/DeploymentNavbar.php @@ -2,41 +2,46 @@ namespace App\Http\Livewire\Project\Application; -use App\Enums\ProcessStatus; +use App\Enums\ApplicationDeploymentStatus; use App\Models\Application; use App\Models\ApplicationDeploymentQueue; +use Illuminate\Support\Facades\Process; use Livewire\Component; +use Illuminate\Support\Str; class DeploymentNavbar extends Component { - public Application $application; - public $activity; - public string $deployment_uuid; protected $listeners = ['deploymentFinished']; + + public ApplicationDeploymentQueue $application_deployment_queue; + public function deploymentFinished() { - $this->activity->refresh(); + $this->application_deployment_queue->refresh(); + } + public function show_debug() + { + $application = Application::find($this->application_deployment_queue->application_id); + $application->settings->is_debug_enabled = !$application->settings->is_debug_enabled; + $application->settings->save(); + $this->emit('refreshQueue'); } public function cancel() { try { - ray('Cancelling deployment: ' . $this->deployment_uuid . ' of application: ' . $this->application->uuid); - - // Update deployment queue - $deployment = ApplicationDeploymentQueue::where('deployment_uuid', $this->deployment_uuid)->first(); - $deployment->status = 'cancelled by user'; - $deployment->save(); - - // Update activity - $this->activity->properties = $this->activity->properties->merge([ - 'exitCode' => 1, - 'status' => ProcessStatus::CANCELLED->value, - ]); - $this->activity->save(); - - // Remove builder container - instant_remote_process(["docker rm -f {$this->deployment_uuid}"], $this->application->destination->server, throwError: false, repeat: 25); - queue_next_deployment($this->application); + $application = Application::find($this->application_deployment_queue->application_id); + $server = $application->destination->server; + if ($this->application_deployment_queue->current_process_id) { + $process = Process::run("ps -p {$this->application_deployment_queue->current_process_id} -o command --no-headers"); + if (Str::of($process->output())->contains([$server->ip, 'EOF-COOLIFY-SSH'])) { + Process::run("kill -9 {$this->application_deployment_queue->current_process_id}"); + } + // TODO: Cancelling text in logs + $this->application_deployment_queue->update([ + 'current_process_id' => null, + 'status' => ApplicationDeploymentStatus::CANCELLED_BY_USER->value, + ]); + } } catch (\Throwable $e) { return general_error_handler(err: $e, that: $this); } diff --git a/app/Jobs/ApplicationDeploymentJobNew.php b/app/Jobs/ApplicationDeploymentJobNew.php new file mode 100644 index 000000000..05469b29a --- /dev/null +++ b/app/Jobs/ApplicationDeploymentJobNew.php @@ -0,0 +1,181 @@ +application_deployment_queue = ApplicationDeploymentQueue::find($application_deployment_queue_id); + $this->application = Application::find($this->application_deployment_queue->application_id); + + $this->application_deployment_queue_id = $application_deployment_queue_id; + $this->deployment_uuid = $this->application_deployment_queue->deployment_uuid; + $this->pull_request_id = $this->application_deployment_queue->pull_request_id; + $this->commit = $this->application_deployment_queue->commit; + $this->force_rebuild = $this->application_deployment_queue->force_rebuild; + + $this->source = $this->application->source->getMorphClass()::where('id', $this->application->source->id)->first(); + $this->destination = $this->application->destination->getMorphClass()::where('id', $this->application->destination->id)->first(); + $this->server = $this->destination->server; + $this->private_key_location = save_private_key_for_server($this->server); + + $this->workdir = "/artifacts/{$this->deployment_uuid}"; + $this->is_debug_enabled = $this->application->settings->is_debug_enabled; + + $this->container_name = generate_container_name($this->application->uuid); + $this->private_key_location = save_private_key_for_server($this->server); + + // Set preview fqdn + if ($this->pull_request_id !== 0) { + $this->preview = ApplicationPreview::findPreviewByApplicationAndPullId($this->application->id, $this->pull_request_id); + if ($this->application->fqdn) { + $preview_fqdn = data_get($this->preview, 'fqdn'); + $template = $this->application->preview_url_template; + $url = Url::fromString($this->application->fqdn); + $host = $url->getHost(); + $schema = $url->getScheme(); + $random = new Cuid2(7); + $preview_fqdn = str_replace('{{random}}', $random, $template); + $preview_fqdn = str_replace('{{domain}}', $host, $preview_fqdn); + $preview_fqdn = str_replace('{{pr_id}}', $this->pull_request_id, $preview_fqdn); + $preview_fqdn = "$schema://$preview_fqdn"; + $this->preview->fqdn = $preview_fqdn; + $this->preview->save(); + } + } + } + + public function handle(): void + { + $this->application_deployment_queue->update([ + 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value, + ]); + try { + if ($this->pull_request_id !== 0) { + // $this->deploy_pull_request(); + } else { + $this->deploy(); + } + } catch (\Exception $e) { + // $this->execute_now([ + // "echo '\nOops something is not okay, are you okay? 😢'", + // "echo '\n\n{$e->getMessage()}'", + // ]); + $this->fail($e->getMessage()); + } finally { + // if (isset($this->docker_compose)) { + // Storage::disk('deployments')->put(Str::kebab($this->application->name) . '/docker-compose.yml', $this->docker_compose); + // } + // execute_remote_command( + // commands: [ + // "docker rm -f {$this->deployment_uuid} >/dev/null 2>&1" + // ], + // server: $this->server, + // queue: $this->application_deployment_queue, + // hide_from_output: true, + // ); + } + } + public function failed(Throwable $exception): void + { + ray($exception); + $this->next(ApplicationDeploymentStatus::FAILED->value); + } + private function execute_in_builder(string $command) + { + return "docker exec {$this->deployment_uuid} bash -c '{$command} |& tee -a /proc/1/fd/1'"; + } + private function deploy() + { + execute_remote_command( + commands: [ + "echo -n 'Pulling latest version of the builder image (ghcr.io/coollabsio/coolify-builder).'", + ], + server: $this->server, + queue: $this->application_deployment_queue, + ); + execute_remote_command( + commands: [ + "docker run --pull=always -d --name {$this->deployment_uuid} --rm -v /var/run/docker.sock:/var/run/docker.sock ghcr.io/coollabsio/coolify-builder", + ], + server: $this->server, + queue: $this->application_deployment_queue, + show_in_output: false, + ); + execute_remote_command( + commands: [ + "echo 'Done.'", + ], + server: $this->server, + queue: $this->application_deployment_queue, + ); + execute_remote_command( + commands: [ + $this->execute_in_builder("mkdir -p {$this->workdir}") + ], + server: $this->server, + queue: $this->application_deployment_queue, + ); + execute_remote_command( + commands: [ + "echos hello" + ], + server: $this->server, + queue: $this->application_deployment_queue, + ); + $this->next(ApplicationDeploymentStatus::FINISHED->value); + } + + private function next(string $status) + { + // If the deployment is cancelled by the user, don't update the status + if ($this->application_deployment_queue->status !== ApplicationDeploymentStatus::CANCELLED_BY_USER->value) { + $this->application_deployment_queue->update([ + 'status' => $status, + ]); + } + queue_next_deployment($this->application); + } +} diff --git a/app/Models/ApplicationDeploymentQueue.php b/app/Models/ApplicationDeploymentQueue.php index 39642813a..1b7ae4781 100644 --- a/app/Models/ApplicationDeploymentQueue.php +++ b/app/Models/ApplicationDeploymentQueue.php @@ -6,13 +6,5 @@ class ApplicationDeploymentQueue extends Model { - protected $fillable = [ - 'application_id', - 'deployment_uuid', - 'pull_request_id', - 'force_rebuild', - 'commit', - 'status', - 'is_webhook', - ]; + protected $guarded = []; } diff --git a/bootstrap/helpers/applications.php b/bootstrap/helpers/applications.php index 0f69d42ca..c7e498463 100644 --- a/bootstrap/helpers/applications.php +++ b/bootstrap/helpers/applications.php @@ -1,6 +1,7 @@ count() > 0) { return; } - dispatch(new ApplicationDeploymentJob( + // dispatch(new ApplicationDeploymentJob( + // application_deployment_queue_id: $deployment->id, + // application_id: $application_id, + // deployment_uuid: $deployment_uuid, + // force_rebuild: $force_rebuild, + // rollback_commit: $commit, + // pull_request_id: $pull_request_id, + // ))->onConnection('long-running')->onQueue('long-running'); + dispatch(new ApplicationDeploymentJobNew( application_deployment_queue_id: $deployment->id, - application_id: $application_id, - deployment_uuid: $deployment_uuid, - force_rebuild: $force_rebuild, - rollback_commit: $commit, - pull_request_id: $pull_request_id, ))->onConnection('long-running')->onQueue('long-running'); } diff --git a/bootstrap/helpers/remoteProcess.php b/bootstrap/helpers/remoteProcess.php index 00ffaa89e..b0e283d70 100644 --- a/bootstrap/helpers/remoteProcess.php +++ b/bootstrap/helpers/remoteProcess.php @@ -3,8 +3,15 @@ use App\Actions\CoolifyTask\PrepareCoolifyTask; use App\Data\CoolifyTaskArgs; use App\Enums\ActivityTypes; +use App\Enums\ApplicationDeploymentStatus; +use App\Jobs\ApplicationDeploymentJobNew; +use App\Models\Application; +use App\Models\ApplicationDeploymentQueue; +use App\Models\InstanceSettings; use App\Models\Server; +use Carbon\Carbon; use Illuminate\Database\Eloquent\Model; +use Illuminate\Support\Collection; use Illuminate\Support\Facades\Process; use Illuminate\Support\Facades\Storage; use Illuminate\Support\Sleep; @@ -47,6 +54,11 @@ function remote_process( ), ])(); } +function get_private_key_for_server(Server $server) +{ + $temp_file = "id.root@{$server->ip}"; + return '/var/www/html/storage/app/ssh/keys/' . $temp_file; +} function save_private_key_for_server(Server $server) { if (data_get($server, 'privateKey.private_key') === null) { @@ -106,3 +118,81 @@ function instant_remote_process(array $command, Server $server, $throwError = tr } return $output; } + +function decode_remote_command_output(?ApplicationDeploymentQueue $application_deployment_queue = null): Collection +{ + $application = Application::find(data_get($application_deployment_queue, 'application_id')); + $is_debug_enabled = data_get($application, 'settings.is_debug_enabled'); + if (is_null($application_deployment_queue)) { + return collect([]); + } + try { + $decoded = json_decode( + data_get($application_deployment_queue, 'log'), + associative: true, + flags: JSON_THROW_ON_ERROR + ); + } catch (\JsonException $exception) { + return collect([]); + } + $formatted = collect($decoded); + if (!$is_debug_enabled) { + + $formatted = $formatted->filter(fn ($i) => $i['show_in_output'] ?? true); + } + $formatted = $formatted->sortBy(fn ($i) => $i['order']) + ->map(function ($i) { + $i['timestamp'] = Carbon::parse($i['timestamp'])->format('Y-M-d H:i:s.u'); + return $i; + }); + + return $formatted; +} +function execute_remote_command(array|Collection $commands, Server $server, ApplicationDeploymentQueue $queue, bool $show_in_output = true, bool $ignore_errors = false) +{ + if ($commands instanceof Collection) { + $commandsText = $commands; + } else { + $commandsText = collect($commands); + } + $ip = data_get($server, 'ip'); + $user = data_get($server, 'user'); + $port = data_get($server, 'port'); + $private_key_location = get_private_key_for_server($server); + $commandsText->each(function ($command) use ($queue, $private_key_location, $ip, $user, $port, $show_in_output, $ignore_errors) { + $remote_command = generate_ssh_command($private_key_location, $ip, $user, $port, $command); + $process = Process::timeout(3600)->idleTimeout(3600)->start($remote_command, function (string $type, string $output) use ($queue, $command, $show_in_output) { + $new_log_entry = [ + 'command' => $command, + 'output' => $output, + 'type' => $type === 'err' ? 'stderr' : 'stdout', + 'timestamp' => Carbon::now('UTC'), + 'show_in_output' => $show_in_output, + ]; + + if (!$queue->log) { + $new_log_entry['order'] = 1; + } else { + $previous_logs = json_decode($queue->log, associative: true, flags: JSON_THROW_ON_ERROR); + $new_log_entry['order'] = count($previous_logs) + 1; + } + + $previous_logs[] = $new_log_entry; + $queue->log = json_encode($previous_logs, flags: JSON_THROW_ON_ERROR);; + $queue->save(); + }); + $queue->update([ + 'current_process_id' => $process->id(), + ]); + + $process_result = $process->wait(); + if ($process_result->exitCode() !== 0) { + if (!$ignore_errors) { + $status = ApplicationDeploymentStatus::FAILED->value; + $queue->status = $status; + $queue->save(); + throw new \RuntimeException($process_result->errorOutput()); + } + } + }); +} diff --git a/database/migrations/2023_06_23_114133_use_application_deployment_queues_as_activity.php b/database/migrations/2023_06_23_114133_use_application_deployment_queues_as_activity.php new file mode 100644 index 000000000..2e142f788 --- /dev/null +++ b/database/migrations/2023_06_23_114133_use_application_deployment_queues_as_activity.php @@ -0,0 +1,30 @@ +text('log')->default(null)->nullable(); + $table->string('current_process_id')->default(null)->nullable(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('application_deployment_queues', function (Blueprint $table) { + $table->dropColumn('log'); + $table->dropColumn('current_process_id'); + }); + } +}; diff --git a/resources/views/livewire/project/application/deployment-logs.blade.php b/resources/views/livewire/project/application/deployment-logs.blade.php index 5a9fb5688..1da9c1bbb 100644 --- a/resources/views/livewire/project/application/deployment-logs.blade.php +++ b/resources/views/livewire/project/application/deployment-logs.blade.php @@ -1,18 +1,32 @@
- - @if (data_get($activity, 'properties.status') === 'in_progress') + + @if (data_get($application_deployment_queue, 'status') === 'in_progress')
Deployment is -
{{ Str::headline(data_get($activity, 'properties.status')) }}.
+
{{ Str::headline(data_get($this->application_deployment_queue, 'status')) }}. +
Logs will be updated automatically.
@else
Deployment is {{ Str::headline(data_get($activity, 'properties.status')) }}. + class="text-warning">{{ Str::headline(data_get($application_deployment_queue, 'status')) }}.
@endif -
- 
{{ \App\Actions\CoolifyTask\RunRemoteProcess::decodeOutput($activity) }}
+
+ + @if (decode_remote_command_output($application_deployment_queue)->count() > 0) + @foreach (decode_remote_command_output($application_deployment_queue) as $line) + $line['type'] == 'stdout', + 'text-error' => $line['type'] == 'stderr', + ])> + [{{ $line['timestamp'] }}] {{ $line['output'] }} + @endforeach + @else + No logs yet. + @endif +
diff --git a/resources/views/livewire/project/application/deployment-navbar.blade.php b/resources/views/livewire/project/application/deployment-navbar.blade.php index 87654251e..8566c1faf 100644 --- a/resources/views/livewire/project/application/deployment-navbar.blade.php +++ b/resources/views/livewire/project/application/deployment-navbar.blade.php @@ -1,6 +1,8 @@

Logs

- @if (data_get($activity, 'properties.status') === 'in_progress' || data_get($activity, 'properties.status') === 'queued') + Show Debug Logs + @if (data_get($application_deployment_queue, 'status') === 'in_progress' || + data_get($application_deployment_queue, 'status') === 'queued') Cancel deployment @endif
diff --git a/resources/views/project/application/deployment.blade.php b/resources/views/project/application/deployment.blade.php index 4b855a16b..8f36eed2b 100644 --- a/resources/views/project/application/deployment.blade.php +++ b/resources/views/project/application/deployment.blade.php @@ -1,5 +1,5 @@

Deployment

- +