diff --git a/app/Actions/CoolifyTask/RunRemoteProcess.php b/app/Actions/CoolifyTask/RunRemoteProcess.php index 1444684a0..e213f57f1 100644 --- a/app/Actions/CoolifyTask/RunRemoteProcess.php +++ b/app/Actions/CoolifyTask/RunRemoteProcess.php @@ -10,9 +10,6 @@ use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Process; use Spatie\Activitylog\Models\Activity; -const TIMEOUT = 3600; -const IDLE_TIMEOUT = 3600; - class RunRemoteProcess { public Activity $activity; @@ -76,8 +73,7 @@ class RunRemoteProcess $this->time_start = hrtime(true); $status = ProcessStatus::IN_PROGRESS; - - $processResult = Process::timeout(TIMEOUT)->idleTimeout(IDLE_TIMEOUT)->run($this->getCommand(), $this->handleOutput(...)); + $processResult = processWithEnv()->forever()->run($this->getCommand(), $this->handleOutput(...)); if ($this->activity->properties->get('status') === ProcessStatus::ERROR->value) { $status = ProcessStatus::ERROR; @@ -108,11 +104,10 @@ class RunRemoteProcess { $user = $this->activity->getExtraProperty('user'); $server_ip = $this->activity->getExtraProperty('server_ip'); - $private_key_location = $this->activity->getExtraProperty('private_key_location'); $port = $this->activity->getExtraProperty('port'); $command = $this->activity->getExtraProperty('command'); - return generate_ssh_command($private_key_location, $server_ip, $user, $port, $command); + return generateSshCommand($server_ip, $user, $port, $command); } protected function handleOutput(string $type, string $output) diff --git a/app/Actions/Proxy/StartProxy.php b/app/Actions/Proxy/StartProxy.php index d8a14d27e..7e2878b7e 100644 --- a/app/Actions/Proxy/StartProxy.php +++ b/app/Actions/Proxy/StartProxy.php @@ -8,7 +8,7 @@ use Spatie\Activitylog\Models\Activity; class StartProxy { - public function __invoke(Server $server): Activity + public function __invoke(Server $server, bool $async = true): Activity|string { $proxy_path = get_proxy_path(); $networks = collect($server->standaloneDockers)->map(function ($docker) { @@ -26,8 +26,7 @@ class StartProxy $docker_compose_yml_base64 = base64_encode($configuration); $server->proxy->last_applied_settings = Str::of($docker_compose_yml_base64)->pipe('md5')->value; $server->save(); - - $activity = remote_process([ + $commands = [ "echo '####### Creating required Docker networks...'", ...$create_networks_command, "cd $proxy_path", @@ -44,8 +43,13 @@ class StartProxy "echo '####### Starting coolify-proxy...'", 'docker compose up -d --remove-orphans', "echo '####### Proxy installed successfully...'" - ], $server); - - return $activity; + ]; + if (!$async) { + instant_remote_process($commands, $server); + return 'OK'; + } else { + $activity = remote_process($commands, $server); + return $activity; + } } } diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index ae40fec49..d15107c19 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -2,21 +2,15 @@ namespace App\Console; -use App\Enums\ProxyTypes; -use App\Jobs\ApplicationContainerStatusJob; use App\Jobs\CheckResaleLicenseJob; use App\Jobs\CleanupInstanceStuffsJob; use App\Jobs\DatabaseBackupJob; -use App\Jobs\DatabaseContainerStatusJob; use App\Jobs\DockerCleanupJob; use App\Jobs\InstanceAutoUpdateJob; -use App\Jobs\ProxyContainerStatusJob; -use App\Jobs\ServerDetailsCheckJob; -use App\Models\Application; +use App\Jobs\ContainerStatusJob; use App\Models\InstanceSettings; use App\Models\ScheduledDatabaseBackup; use App\Models\Server; -use App\Models\StandalonePostgresql; use Illuminate\Console\Scheduling\Schedule; use Illuminate\Foundation\Console\Kernel as ConsoleKernel; @@ -25,15 +19,14 @@ class Kernel extends ConsoleKernel protected function schedule(Schedule $schedule): void { if (isDev()) { - $schedule->job(new ServerDetailsCheckJob(Server::find(0)))->everyTenMinutes()->onOneServer(); + // $schedule->job(new ContainerStatusJob(Server::find(0)))->everyTenMinutes()->onOneServer(); // $schedule->command('horizon:snapshot')->everyMinute(); // $schedule->job(new CleanupInstanceStuffsJob)->everyMinute(); // $schedule->job(new CheckResaleLicenseJob)->hourly(); // $schedule->job(new DockerCleanupJob)->everyOddHour(); // $this->instance_auto_update($schedule); // $this->check_scheduled_backups($schedule); - // $this->check_resources($schedule); - // $this->check_proxies($schedule); + $this->check_resources($schedule); } else { $schedule->command('horizon:snapshot')->everyFiveMinutes(); $schedule->job(new CleanupInstanceStuffsJob)->everyTenMinutes()->onOneServer(); @@ -42,26 +35,15 @@ class Kernel extends ConsoleKernel $this->instance_auto_update($schedule); $this->check_scheduled_backups($schedule); $this->check_resources($schedule); - $this->check_proxies($schedule); - } - } - private function check_proxies($schedule) - { - $servers = Server::all()->where('settings.is_usable', true)->where('settings.is_reachable', true)->whereNotNull('proxy.type')->where('proxy.type', '!=', ProxyTypes::NONE->value); - foreach ($servers as $server) { - $schedule->job(new ProxyContainerStatusJob($server))->everyMinute()->onOneServer(); } } private function check_resources($schedule) { - $applications = Application::all(); - foreach ($applications as $application) { - $schedule->job(new ApplicationContainerStatusJob($application))->everyMinute()->onOneServer(); - } + $servers = Server::all()->where('settings.is_usable', true)->where('settings.is_reachable', true); + ray($servers); - $postgresqls = StandalonePostgresql::all(); - foreach ($postgresqls as $postgresql) { - $schedule->job(new DatabaseContainerStatusJob($postgresql))->everyMinute()->onOneServer(); + foreach ($servers as $server) { + $schedule->job(new ContainerStatusJob($server))->everyMinute()->onOneServer(); } } private function instance_auto_update($schedule) diff --git a/app/Data/CoolifyTaskArgs.php b/app/Data/CoolifyTaskArgs.php index 1b30126f0..44e62147c 100644 --- a/app/Data/CoolifyTaskArgs.php +++ b/app/Data/CoolifyTaskArgs.php @@ -13,7 +13,6 @@ class CoolifyTaskArgs extends Data { public function __construct( public string $server_ip, - public string $private_key_location, public string $command, public int $port, public string $user, diff --git a/app/Http/Livewire/Boarding/Index.php b/app/Http/Livewire/Boarding/Index.php index 2cb3b1049..77675441d 100644 --- a/app/Http/Livewire/Boarding/Index.php +++ b/app/Http/Livewire/Boarding/Index.php @@ -53,12 +53,13 @@ uZx9iFkCELtxrh31QJ68AAAAEXNhaWxANzZmZjY2ZDJlMmRkAQIDBA== $this->remoteServerHost = 'coolify-testing-host'; } } - public function welcome() { + public function explanation() { if (isCloud()) { return $this->setServerType('remote'); } $this->currentState = 'select-server-type'; } + public function restartBoarding() { if ($this->createdServer) { diff --git a/app/Http/Livewire/Project/Application/Heading.php b/app/Http/Livewire/Project/Application/Heading.php index 5b9560007..8c0da7a1e 100644 --- a/app/Http/Livewire/Project/Application/Heading.php +++ b/app/Http/Livewire/Project/Application/Heading.php @@ -2,9 +2,8 @@ namespace App\Http\Livewire\Project\Application; -use App\Jobs\ApplicationContainerStatusJob; +use App\Jobs\ContainerStatusJob; use App\Models\Application; -use App\Notifications\Application\StatusChanged; use Livewire\Component; use Visus\Cuid2\Cuid2; @@ -22,10 +21,11 @@ class Heading extends Component public function check_status() { - dispatch_sync(new ApplicationContainerStatusJob( - application: $this->application, - )); + dispatch_sync(new ContainerStatusJob($this->application->destination->server)); $this->application->refresh(); + $this->application->previews->each(function ($preview) { + $preview->refresh(); + }); } public function force_deploy_without_cache() diff --git a/app/Http/Livewire/Project/Application/Previews.php b/app/Http/Livewire/Project/Application/Previews.php index ad3ca5ad9..9633d03a6 100644 --- a/app/Http/Livewire/Project/Application/Previews.php +++ b/app/Http/Livewire/Project/Application/Previews.php @@ -3,6 +3,7 @@ namespace App\Http\Livewire\Project\Application; use App\Jobs\ApplicationContainerStatusJob; +use App\Jobs\ContainerStatusJob; use App\Models\Application; use App\Models\ApplicationPreview; use Illuminate\Support\Collection; @@ -23,14 +24,6 @@ class Previews extends Component $this->parameters = get_route_parameters(); } - public function loadStatus($pull_request_id) - { - dispatch(new ApplicationContainerStatusJob( - application: $this->application, - pullRequestId: $pull_request_id - )); - } - public function load_prs() { try { diff --git a/app/Http/Livewire/Project/Database/Heading.php b/app/Http/Livewire/Project/Database/Heading.php index 178cbfca5..f2fe8f033 100644 --- a/app/Http/Livewire/Project/Database/Heading.php +++ b/app/Http/Livewire/Project/Database/Heading.php @@ -3,8 +3,7 @@ namespace App\Http\Livewire\Project\Database; use App\Actions\Database\StartPostgresql; -use App\Jobs\DatabaseContainerStatusJob; -use App\Notifications\Application\StatusChanged; +use App\Jobs\ContainerStatusJob; use Livewire\Component; class Heading extends Component @@ -25,9 +24,7 @@ class Heading extends Component public function check_status() { - dispatch_sync(new DatabaseContainerStatusJob( - database: $this->database, - )); + dispatch_sync(new ContainerStatusJob($this->database->destination->server)); $this->database->refresh(); } diff --git a/app/Http/Livewire/Server/Form.php b/app/Http/Livewire/Server/Form.php index 53cf8234d..b288d3bfa 100644 --- a/app/Http/Livewire/Server/Form.php +++ b/app/Http/Livewire/Server/Form.php @@ -56,7 +56,7 @@ class Form extends Component $this->uptime = $uptime; $this->emit('success', 'Server is reachable!'); } else { - $this->emit('error', 'Server is not rachable'); + $this->emit('error', 'Server is not reachable'); return; } if ($dockerVersion) { @@ -88,17 +88,11 @@ class Form extends Component public function submit() { $this->validate(); - // $validation = Validator::make($this->server->toArray(), [ - // 'ip' => [ - // 'ip' - // ], - // ]); - // if ($validation->fails()) { - // foreach ($validation->errors()->getMessages() as $key => $value) { - // $this->addError("server.{$key}", $value[0]); - // } - // return; - // } + $uniqueIPs = Server::all()->pluck('ip')->toArray(); + if (in_array($this->server->ip, $uniqueIPs)) { + $this->emit('error', 'IP address is already in use by another team.'); + return; + } $this->server->settings->wildcard_domain = $this->wildcard_domain; $this->server->settings->cleanup_after_percentage = $this->cleanup_after_percentage; $this->server->settings->save(); diff --git a/app/Http/Livewire/Server/ShowPrivateKey.php b/app/Http/Livewire/Server/ShowPrivateKey.php index cfb1cdd67..1ec98b8b2 100644 --- a/app/Http/Livewire/Server/ShowPrivateKey.php +++ b/app/Http/Livewire/Server/ShowPrivateKey.php @@ -14,6 +14,7 @@ class ShowPrivateKey extends Component public function setPrivateKey($newPrivateKeyId) { try { + refresh_server_connection($this->server->privateKey); $oldPrivateKeyId = $this->server->private_key_id; $this->server->update([ 'private_key_id' => $newPrivateKeyId @@ -26,7 +27,7 @@ class ShowPrivateKey extends Component 'private_key_id' => $oldPrivateKeyId ]); $this->server->refresh(); - refresh_server_connection($this->server->privateKey); + refresh_server_connection($this->server->privateKey); return general_error_handler($e, that: $this); } } diff --git a/app/Http/Livewire/Settings/Configuration.php b/app/Http/Livewire/Settings/Configuration.php index 6801823fe..9bc7c20cb 100644 --- a/app/Http/Livewire/Settings/Configuration.php +++ b/app/Http/Livewire/Settings/Configuration.php @@ -2,7 +2,7 @@ namespace App\Http\Livewire\Settings; -use App\Jobs\ProxyContainerStatusJob; +use App\Jobs\ContainerStatusJob; use App\Models\InstanceSettings as ModelsInstanceSettings; use App\Models\Server; use Livewire\Component; @@ -124,7 +124,7 @@ class Configuration extends Component ]; } $this->save_configuration_to_disk($traefik_dynamic_conf, $file); - dispatch(new ProxyContainerStatusJob($this->server)); + dispatch(new ContainerStatusJob($this->server)); } } diff --git a/app/Http/Livewire/Waitlist/Index.php b/app/Http/Livewire/Waitlist/Index.php index fb040c6d5..d2ce6fe19 100644 --- a/app/Http/Livewire/Waitlist/Index.php +++ b/app/Http/Livewire/Waitlist/Index.php @@ -6,6 +6,7 @@ use App\Jobs\SendConfirmationForWaitlistJob; use App\Models\User; use App\Models\Waitlist; use Livewire\Component; +use Str; class Index extends Component { @@ -46,7 +47,7 @@ class Index extends Component return; } $waitlist = Waitlist::create([ - 'email' => $this->email, + 'email' => Str::lower($this->email), 'type' => 'registration', ]); diff --git a/app/Jobs/ApplicationContainerStatusJob.php b/app/Jobs/ApplicationContainerStatusJob.php index 67e629425..ae41a5c41 100644 --- a/app/Jobs/ApplicationContainerStatusJob.php +++ b/app/Jobs/ApplicationContainerStatusJob.php @@ -4,15 +4,15 @@ namespace App\Jobs; use App\Models\Application; use App\Models\ApplicationPreview; -use App\Notifications\Application\StatusChanged; use Illuminate\Bus\Queueable; +use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldBeUnique; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; -class ApplicationContainerStatusJob implements ShouldQueue, ShouldBeUnique +class ApplicationContainerStatusJob implements ShouldQueue, ShouldBeUnique, ShouldBeEncrypted { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index 81778203d..ea682b03a 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -17,10 +17,10 @@ use App\Notifications\Application\DeploymentSuccess; use App\Traits\ExecuteRemoteCommand; use Exception; use Illuminate\Bus\Queueable; +use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; -use Illuminate\Queue\Middleware\WithoutOverlapping; use Illuminate\Queue\SerializesModels; use Illuminate\Support\Collection; use Illuminate\Support\Str; @@ -28,10 +28,8 @@ use Spatie\Url\Url; use Symfony\Component\Yaml\Yaml; use Throwable; use Visus\Cuid2\Cuid2; -use Yosymfony\Toml\Toml; -use Yosymfony\Toml\TomlArray; -class ApplicationDeploymentJob implements ShouldQueue +class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels, ExecuteRemoteCommand; @@ -49,7 +47,6 @@ class ApplicationDeploymentJob implements ShouldQueue private GithubApp|GitlabApp $source; private StandaloneDocker|SwarmDocker $destination; private Server $server; - private string $private_key_location; private ApplicationPreview|null $preview = null; private string $container_name; @@ -92,7 +89,7 @@ class ApplicationDeploymentJob implements ShouldQueue $this->is_debug_enabled = $this->application->settings->is_debug_enabled; $this->container_name = generateApplicationContainerName($this->application->uuid, $this->pull_request_id); - $this->private_key_location = save_private_key_for_server($this->server); + addPrivateKeyToSshAgent($this->server); $this->saved_outputs = collect(); // Set preview fqdn @@ -122,6 +119,9 @@ class ApplicationDeploymentJob implements ShouldQueue if ($containers->count() > 0) { $this->currently_running_container_name = data_get($containers[0], 'Names'); } + if ($this->pull_request_id !== 0 && $this->pull_request_id !== null) { + $this->currently_running_container_name = $this->container_name; + } $this->application_deployment_queue->update([ 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value, ]); @@ -135,7 +135,9 @@ class ApplicationDeploymentJob implements ShouldQueue $this->deploy(); } } - if ($this->application->fqdn) dispatch(new ProxyContainerStatusJob($this->server)); + if ($this->server->isProxyShouldRun()) { + dispatch(new ContainerStatusJob($this->server)); + } $this->next(ApplicationDeploymentStatus::FINISHED->value); } catch (Exception $e) { ray($e); @@ -270,6 +272,7 @@ class ApplicationDeploymentJob implements ShouldQueue "echo 'Rolling update completed.'" ], ); + $this->application->update(['status' => 'running']); break; } $counter++; @@ -296,7 +299,11 @@ class ApplicationDeploymentJob implements ShouldQueue // $this->generate_build_env_variables(); // $this->add_build_env_variables_to_dockerfile(); $this->build_image(); - $this->rolling_update(); + $this->stop_running_container(); + $this->execute_remote_command( + ["echo -n 'Starting preview deployment.'"], + [$this->execute_in_builder("docker compose --project-directory {$this->workdir} up -d >/dev/null"), "hidden" => true], + ); } private function prepare_builder_image() @@ -576,10 +583,15 @@ class ApplicationDeploymentJob implements ShouldQueue private function set_labels_for_applications() { + + $appId = $this->application->id; + if ($this->pull_request_id !== 0) { + $appId = $appId . '-pr-' . $this->pull_request_id; + } $labels = []; $labels[] = 'coolify.managed=true'; $labels[] = 'coolify.version=' . config('version'); - $labels[] = 'coolify.applicationId=' . $this->application->id; + $labels[] = 'coolify.applicationId=' . $appId; $labels[] = 'coolify.type=application'; $labels[] = 'coolify.name=' . $this->application->name; if ($this->pull_request_id !== 0) { diff --git a/app/Jobs/ApplicationPullRequestUpdateJob.php b/app/Jobs/ApplicationPullRequestUpdateJob.php index 19fbcd7e7..851c04169 100755 --- a/app/Jobs/ApplicationPullRequestUpdateJob.php +++ b/app/Jobs/ApplicationPullRequestUpdateJob.php @@ -6,12 +6,13 @@ use App\Enums\ProcessStatus; use App\Models\Application; use App\Models\ApplicationPreview; use Illuminate\Bus\Queueable; +use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; -class ApplicationPullRequestUpdateJob implements ShouldQueue +class ApplicationPullRequestUpdateJob implements ShouldQueue, ShouldBeEncrypted { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; diff --git a/app/Jobs/CheckResaleLicenseJob.php b/app/Jobs/CheckResaleLicenseJob.php index fe4eed27f..08530aeae 100644 --- a/app/Jobs/CheckResaleLicenseJob.php +++ b/app/Jobs/CheckResaleLicenseJob.php @@ -4,12 +4,13 @@ namespace App\Jobs; use App\Actions\License\CheckResaleLicense; use Illuminate\Bus\Queueable; +use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; -class CheckResaleLicenseJob implements ShouldQueue +class CheckResaleLicenseJob implements ShouldQueue, ShouldBeEncrypted { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; diff --git a/app/Jobs/CleanupInstanceStuffsJob.php b/app/Jobs/CleanupInstanceStuffsJob.php index 3d08203e3..f9d88cd4d 100644 --- a/app/Jobs/CleanupInstanceStuffsJob.php +++ b/app/Jobs/CleanupInstanceStuffsJob.php @@ -4,13 +4,14 @@ namespace App\Jobs; use App\Models\Waitlist; use Illuminate\Bus\Queueable; +use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldBeUnique; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; -class CleanupInstanceStuffsJob implements ShouldQueue, ShouldBeUnique +class CleanupInstanceStuffsJob implements ShouldQueue, ShouldBeUnique, ShouldBeEncrypted { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; diff --git a/app/Jobs/ContainerStatusJob.php b/app/Jobs/ContainerStatusJob.php new file mode 100644 index 000000000..8f59c7a2c --- /dev/null +++ b/app/Jobs/ContainerStatusJob.php @@ -0,0 +1,291 @@ +server->uuid)]; + } + + public function uniqueId(): string + { + return $this->server->uuid; + } + + private function checkServerConnection() { + ray("Checking server connection to {$this->server->ip}"); + $uptime = instant_remote_process(['uptime'], $this->server, false); + if (!is_null($uptime)) { + ray('Server is up'); + return true; + } + } + public function handle(): void + { + try { + ray()->clearAll(); + $serverUptimeCheckNumber = 0; + $serverUptimeCheckNumberMax = 5; + while (true) { + if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) { + $this->server->settings()->update(['is_reachable' => false]); + $this->server->team->notify(new Unreachable($this->server)); + return; + } + $result = $this->checkServerConnection(); + if ($result) { + break; + } + $serverUptimeCheckNumber++; + sleep(5); + } + $containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this->server); + $containers = format_docker_command_output_to_json($containers); + $applications = $this->server->applications(); + $databases = $this->server->databases(); + $previews = $this->server->previews(); + if ($this->server->isProxyShouldRun()) { + $foundProxyContainer = $containers->filter(function ($value, $key) { + return data_get($value, 'Name') === '/coolify-proxy'; + })->first(); + if (!$foundProxyContainer) { + resolve(StartProxy::class)($this->server, false); + $this->server->team->notify(new ContainerRestarted('coolify-proxy', $this->server)); + } + } + $foundApplications = []; + $foundApplicationPreviews = []; + $foundDatabases = []; + foreach ($containers as $container) { + $containerStatus = data_get($container, 'State.Status'); + $labels = data_get($container, 'Config.Labels'); + $labels = Arr::undot(format_docker_labels_to_json($labels)); + $labelId = data_get($labels, 'coolify.applicationId'); + ray($labelId); + if ($labelId) { + if (str_contains($labelId,'-pr-')) { + $previewId = (int) Str::after($labelId, '-pr-'); + $applicationId = (int) Str::before($labelId, '-pr-'); + $preview = ApplicationPreview::where('application_id', $applicationId)->where('pull_request_id',$previewId)->first(); + if ($preview) { + $foundApplicationPreviews[] = $preview->id; + $statusFromDb = $preview->status; + if ($statusFromDb !== $containerStatus) { + $preview->update(['status' => $containerStatus]); + } + } else { + //Notify user that this container should not be there. + } + } else { + $application = $applications->where('id', $labelId)->first(); + if ($application) { + $foundApplications[] = $application->id; + $statusFromDb = $application->status; + if ($statusFromDb !== $containerStatus) { + $application->update(['status' => $containerStatus]); + } + } else { + //Notify user that this container should not be there. + } + } + } else { + $uuid = data_get($labels, 'com.docker.compose.service'); + if ($uuid) { + $database = $databases->where('uuid', $uuid)->first(); + if ($database) { + $foundDatabases[] = $database->id; + $statusFromDb = $database->status; + if ($statusFromDb !== $containerStatus) { + $database->update(['status' => $containerStatus]); + } + } else { + // Notify user that this container should not be there. + } + } + } + + } + $notRunningApplications = $applications->pluck('id')->diff($foundApplications); + foreach($notRunningApplications as $applicationId) { + $application = $applications->where('id', $applicationId)->first(); + if ($application->status === 'exited') { + continue; + } + $application->update(['status' => 'exited']); + + $name = data_get($application, 'name'); + $fqdn = data_get($application, 'fqdn'); + + $containerName = $name ? "$name ($fqdn)" : $fqdn; + + $project = data_get($application, 'environment.project'); + $environment = data_get($application, 'environment'); + + $url = base_url() . '/project/' . $project->uuid . "/" . $environment->name . "/application/" . $application->uuid; + + $this->server->team->notify(new ContainerStopped($containerName, $this->server, $url)); + } + $notRunningApplicationPreviews = $previews->pluck('id')->diff($foundApplicationPreviews); + foreach ($notRunningApplicationPreviews as $previewId) { + $preview = $previews->where('id', $previewId)->first(); + if ($preview->status === 'exited') { + continue; + } + $preview->update(['status' => 'exited']); + + $name = data_get($preview, 'name'); + $fqdn = data_get($preview, 'fqdn'); + + $containerName = $name ? "$name ($fqdn)" : $fqdn; + + $project = data_get($preview, 'application.environment.project'); + $environment = data_get($preview, 'application.environment'); + + $url = base_url() . '/project/' . $project->uuid . "/" . $environment->name . "/application/" . $preview->application->uuid; + $this->server->team->notify(new ContainerStopped($containerName, $this->server, $url)); + } + $notRunningDatabases = $databases->pluck('id')->diff($foundDatabases); + foreach($notRunningDatabases as $database) { + $database = $databases->where('id', $database)->first(); + if ($database->status === 'exited') { + continue; + } + $database->update(['status' => 'exited']); + + $name = data_get($database, 'name'); + $fqdn = data_get($database, 'fqdn'); + + $containerName = $name; + + $project = data_get($database, 'environment.project'); + $environment = data_get($database, 'environment'); + + $url = base_url() . '/project/' . $project->uuid . "/" . $environment->name . "/database/" . $database->uuid; + $this->server->team->notify(new ContainerStopped($containerName, $this->server, $url)); + } + + + + + + + + + + + + + return; + foreach ($applications as $application) { + $uuid = data_get($application, 'uuid'); + $id = data_get($application, 'id'); + $foundContainer = $containers->filter(function ($value, $key) use ($id, $uuid) { + $labels = data_get($value, 'Config.Labels'); + $labels = Arr::undot(format_docker_labels_to_json($labels)); + $labelId = data_get($labels, 'coolify.applicationId'); + if ($labelId == $id) { + return $value; + } + $isPR = Str::startsWith(data_get($value, 'Name'), "/$uuid"); + $isPR = Str::contains(data_get($value, 'Name'), "-pr-"); + if ($isPR) { + ray('is pr'); + return false; + } + return $value; + })->first(); + ray($foundContainer); + if ($foundContainer) { + $containerStatus = data_get($foundContainer, 'State.Status'); + $databaseStatus = data_get($application, 'status'); + if ($containerStatus !== $databaseStatus) { + $application->update(['status' => $containerStatus]); + } + } else { + $databaseStatus = data_get($application, 'status'); + if ($databaseStatus !== 'exited') { + $application->update(['status' => 'exited']); + $name = data_get($application, 'name'); + $fqdn = data_get($application, 'fqdn'); + $containerName = $name ? "$name ($fqdn)" : $fqdn; + $project = data_get($application, 'environment.project'); + $environment = data_get($application, 'environment'); + $url = base_url() . '/project/' . $project->uuid . "/" . $environment->name . "/application/" . $application->uuid; + $this->server->team->notify(new ContainerStopped($containerName, $this->server, $url)); + } + } + $previews = $application->previews; + foreach ($previews as $preview) { + $foundContainer = $containers->filter(function ($value, $key) use ($id, $uuid, $preview) { + $labels = data_get($value, 'Config.Labels'); + $labels = Arr::undot(format_docker_labels_to_json($labels)); + $labelId = data_get($labels, 'coolify.applicationId'); + if ($labelId == "$id-pr-{$preview->id}") { + return $value; + } + return Str::startsWith(data_get($value, 'Name'), "/$uuid-pr-{$preview->id}"); + })->first(); + } + + } + foreach ($databases as $database) { + $uuid = data_get($database, 'uuid'); + $foundContainer = $containers->filter(function ($value, $key) use ($uuid) { + return Str::startsWith(data_get($value, 'Name'), "/$uuid"); + })->first(); + + if ($foundContainer) { + $containerStatus = data_get($foundContainer, 'State.Status'); + $databaseStatus = data_get($database, 'status'); + if ($containerStatus !== $databaseStatus) { + $database->update(['status' => $containerStatus]); + } + } else { + $databaseStatus = data_get($database, 'status'); + if ($databaseStatus !== 'exited') { + $database->update(['status' => 'exited']); + $name = data_get($database, 'name'); + $containerName = $name; + $project = data_get($database, 'environment.project'); + $environment = data_get($database, 'environment'); + $url = base_url() . '/project/' . $project->uuid . "/" . $environment->name . "/database/" . $database->uuid; + $this->server->team->notify(new ContainerStopped($containerName, $this->server, $url)); + } + } + } + // TODO Monitor other containers not managed by Coolify + } catch (\Throwable $e) { + send_internal_notification('ContainerStatusJob failed with: ' . $e->getMessage()); + ray($e->getMessage()); + throw $e; + } + } +} diff --git a/app/Jobs/CoolifyTask.php b/app/Jobs/CoolifyTask.php index 176841b20..3ea32c27a 100755 --- a/app/Jobs/CoolifyTask.php +++ b/app/Jobs/CoolifyTask.php @@ -4,13 +4,14 @@ namespace App\Jobs; use App\Actions\CoolifyTask\RunRemoteProcess; use Illuminate\Bus\Queueable; +use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; use Spatie\Activitylog\Models\Activity; -class CoolifyTask implements ShouldQueue +class CoolifyTask implements ShouldQueue, ShouldBeEncrypted { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; diff --git a/app/Jobs/DatabaseBackupJob.php b/app/Jobs/DatabaseBackupJob.php index 778272705..578a5ec43 100644 --- a/app/Jobs/DatabaseBackupJob.php +++ b/app/Jobs/DatabaseBackupJob.php @@ -12,6 +12,7 @@ use App\Notifications\Database\BackupFailed; use App\Notifications\Database\BackupSuccess; use Carbon\Carbon; use Illuminate\Bus\Queueable; +use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; @@ -20,7 +21,7 @@ use Illuminate\Queue\SerializesModels; use Throwable; use Illuminate\Support\Str; -class DatabaseBackupJob implements ShouldQueue +class DatabaseBackupJob implements ShouldQueue, ShouldBeEncrypted { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; diff --git a/app/Jobs/DatabaseContainerStatusJob.php b/app/Jobs/DatabaseContainerStatusJob.php index d7b4d0085..f1d0de301 100644 --- a/app/Jobs/DatabaseContainerStatusJob.php +++ b/app/Jobs/DatabaseContainerStatusJob.php @@ -6,13 +6,14 @@ use App\Models\ApplicationPreview; use App\Models\StandalonePostgresql; use App\Notifications\Application\StatusChanged; use Illuminate\Bus\Queueable; +use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldBeUnique; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; -class DatabaseContainerStatusJob implements ShouldQueue, ShouldBeUnique +class DatabaseContainerStatusJob implements ShouldQueue, ShouldBeUnique, ShouldBeEncrypted { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; diff --git a/app/Jobs/DockerCleanupJob.php b/app/Jobs/DockerCleanupJob.php index b1fccbe4b..80c76e067 100644 --- a/app/Jobs/DockerCleanupJob.php +++ b/app/Jobs/DockerCleanupJob.php @@ -5,13 +5,14 @@ namespace App\Jobs; use App\Models\ApplicationDeploymentQueue; use App\Models\Server; use Illuminate\Bus\Queueable; +use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\Middleware\WithoutOverlapping; use Illuminate\Queue\SerializesModels; -class DockerCleanupJob implements ShouldQueue +class DockerCleanupJob implements ShouldQueue, ShouldBeEncrypted { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; diff --git a/app/Jobs/InstanceAutoUpdateJob.php b/app/Jobs/InstanceAutoUpdateJob.php index 957b8fe23..814ae9cc9 100644 --- a/app/Jobs/InstanceAutoUpdateJob.php +++ b/app/Jobs/InstanceAutoUpdateJob.php @@ -4,13 +4,14 @@ namespace App\Jobs; use App\Actions\Server\UpdateCoolify; use Illuminate\Bus\Queueable; +use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldBeUnique; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; -class InstanceAutoUpdateJob implements ShouldQueue, ShouldBeUnique +class InstanceAutoUpdateJob implements ShouldQueue, ShouldBeUnique, ShouldBeEncrypted { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; diff --git a/app/Jobs/ProxyContainerStatusJob.php b/app/Jobs/ProxyContainerStatusJob.php index 1e7eac6a3..6cfd2692f 100644 --- a/app/Jobs/ProxyContainerStatusJob.php +++ b/app/Jobs/ProxyContainerStatusJob.php @@ -7,6 +7,7 @@ use App\Enums\ProxyStatus; use App\Enums\ProxyTypes; use App\Models\Server; use Illuminate\Bus\Queueable; +use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldBeUnique; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; @@ -14,7 +15,7 @@ use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\Middleware\WithoutOverlapping; use Illuminate\Queue\SerializesModels; -class ProxyContainerStatusJob implements ShouldQueue, ShouldBeUnique +class ProxyContainerStatusJob implements ShouldQueue, ShouldBeUnique, ShouldBeEncrypted { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; diff --git a/app/Jobs/SendConfirmationForWaitlistJob.php b/app/Jobs/SendConfirmationForWaitlistJob.php index 470b1bafe..bee15975c 100755 --- a/app/Jobs/SendConfirmationForWaitlistJob.php +++ b/app/Jobs/SendConfirmationForWaitlistJob.php @@ -5,6 +5,7 @@ namespace App\Jobs; use App\Models\InstanceSettings; use App\Models\Waitlist; use Illuminate\Bus\Queueable; +use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Mail\Message; @@ -13,7 +14,7 @@ use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; use Illuminate\Support\Facades\Mail; -class SendConfirmationForWaitlistJob implements ShouldQueue +class SendConfirmationForWaitlistJob implements ShouldQueue, ShouldBeEncrypted { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; diff --git a/app/Jobs/SendMessageToDiscordJob.php b/app/Jobs/SendMessageToDiscordJob.php index 66e3c1c1a..b9255baaa 100644 --- a/app/Jobs/SendMessageToDiscordJob.php +++ b/app/Jobs/SendMessageToDiscordJob.php @@ -3,13 +3,14 @@ namespace App\Jobs; use Illuminate\Bus\Queueable; +use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; use Illuminate\Support\Facades\Http; -class SendMessageToDiscordJob implements ShouldQueue +class SendMessageToDiscordJob implements ShouldQueue, ShouldBeEncrypted { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; diff --git a/app/Jobs/SendMessageToTelegramJob.php b/app/Jobs/SendMessageToTelegramJob.php index 1fa1347c4..37f1cf381 100644 --- a/app/Jobs/SendMessageToTelegramJob.php +++ b/app/Jobs/SendMessageToTelegramJob.php @@ -3,6 +3,7 @@ namespace App\Jobs; use Illuminate\Bus\Queueable; +use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; @@ -10,7 +11,7 @@ use Illuminate\Queue\SerializesModels; use Illuminate\Support\Facades\Http; use Str; -class SendMessageToTelegramJob implements ShouldQueue +class SendMessageToTelegramJob implements ShouldQueue, ShouldBeEncrypted { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; diff --git a/app/Jobs/ServerDetailsCheckJob.php b/app/Jobs/ServerDetailsCheckJob.php deleted file mode 100644 index 606931c42..000000000 --- a/app/Jobs/ServerDetailsCheckJob.php +++ /dev/null @@ -1,79 +0,0 @@ -server->uuid)]; - } - - public function uniqueId(): string - { - return $this->server->uuid; - } - - public function handle(): void - { - try { - ray()->clearAll(); - $containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this->server); - $containers = format_docker_command_output_to_json($containers); - $applications = $this->server->applications(); - // ray($applications); - // ray(format_docker_command_output_to_json($containers)); - foreach ($applications as $application) { - $uuid = data_get($application, 'uuid'); - $foundContainer = $containers->filter(function ($value, $key) use ($uuid) { - $image = data_get($value, 'Config.Image'); - return Str::startsWith($image, $uuid); - })->first(); - - if ($foundContainer) { - $containerStatus = data_get($foundContainer, 'State.Status'); - $databaseStatus = data_get($application, 'status'); - ray($containerStatus, $databaseStatus); - if ($containerStatus !== $databaseStatus) { - // $application->update(['status' => $containerStatus]); - } - } - } - // foreach ($containers as $container) { - // $labels = format_docker_labels_to_json(data_get($container,'Config.Labels')); - // $foundLabel = $labels->filter(fn ($value, $key) => Str::startsWith($key, 'coolify.applicationId')); - // if ($foundLabel->count() > 0) { - // $appFound = $applications->where('id', $foundLabel['coolify.applicationId'])->first(); - // if ($appFound) { - // $containerStatus = data_get($container, 'State.Status'); - // $databaseStatus = data_get($appFound, 'status'); - // ray($containerStatus, $databaseStatus); - // } - // } - // } - } catch (\Throwable $e) { - // send_internal_notification('ServerDetailsCheckJob failed with: ' . $e->getMessage()); - ray($e->getMessage()); - throw $e; - } - } -} diff --git a/app/Jobs/SubscriptionInvoiceFailedJob.php b/app/Jobs/SubscriptionInvoiceFailedJob.php index 5e8654673..9b8534060 100755 --- a/app/Jobs/SubscriptionInvoiceFailedJob.php +++ b/app/Jobs/SubscriptionInvoiceFailedJob.php @@ -4,13 +4,14 @@ namespace App\Jobs; use App\Models\Team; use Illuminate\Bus\Queueable; +use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Notifications\Messages\MailMessage; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; -class SubscriptionInvoiceFailedJob implements ShouldQueue +class SubscriptionInvoiceFailedJob implements ShouldQueue, ShouldBeEncrypted { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; diff --git a/app/Jobs/SubscriptionTrialEndedJob.php b/app/Jobs/SubscriptionTrialEndedJob.php index 39acd19a2..3f4ef187e 100755 --- a/app/Jobs/SubscriptionTrialEndedJob.php +++ b/app/Jobs/SubscriptionTrialEndedJob.php @@ -4,13 +4,14 @@ namespace App\Jobs; use App\Models\Team; use Illuminate\Bus\Queueable; +use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Notifications\Messages\MailMessage; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; -class SubscriptionTrialEndedJob implements ShouldQueue +class SubscriptionTrialEndedJob implements ShouldQueue, ShouldBeEncrypted { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; diff --git a/app/Jobs/SubscriptionTrialEndsSoonJob.php b/app/Jobs/SubscriptionTrialEndsSoonJob.php index 84bd8ee66..5e8b35aa8 100755 --- a/app/Jobs/SubscriptionTrialEndsSoonJob.php +++ b/app/Jobs/SubscriptionTrialEndsSoonJob.php @@ -4,13 +4,14 @@ namespace App\Jobs; use App\Models\Team; use Illuminate\Bus\Queueable; +use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Notifications\Messages\MailMessage; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; -class SubscriptionTrialEndsSoonJob implements ShouldQueue +class SubscriptionTrialEndsSoonJob implements ShouldQueue, ShouldBeEncrypted { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; diff --git a/app/Models/Server.php b/app/Models/Server.php index f45cf3b2d..1fd19e119 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -105,6 +105,14 @@ class Server extends BaseModel })->flatten(); } + public function previews() { + return $this->destinations()->map(function ($standaloneDocker) { + return $standaloneDocker->applications->map(function ($application) { + return $application->previews; + })->flatten(); + })->flatten(); + } + public function destinations() { $standalone_docker = $this->hasMany(StandaloneDocker::class)->get(); diff --git a/app/Notifications/Application/DeploymentSuccess.php b/app/Notifications/Application/DeploymentSuccess.php index 0705e11ae..ea789b69a 100644 --- a/app/Notifications/Application/DeploymentSuccess.php +++ b/app/Notifications/Application/DeploymentSuccess.php @@ -53,7 +53,7 @@ class DeploymentSuccess extends Notification implements ShouldQueue $pull_request_id = data_get($this->preview, 'pull_request_id', 0); $fqdn = $this->fqdn; if ($pull_request_id === 0) { - $mail->subject("✅New version is deployed of {$this->application_name}"); + $mail->subject("✅ New version is deployed of {$this->application_name}"); } else { $fqdn = $this->preview->fqdn; $mail->subject("✅ Pull request #{$pull_request_id} of {$this->application_name} deployed successfully"); diff --git a/app/Notifications/Container/ContainerRestarted.php b/app/Notifications/Container/ContainerRestarted.php new file mode 100644 index 000000000..480fe2cb0 --- /dev/null +++ b/app/Notifications/Container/ContainerRestarted.php @@ -0,0 +1,62 @@ +subject("✅ Container ({$this->name}) has been restarted automatically on {$this->server->name}"); + $mail->view('emails.container-restarted', [ + 'containerName' => $this->name, + 'serverName' => $this->server->name, + 'url' => $this->url , + ]); + return $mail; + } + + public function toDiscord(): string + { + $message = "✅ Container ({$this->name}) has been restarted automatically on {$this->server->name}"; + return $message; + } + public function toTelegram(): array + { + $message = "✅ Container ({$this->name}) has been restarted automatically on {$this->server->name}"; + $payload = [ + "message" => $message, + ]; + if ($this->url) { + $payload['buttons'] = [ + [ + [ + "text" => "Check Proxy in Coolify", + "url" => $this->url + ] + ] + ]; + }; + return $payload; + } +} diff --git a/app/Notifications/Container/ContainerStopped.php b/app/Notifications/Container/ContainerStopped.php new file mode 100644 index 000000000..390c9552b --- /dev/null +++ b/app/Notifications/Container/ContainerStopped.php @@ -0,0 +1,61 @@ +subject("⛔ Container {$this->name} has been stopped on {$this->server->name}"); + $mail->view('emails.container-stopped', [ + 'containerName' => $this->name, + 'serverName' => $this->server->name, + 'url' => $this->url, + ]); + return $mail; + } + + public function toDiscord(): string + { + $message = "⛔ Container {$this->name} has been stopped on {$this->server->name}"; + return $message; + } + public function toTelegram(): array + { + $message = "⛔ Container ($this->name} has been stopped on {$this->server->name}"; + $payload = [ + "message" => $message, + ]; + if ($this->url) { + $payload['buttons'] = [ + [ + [ + "text" => "Open Application in Coolify", + "url" => $this->url + ] + ] + ]; + } + return $payload; + } +} diff --git a/app/Notifications/Server/NotReachable.php b/app/Notifications/Server/NotReachable.php deleted file mode 100644 index 672636c57..000000000 --- a/app/Notifications/Server/NotReachable.php +++ /dev/null @@ -1,53 +0,0 @@ -fqdn; - $mail->subject("⛔ Server '{$this->server->name}' is unreachable"); - // $mail->view('emails.application-status-changes', [ - // 'name' => $this->application_name, - // 'fqdn' => $fqdn, - // 'application_url' => $this->application_url, - // ]); - return $mail; - } - - public function toDiscord(): string - { - $message = '⛔ Server \'' . $this->server->name . '\' is unreachable (could be a temporary issue). If you receive this more than twice in a row, please check your server.'; - return $message; - } - public function toTelegram(): array - { - return [ - "message" => '⛔ Server \'' . $this->server->name . '\' is unreachable (could be a temporary issue). If you receive this more than twice in a row, please check your server.' - ]; - } -} diff --git a/app/Notifications/Server/Unreachable.php b/app/Notifications/Server/Unreachable.php new file mode 100644 index 000000000..ec9c11d11 --- /dev/null +++ b/app/Notifications/Server/Unreachable.php @@ -0,0 +1,47 @@ +subject("⛔ Server ({$this->server->name}) is unreachable after trying to connect to it 5 times"); + $mail->view('emails.server-lost-connection', [ + 'name' => $this->server->name, + ]); + return $mail; + } + + public function toDiscord(): string + { + $message = "⛔ Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: You have to validate your server again after you fix the issue."; + return $message; + } + public function toTelegram(): array + { + return [ + "message" => "⛔ Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: You have to validate your server again after you fix the issue." + ]; + } +} diff --git a/app/Traits/ExecuteRemoteCommand.php b/app/Traits/ExecuteRemoteCommand.php index fd20c3764..e03c414b8 100644 --- a/app/Traits/ExecuteRemoteCommand.php +++ b/app/Traits/ExecuteRemoteCommand.php @@ -28,9 +28,8 @@ trait ExecuteRemoteCommand $ip = data_get($this->server, 'ip'); $user = data_get($this->server, 'user'); $port = data_get($this->server, 'port'); - $private_key_location = get_private_key_for_server($this->server); - $commandsText->each(function ($single_command) use ($private_key_location, $ip, $user, $port) { + $commandsText->each(function ($single_command) use ($ip, $user, $port) { $command = data_get($single_command, 'command') ?? $single_command[0] ?? null; if ($command === null) { throw new \RuntimeException('Command is not set'); @@ -39,8 +38,8 @@ trait ExecuteRemoteCommand $ignore_errors = data_get($single_command, 'ignore_errors', false); $this->save = data_get($single_command, 'save'); - $remote_command = generate_ssh_command($private_key_location, $ip, $user, $port, $command); - $process = Process::timeout(3600)->idleTimeout(3600)->start($remote_command, function (string $type, string $output) use ($command, $hidden) { + $remote_command = generateSshCommand( $ip, $user, $port, $command); + $process = processWithEnv()->timeout(3600)->idleTimeout(3600)->start($remote_command, function (string $type, string $output) use ($command, $hidden) { $output = Str::of($output)->trim(); $new_log_entry = [ 'command' => $command, diff --git a/bootstrap/helpers/docker.php b/bootstrap/helpers/docker.php index e3629e71c..03483afdc 100644 --- a/bootstrap/helpers/docker.php +++ b/bootstrap/helpers/docker.php @@ -91,7 +91,7 @@ function generateApplicationContainerName(string $uuid, int $pull_request_id = 0 { $now = now()->format('Hisu'); if ($pull_request_id !== 0 && $pull_request_id !== null) { - return $uuid . '-pr-' . $pull_request_id . '-' . $now; + return $uuid . '-pr-' . $pull_request_id; } else { return $uuid . '-' . $now; } diff --git a/bootstrap/helpers/remoteProcess.php b/bootstrap/helpers/remoteProcess.php index 40ff2f958..84f2dc9d3 100644 --- a/bootstrap/helpers/remoteProcess.php +++ b/bootstrap/helpers/remoteProcess.php @@ -33,12 +33,9 @@ function remote_process( } } - $private_key_location = save_private_key_for_server($server); - return resolve(PrepareCoolifyTask::class, [ 'remoteProcessArgs' => new CoolifyTaskArgs( server_ip: $server->ip, - private_key_location: $private_key_location, command: <<name} does not have a private key"); } - $temp_file = "id.root@{$server->ip}"; - Storage::disk('ssh-keys')->put($temp_file, $server->privateKey->private_key); - Storage::disk('ssh-mux')->makeDirectory('.'); - return '/var/www/html/storage/app/ssh/keys/' . $temp_file; + processWithEnv()->run("echo '{$server->privateKey->private_key}' | ssh-add -d -"); +} +function addPrivateKeyToSshAgent(Server $server) +{ + if (data_get($server, 'privateKey.private_key') === null) { + throw new \Exception("Server {$server->name} does not have a private key"); + } + // ray('adding key', $server->privateKey->private_key); + processWithEnv()->run("echo '{$server->privateKey->private_key}' | ssh-add -q -"); } -function generate_ssh_command(string $private_key_location, string $server_ip, string $user, string $port, string $command, bool $isMux = true) +function generateSshCommand(string $server_ip, string $user, string $port, string $command, bool $isMux = true) { + $server = Server::where('ip', $server_ip)->first(); + if (!$server) { + throw new \Exception("Server with ip {$server_ip} not found"); + } + addPrivateKeyToSshAgent($server); + $timeout = config('constants.ssh.command_timeout'); + $connectionTimeout = config('constants.ssh.connection_timeout'); + $serverInterval = config('constants.ssh.server_interval'); + $delimiter = 'EOF-COOLIFY-SSH'; - $ssh_command = "ssh "; + $ssh_command = "timeout $timeout ssh "; if ($isMux && config('coolify.mux_enabled')) { $ssh_command .= '-o ControlMaster=auto -o ControlPersist=1m -o ControlPath=/var/www/html/storage/app/ssh/mux/%h_%p_%r '; } $command = "PATH=\$PATH:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/host/usr/local/sbin:/host/usr/local/bin:/host/usr/sbin:/host/usr/bin:/host/sbin:/host/bin && $command"; - $ssh_command .= "-i {$private_key_location} " - . '-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ' + $ssh_command .= '-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ' . '-o PasswordAuthentication=no ' - . '-o ConnectTimeout=3600 ' - . '-o ServerAliveInterval=20 ' + . "-o ConnectTimeout=$connectionTimeout " + . "-o ServerAliveInterval=$serverInterval " . '-o RequestTTY=no ' . '-o LogLevel=ERROR ' . "-p {$port} " @@ -90,11 +94,16 @@ function generate_ssh_command(string $private_key_location, string $server_ip, s . " 'bash -se' << \\$delimiter" . PHP_EOL . $command . PHP_EOL . $delimiter; - + // ray($ssh_command); return $ssh_command; } -function instantCommand(string $command, $throwError = true) { - $process = Process::run($command); +function processWithEnv() +{ + return Process::env(['SSH_AUTH_SOCK' => config('coolify.ssh_auth_sock')]); +} +function instantCommand(string $command, $throwError = true) +{ + $process = processWithEnv()->run($command); $output = trim($process->output()); $exitCode = $process->exitCode(); if ($exitCode !== 0) { @@ -108,9 +117,8 @@ function instantCommand(string $command, $throwError = true) { function instant_remote_process(array $command, Server $server, $throwError = true, $repeat = 1) { $command_string = implode("\n", $command); - $private_key_location = save_private_key_for_server($server); - $ssh_command = generate_ssh_command($private_key_location, $server->ip, $server->user, $server->port, $command_string); - $process = Process::run($ssh_command); + $ssh_command = generateSshCommand($server->ip, $server->user, $server->port, $command_string); + $process = processWithEnv()->run($ssh_command); $output = trim($process->output()); $exitCode = $process->exitCode(); if ($exitCode !== 0) { @@ -168,6 +176,7 @@ function refresh_server_connection(PrivateKey $private_key) // currentTeam()->privateKeys = PrivateKey::where('team_id', currentTeam()->id)->get(); // } } + removePrivateKeyFromSshAgent($server); } function validateServer(Server $server) @@ -208,7 +217,7 @@ function validateServer(Server $server) $server->settings->is_usable = false; throw $e; } finally { - if(data_get($server,'settings')) $server->settings->save(); + if (data_get($server, 'settings')) $server->settings->save(); } } diff --git a/config/constants.php b/config/constants.php index 4cb112855..4630adcda 100644 --- a/config/constants.php +++ b/config/constants.php @@ -1,5 +1,10 @@ [ + 'connection_timeout' => 10, + 'server_interval' => 20, + 'command_timeout' => 7200, + ], 'waitlist' => [ 'expiration' => 10, ], diff --git a/config/coolify.php b/config/coolify.php index cd16d6b6f..68960015b 100644 --- a/config/coolify.php +++ b/config/coolify.php @@ -8,4 +8,5 @@ return [ 'dev_webhook' => env('SERVEO_URL'), 'base_config_path' => env('BASE_CONFIG_PATH', '/data/coolify'), 'helper_image' => env('HELPER_IMAGE', 'ghcr.io/coollabsio/coolify-helper:latest'), + 'ssh_auth_sock' => env('SSH_AUTH_SOCK', '/tmp/coolify-ssh-agent.sock'), ]; diff --git a/config/sentry.php b/config/sentry.php index 3e42e7029..b0db797f1 100644 --- a/config/sentry.php +++ b/config/sentry.php @@ -7,7 +7,7 @@ return [ // The release version of your application // Example with dynamic git hash: trim(exec('git --git-dir ' . base_path('.git') . ' log --pretty="%h" -n1 HEAD')) - 'release' => '4.0.0-beta.35', + 'release' => '4.0.0-beta.36', 'server_name' => env('APP_ID', 'coolify'), // When left empty or `null` the Laravel environment will be used 'environment' => config('app.env'), diff --git a/config/version.php b/config/version.php index c31bdf903..c3cb2484c 100644 --- a/config/version.php +++ b/config/version.php @@ -1,3 +1,3 @@ >/etc/bash.bashrc RUN echo "alias cda='composer dump-autoload'" >>/etc/bash.bashrc RUN echo "alias run='./scripts/run'" >>/etc/bash.bashrc -# COPY --chmod=755 docker/dev-ssu/etc/s6-overlay/ /etc/s6-overlay/ +COPY --chmod=755 docker/dev-ssu/etc/s6-overlay/ /etc/s6-overlay/ diff --git a/docker/dev-ssu/etc/s6-overlay/s6-rc.d/horizon/run b/docker/dev-ssu/etc/s6-overlay/s6-rc.d/horizon/run new file mode 100644 index 000000000..cec4f1dda --- /dev/null +++ b/docker/dev-ssu/etc/s6-overlay/s6-rc.d/horizon/run @@ -0,0 +1,5 @@ +#!/command/execlineb -P +foreground { + s6-sleep 5 + su - webuser -c "php /var/www/html/artisan horizon" +} diff --git a/docker/dev-ssu/etc/s6-overlay/s6-rc.d/queue-worker/type b/docker/dev-ssu/etc/s6-overlay/s6-rc.d/horizon/type similarity index 100% rename from docker/dev-ssu/etc/s6-overlay/s6-rc.d/queue-worker/type rename to docker/dev-ssu/etc/s6-overlay/s6-rc.d/horizon/type diff --git a/docker/dev-ssu/etc/s6-overlay/s6-rc.d/queue-worker/run b/docker/dev-ssu/etc/s6-overlay/s6-rc.d/queue-worker/run deleted file mode 100644 index 0029f9615..000000000 --- a/docker/dev-ssu/etc/s6-overlay/s6-rc.d/queue-worker/run +++ /dev/null @@ -1,2 +0,0 @@ -#!/command/execlineb -P -su - webuser -c "php /var/www/html/artisan queue:listen" diff --git a/docker/dev-ssu/etc/s6-overlay/s6-rc.d/scheduler-worker/run b/docker/dev-ssu/etc/s6-overlay/s6-rc.d/scheduler-worker/run index 8021572af..0f205c897 100644 --- a/docker/dev-ssu/etc/s6-overlay/s6-rc.d/scheduler-worker/run +++ b/docker/dev-ssu/etc/s6-overlay/s6-rc.d/scheduler-worker/run @@ -1,2 +1,5 @@ #!/command/execlineb -P -su - webuser -c "php /var/www/html/artisan schedule:work" +foreground { + s6-sleep 5 + su - webuser -c "php /var/www/html/artisan schedule:work" +} diff --git a/docker/dev-ssu/etc/s6-overlay/s6-rc.d/ssh-agent/type b/docker/dev-ssu/etc/s6-overlay/s6-rc.d/ssh-agent/type new file mode 100644 index 000000000..bdd22a185 --- /dev/null +++ b/docker/dev-ssu/etc/s6-overlay/s6-rc.d/ssh-agent/type @@ -0,0 +1 @@ +oneshot diff --git a/docker/dev-ssu/etc/s6-overlay/s6-rc.d/ssh-agent/up b/docker/dev-ssu/etc/s6-overlay/s6-rc.d/ssh-agent/up new file mode 100644 index 000000000..0490fb31f --- /dev/null +++ b/docker/dev-ssu/etc/s6-overlay/s6-rc.d/ssh-agent/up @@ -0,0 +1,5 @@ +#!/usr/bin/execlineb -P +foreground { + s6-sleep 5 + su - webuser -c "ssh-agent -a /tmp/coolify-ssh-agent.sock" +} diff --git a/docker/dev-ssu/etc/s6-overlay/s6-rc.d/user/contents.d/queue-worker b/docker/dev-ssu/etc/s6-overlay/s6-rc.d/user/contents.d/horizon similarity index 100% rename from docker/dev-ssu/etc/s6-overlay/s6-rc.d/user/contents.d/queue-worker rename to docker/dev-ssu/etc/s6-overlay/s6-rc.d/user/contents.d/horizon diff --git a/docker/dev-ssu/etc/s6-overlay/s6-rc.d/user/contents.d/scheduler-worker b/docker/dev-ssu/etc/s6-overlay/s6-rc.d/user/contents.d/ssh-agent similarity index 100% rename from docker/dev-ssu/etc/s6-overlay/s6-rc.d/user/contents.d/scheduler-worker rename to docker/dev-ssu/etc/s6-overlay/s6-rc.d/user/contents.d/ssh-agent diff --git a/docker/prod-ssu/etc/s6-overlay/s6-rc.d/ssh-agent/type b/docker/prod-ssu/etc/s6-overlay/s6-rc.d/ssh-agent/type new file mode 100644 index 000000000..bdd22a185 --- /dev/null +++ b/docker/prod-ssu/etc/s6-overlay/s6-rc.d/ssh-agent/type @@ -0,0 +1 @@ +oneshot diff --git a/docker/prod-ssu/etc/s6-overlay/s6-rc.d/ssh-agent/up b/docker/prod-ssu/etc/s6-overlay/s6-rc.d/ssh-agent/up new file mode 100644 index 000000000..0490fb31f --- /dev/null +++ b/docker/prod-ssu/etc/s6-overlay/s6-rc.d/ssh-agent/up @@ -0,0 +1,5 @@ +#!/usr/bin/execlineb -P +foreground { + s6-sleep 5 + su - webuser -c "ssh-agent -a /tmp/coolify-ssh-agent.sock" +} diff --git a/docker/prod-ssu/etc/s6-overlay/s6-rc.d/user/contents.d/ssh-agent b/docker/prod-ssu/etc/s6-overlay/s6-rc.d/user/contents.d/ssh-agent new file mode 100644 index 000000000..e69de29bb diff --git a/resources/views/auth/forgot-password.blade.php b/resources/views/auth/forgot-password.blade.php index 3f815646b..04a9a5695 100644 --- a/resources/views/auth/forgot-password.blade.php +++ b/resources/views/auth/forgot-password.blade.php @@ -15,7 +15,7 @@ @if (is_transactional_emails_active())
@csrf - {{ __('auth.forgot_password_send_email') }} diff --git a/resources/views/components/boarding-step.blade.php b/resources/views/components/boarding-step.blade.php index e9341cbd9..9314727bd 100644 --- a/resources/views/components/boarding-step.blade.php +++ b/resources/views/components/boarding-step.blade.php @@ -14,12 +14,12 @@ @endif - @if($explanation) + @isset($explanation)

Explanation

{{$explanation}}
- @endif + @endisset diff --git a/resources/views/emails/container-restarted.blade.php b/resources/views/emails/container-restarted.blade.php new file mode 100644 index 000000000..fef7b02cb --- /dev/null +++ b/resources/views/emails/container-restarted.blade.php @@ -0,0 +1,15 @@ + + +Container ({{ $containerName }}) has been restarted automatically on {{$serverName}}, because it was stopped unexpected. + +@if ($containerName === 'coolify-proxy') +Coolify Proxy should run on your server as you have FQDNs set up in one of your resources. + +Note: The proxy should not stop unexpectedly, so please check what is going on your server. + + + +If you don't want to use Coolify Proxy, please remove FQDN from your resources or set Proxy type to Custom(None). +@endif + + diff --git a/resources/views/emails/container-stopped.blade.php b/resources/views/emails/container-stopped.blade.php new file mode 100644 index 000000000..77e8a2597 --- /dev/null +++ b/resources/views/emails/container-stopped.blade.php @@ -0,0 +1,9 @@ + + +Container {{ $containerName }} has been stopped unexpected on {{$serverName}}. + +@if ($url) +Please check what is going on [here]({{ $url }}). +@endif + + diff --git a/resources/views/emails/server-lost-connection.blade.php b/resources/views/emails/server-lost-connection.blade.php index 67aa0816a..83b5e2db5 100644 --- a/resources/views/emails/server-lost-connection.blade.php +++ b/resources/views/emails/server-lost-connection.blade.php @@ -1,5 +1,10 @@ -Coolify Cloud cannot connect to your server ({{$name}}). Please check your server and make sure it is running. + +Coolify cannot connect to your server ({{$name}}). Please check your server and make sure it is running. + +All automations & integrations are turned off! + +IMPORTANT: You have to validate your server again after you fix the issue. If you have any questions, please contact us. diff --git a/resources/views/livewire/boarding/index.blade.php b/resources/views/livewire/boarding/index.blade.php index bee5f04b7..8c1e7125c 100644 --- a/resources/views/livewire/boarding/index.blade.php +++ b/resources/views/livewire/boarding/index.blade.php @@ -5,12 +5,34 @@

Welcome to Coolify

Let me help you to set the basics.

- Get Started + Get Started
@endif
+ @if ($currentState === 'explanation') + + + Coolify is an all-in-one application to automate tasks on your servers, deploy application with Git + integrations, deploy databases and services, monitor these resources with notifications and alerts + without vendor lock-in + and much much more. +

+ + + + +

You do not to manage your servers too much. Coolify do it for you.

+

All configurations are stored on your server, so everything works without Coolify (except integrations and automations).

+

You will get notified on your favourite platform (Discord, Telegram, Email, etc.) when something goes wrong, or an action needed from your side.

+ + + Next + + + + @endif @if ($currentState === 'select-server-type') @@ -18,9 +40,11 @@ or on a ? - Localhost + Localhost - Remote Server + Remote Server @@ -42,9 +66,11 @@ Do you have your own SSH Private Key? - Yes + Yes - No (create one for me) + No (create one for me) @if (count($privateKeys) > 0)
@@ -115,9 +141,10 @@ @if ($privateKeyType === 'create') - - ACTION REQUIRED: Copy the 'Public Key' to your server's ~/.ssh/authorized_keys - file. + + ACTION REQUIRED: Copy the 'Public Key' to your server's + ~/.ssh/authorized_keys + file. @endif Save @@ -182,7 +209,8 @@ Could not find Docker Engine on your server. Do you want me to install it for you? - + Let's do it! @@ -233,12 +261,14 @@ @endif - Let's create a new one! + Let's create a new + one!
@if (count($projects) > 0)
- + @foreach ($projects as $project) diff --git a/resources/views/livewire/project/application/previews.blade.php b/resources/views/livewire/project/application/previews.blade.php index 89e3e8a23..554e6606e 100644 --- a/resources/views/livewire/project/application/previews.blade.php +++ b/resources/views/livewire/project/application/previews.blade.php @@ -48,10 +48,10 @@ @endif
@if ($application->previews->count() > 0) -

Deployed Previews

+

Deployed Previews

@foreach ($application->previews as $preview) -
+
PR #{{ data_get($preview, 'pull_request_id') }} | @if (data_get($preview, 'status') === 'running') diff --git a/resources/views/livewire/server/form.blade.php b/resources/views/livewire/server/form.blade.php index 9af0440a0..ebb40f78f 100644 --- a/resources/views/livewire/server/form.blade.php +++ b/resources/views/livewire/server/form.blade.php @@ -35,11 +35,7 @@ label="Is it part of a Swarm cluster?" /> --}}
- @if ($server->id === 0) - - @else - - @endif +
@@ -52,15 +48,15 @@ @endif @if ($server->settings->is_reachable && !$server->settings->is_usable && $server->id !== 0) - + Install Docker Engine 24.0 @endif @if ($server->isFunctional())

Settings

- + @endif

Danger Zone

diff --git a/resources/views/livewire/server/proxy/deploy.blade.php b/resources/views/livewire/server/proxy/deploy.blade.php index 66cf4301b..9f133d33a 100644 --- a/resources/views/livewire/server/proxy/deploy.blade.php +++ b/resources/views/livewire/server/proxy/deploy.blade.php @@ -11,7 +11,7 @@ @if (data_get($server, 'proxy.status') === 'running')