From f6f959a8973b3d6e11ea477758e92e848adb3d33 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 7 May 2024 15:41:50 +0200 Subject: [PATCH] feat: experimental sentinel --- app/Actions/Docker/GetContainersStatus.php | 655 ++++++++++++++++++ app/Jobs/ApplicationDeploymentJob.php | 4 +- app/Jobs/ContainerStatusJob.php | 632 +++++++++-------- app/Livewire/Project/Application/Heading.php | 4 +- app/Livewire/Project/Database/Heading.php | 4 +- .../Project/Service/Configuration.php | 4 +- app/Livewire/Project/Shared/Destination.php | 4 +- app/Livewire/Server/Proxy/Status.php | 4 +- app/Notifications/Server/Revived.php | 4 +- .../2024_05_07_124019_add_server_metrics.php | 28 + docker-compose.dev.yml | 6 + docker-compose.prod.yml | 9 + docker-compose.yml | 7 +- 13 files changed, 1038 insertions(+), 327 deletions(-) create mode 100644 app/Actions/Docker/GetContainersStatus.php create mode 100644 database/migrations/2024_05_07_124019_add_server_metrics.php diff --git a/app/Actions/Docker/GetContainersStatus.php b/app/Actions/Docker/GetContainersStatus.php new file mode 100644 index 000000000..952dbfa1b --- /dev/null +++ b/app/Actions/Docker/GetContainersStatus.php @@ -0,0 +1,655 @@ +server = Server::find(0); + } + } + public function handle() + { + if (!$this->server->isFunctional()) { + return 'Server is not ready.'; + }; + $this->applications = $this->server->applications(); + $skip_these_applications = collect([]); + foreach ($this->applications as $application) { + if ($application->additional_servers->count() > 0) { + $skip_these_applications->push($application); + ComplexStatusCheck::run($application); + $this->applications = $this->applications->filter(function ($value, $key) use ($application) { + return $value->id !== $application->id; + }); + } + } + $this->applications = $this->applications->filter(function ($value, $key) use ($skip_these_applications) { + return !$skip_these_applications->pluck('id')->contains($value->id); + }); + if ($this->server->isSwarm()) { + $this->old_way(); + } else { + if (!$this->server->is_metrics_enabled) { + $this->old_way(); + return; + } + $sentinel_found = instant_remote_process(["docker inspect coolify-sentinel"], $this->server, false); + if ($sentinel_found) { + raY('Sentinel'); + $this->sentinel(); + } else { + raY('Old way'); + $this->old_way(); + } + } + } + + private function sentinel() + { + try { + $containers = instant_remote_process(['docker exec coolify-sentinel sh -c "curl http://127.0.0.1:8888/api/containers"'], $this->server, false); + if (is_null($containers)) { + return; + } + $containers = data_get(json_decode($containers, true), 'containers', []); + $containers = collect($containers); + + $databases = $this->server->databases(); + $services = $this->server->services()->get(); + $previews = $this->server->previews(); + $foundApplications = []; + $foundApplicationPreviews = []; + $foundDatabases = []; + $foundServices = []; + + foreach ($containers as $container) { + $labels = Arr::undot(data_get($container, 'labels')); + $containerStatus = data_get($container, 'state'); + $containerHealth = data_get($container, 'health_status', 'unhealthy'); + $containerStatus = "$containerStatus ($containerHealth)"; + $applicationId = data_get($labels, 'coolify.applicationId'); + if ($applicationId) { + $pullRequestId = data_get($labels, 'coolify.pullRequestId'); + if ($pullRequestId) { + if (str($applicationId)->contains('-')) { + $applicationId = str($applicationId)->before('-'); + } + $preview = ApplicationPreview::where('application_id', $applicationId)->where('pull_request_id', $pullRequestId)->first(); + if ($preview) { + $foundApplicationPreviews[] = $preview->id; + $statusFromDb = $preview->status; + if ($statusFromDb !== $containerStatus) { + $preview->update(['status' => $containerStatus]); + } + } else { + //Notify user that this container should not be there. + } + } else { + $application = $this->applications->where('id', $applicationId)->first(); + if ($application) { + $foundApplications[] = $application->id; + $statusFromDb = $application->status; + ray($statusFromDb, $containerStatus); + if ($statusFromDb !== $containerStatus) { + $application->update(['status' => $containerStatus]); + } + } else { + //Notify user that this container should not be there. + } + } + } else { + $uuid = data_get($labels, 'com.docker.compose.service'); + $type = data_get($labels, 'coolify.type'); + + if ($uuid) { + if ($type === 'service') { + $database_id = data_get($labels, 'coolify.service.subId'); + if ($database_id) { + $service_db = ServiceDatabase::where('id', $database_id)->first(); + if ($service_db) { + $uuid = $service_db->service->uuid; + $isPublic = data_get($service_db, 'is_public'); + if ($isPublic) { + $foundTcpProxy = $containers->filter(function ($value, $key) use ($uuid) { + if ($this->server->isSwarm()) { + return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid"; + } else { + return data_get($value, 'Name') === "/$uuid-proxy"; + } + })->first(); + if (!$foundTcpProxy) { + StartDatabaseProxy::run($service_db); + // $this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$service_db->service->name}", $this->server)); + } + } + } + } + } else { + $database = $databases->where('uuid', $uuid)->first(); + if ($database) { + $isPublic = data_get($database, 'is_public'); + $foundDatabases[] = $database->id; + $statusFromDb = $database->status; + if ($statusFromDb !== $containerStatus) { + $database->update(['status' => $containerStatus]); + } + if ($isPublic) { + $foundTcpProxy = $containers->filter(function ($value, $key) use ($uuid) { + if ($this->server->isSwarm()) { + return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid"; + } else { + return data_get($value, 'Name') === "/$uuid-proxy"; + } + })->first(); + if (!$foundTcpProxy) { + StartDatabaseProxy::run($database); + $this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$database->name}", $this->server)); + } + } + } else { + // Notify user that this container should not be there. + } + } + } + if (data_get($container, 'Name') === '/coolify-db') { + $foundDatabases[] = 0; + } + } + $serviceLabelId = data_get($labels, 'coolify.serviceId'); + if ($serviceLabelId) { + $subType = data_get($labels, 'coolify.service.subType'); + $subId = data_get($labels, 'coolify.service.subId'); + $service = $services->where('id', $serviceLabelId)->first(); + if (!$service) { + continue; + } + if ($subType === 'application') { + $service = $service->applications()->where('id', $subId)->first(); + } else { + $service = $service->databases()->where('id', $subId)->first(); + } + if ($service) { + $foundServices[] = "$service->id-$service->name"; + $statusFromDb = $service->status; + if ($statusFromDb !== $containerStatus) { + // ray('Updating status: ' . $containerStatus); + $service->update(['status' => $containerStatus]); + } + } + } + } + $exitedServices = collect([]); + foreach ($services as $service) { + $apps = $service->applications()->get(); + $dbs = $service->databases()->get(); + foreach ($apps as $app) { + if (in_array("$app->id-$app->name", $foundServices)) { + continue; + } else { + $exitedServices->push($app); + } + } + foreach ($dbs as $db) { + if (in_array("$db->id-$db->name", $foundServices)) { + continue; + } else { + $exitedServices->push($db); + } + } + } + $exitedServices = $exitedServices->unique('id'); + foreach ($exitedServices as $exitedService) { + if (str($exitedService->status)->startsWith('exited')) { + continue; + } + $name = data_get($exitedService, 'name'); + $fqdn = data_get($exitedService, 'fqdn'); + $containerName = $name ? "$name, available at $fqdn" : $fqdn; + $projectUuid = data_get($service, 'environment.project.uuid'); + $serviceUuid = data_get($service, 'uuid'); + $environmentName = data_get($service, 'environment.name'); + + if ($projectUuid && $serviceUuid && $environmentName) { + $url = base_url() . '/project/' . $projectUuid . "/" . $environmentName . "/service/" . $serviceUuid; + } else { + $url = null; + } + $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); + $exitedService->update(['status' => 'exited']); + } + + $notRunningApplications = $this->applications->pluck('id')->diff($foundApplications); + foreach ($notRunningApplications as $applicationId) { + $application = $this->applications->where('id', $applicationId)->first(); + if (str($application->status)->startsWith('exited')) { + continue; + } + $application->update(['status' => 'exited']); + + $name = data_get($application, 'name'); + $fqdn = data_get($application, 'fqdn'); + + $containerName = $name ? "$name ($fqdn)" : $fqdn; + + $projectUuid = data_get($application, 'environment.project.uuid'); + $applicationUuid = data_get($application, 'uuid'); + $environment = data_get($application, 'environment.name'); + + if ($projectUuid && $applicationUuid && $environment) { + $url = base_url() . '/project/' . $projectUuid . "/" . $environment . "/application/" . $applicationUuid; + } else { + $url = null; + } + + $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); + } + $notRunningApplicationPreviews = $previews->pluck('id')->diff($foundApplicationPreviews); + foreach ($notRunningApplicationPreviews as $previewId) { + $preview = $previews->where('id', $previewId)->first(); + if (str($preview->status)->startsWith('exited')) { + continue; + } + $preview->update(['status' => 'exited']); + + $name = data_get($preview, 'name'); + $fqdn = data_get($preview, 'fqdn'); + + $containerName = $name ? "$name ($fqdn)" : $fqdn; + + $projectUuid = data_get($preview, 'application.environment.project.uuid'); + $environmentName = data_get($preview, 'application.environment.name'); + $applicationUuid = data_get($preview, 'application.uuid'); + + if ($projectUuid && $applicationUuid && $environmentName) { + $url = base_url() . '/project/' . $projectUuid . "/" . $environmentName . "/application/" . $applicationUuid; + } else { + $url = null; + } + + $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); + } + $notRunningDatabases = $databases->pluck('id')->diff($foundDatabases); + foreach ($notRunningDatabases as $database) { + $database = $databases->where('id', $database)->first(); + if (str($database->status)->startsWith('exited')) { + continue; + } + $database->update(['status' => 'exited']); + + $name = data_get($database, 'name'); + $fqdn = data_get($database, 'fqdn'); + + $containerName = $name; + + $projectUuid = data_get($database, 'environment.project.uuid'); + $environmentName = data_get($database, 'environment.name'); + $databaseUuid = data_get($database, 'uuid'); + + if ($projectUuid && $databaseUuid && $environmentName) { + $url = base_url() . '/project/' . $projectUuid . "/" . $environmentName . "/database/" . $databaseUuid; + } else { + $url = null; + } + $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); + } + + // Check if proxy is running + $this->server->proxyType(); + $foundProxyContainer = $containers->filter(function ($value, $key) { + if ($this->server->isSwarm()) { + return data_get($value, 'Spec.Name') === 'coolify-proxy_traefik'; + } else { + return data_get($value, 'Name') === '/coolify-proxy'; + } + })->first(); + if (!$foundProxyContainer) { + try { + $shouldStart = CheckProxy::run($this->server); + if ($shouldStart) { + StartProxy::run($this->server, false); + $this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server)); + } + } catch (\Throwable $e) { + ray($e); + } + } else { + $this->server->proxy->status = data_get($foundProxyContainer, 'State.Status'); + $this->server->save(); + $connectProxyToDockerNetworks = connectProxyToNetworks($this->server); + instant_remote_process($connectProxyToDockerNetworks, $this->server, false); + } + } catch (\Exception $e) { + send_internal_notification("ContainerStatusJob failed on ({$this->server->id}) with: " . $e->getMessage()); + ray($e->getMessage()); + return handleError($e); + } + } + private function old_way() + { + if ($this->server->isSwarm()) { + $containers = instant_remote_process(["docker service inspect $(docker service ls -q) --format '{{json .}}'"], $this->server, false); + $containerReplicates = instant_remote_process(["docker service ls --format '{{json .}}'"], $this->server, false); + } else { + // Precheck for containers + $containers = instant_remote_process(["docker container ls -q"], $this->server, false); + if (!$containers) { + return; + } + $containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this->server, false); + $containerReplicates = null; + } + if (is_null($containers)) { + return; + } + + $containers = format_docker_command_output_to_json($containers); + if ($containerReplicates) { + $containerReplicates = format_docker_command_output_to_json($containerReplicates); + foreach ($containerReplicates as $containerReplica) { + $name = data_get($containerReplica, 'Name'); + $containers = $containers->map(function ($container) use ($name, $containerReplica) { + if (data_get($container, 'Spec.Name') === $name) { + $replicas = data_get($containerReplica, 'Replicas'); + $running = str($replicas)->explode('/')[0]; + $total = str($replicas)->explode('/')[1]; + if ($running === $total) { + data_set($container, 'State.Status', 'running'); + data_set($container, 'State.Health.Status', 'healthy'); + } else { + data_set($container, 'State.Status', 'starting'); + data_set($container, 'State.Health.Status', 'unhealthy'); + } + } + return $container; + }); + } + } + $databases = $this->server->databases(); + $services = $this->server->services()->get(); + $previews = $this->server->previews(); + $foundApplications = []; + $foundApplicationPreviews = []; + $foundDatabases = []; + $foundServices = []; + + foreach ($containers as $container) { + if ($this->server->isSwarm()) { + $labels = data_get($container, 'Spec.Labels'); + $uuid = data_get($labels, 'coolify.name'); + } else { + $labels = data_get($container, 'Config.Labels'); + } + $containerStatus = data_get($container, 'State.Status'); + $containerHealth = data_get($container, 'State.Health.Status', 'unhealthy'); + $containerStatus = "$containerStatus ($containerHealth)"; + $labels = Arr::undot(format_docker_labels_to_json($labels)); + $applicationId = data_get($labels, 'coolify.applicationId'); + if ($applicationId) { + $pullRequestId = data_get($labels, 'coolify.pullRequestId'); + if ($pullRequestId) { + if (str($applicationId)->contains('-')) { + $applicationId = str($applicationId)->before('-'); + } + $preview = ApplicationPreview::where('application_id', $applicationId)->where('pull_request_id', $pullRequestId)->first(); + if ($preview) { + $foundApplicationPreviews[] = $preview->id; + $statusFromDb = $preview->status; + if ($statusFromDb !== $containerStatus) { + $preview->update(['status' => $containerStatus]); + } + } else { + //Notify user that this container should not be there. + } + } else { + $application = $this->applications->where('id', $applicationId)->first(); + if ($application) { + $foundApplications[] = $application->id; + $statusFromDb = $application->status; + if ($statusFromDb !== $containerStatus) { + $application->update(['status' => $containerStatus]); + } + } else { + //Notify user that this container should not be there. + } + } + } else { + $uuid = data_get($labels, 'com.docker.compose.service'); + $type = data_get($labels, 'coolify.type'); + + if ($uuid) { + if ($type === 'service') { + $database_id = data_get($labels, 'coolify.service.subId'); + if ($database_id) { + $service_db = ServiceDatabase::where('id', $database_id)->first(); + if ($service_db) { + $uuid = $service_db->service->uuid; + $isPublic = data_get($service_db, 'is_public'); + if ($isPublic) { + $foundTcpProxy = $containers->filter(function ($value, $key) use ($uuid) { + if ($this->server->isSwarm()) { + return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid"; + } else { + return data_get($value, 'Name') === "/$uuid-proxy"; + } + })->first(); + if (!$foundTcpProxy) { + StartDatabaseProxy::run($service_db); + // $this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$service_db->service->name}", $this->server)); + } + } + } + } + } else { + $database = $databases->where('uuid', $uuid)->first(); + if ($database) { + $isPublic = data_get($database, 'is_public'); + $foundDatabases[] = $database->id; + $statusFromDb = $database->status; + if ($statusFromDb !== $containerStatus) { + $database->update(['status' => $containerStatus]); + } + if ($isPublic) { + $foundTcpProxy = $containers->filter(function ($value, $key) use ($uuid) { + if ($this->server->isSwarm()) { + return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid"; + } else { + return data_get($value, 'Name') === "/$uuid-proxy"; + } + })->first(); + if (!$foundTcpProxy) { + StartDatabaseProxy::run($database); + $this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$database->name}", $this->server)); + } + } + } else { + // Notify user that this container should not be there. + } + } + } + if (data_get($container, 'Name') === '/coolify-db') { + $foundDatabases[] = 0; + } + } + $serviceLabelId = data_get($labels, 'coolify.serviceId'); + if ($serviceLabelId) { + $subType = data_get($labels, 'coolify.service.subType'); + $subId = data_get($labels, 'coolify.service.subId'); + $service = $services->where('id', $serviceLabelId)->first(); + if (!$service) { + continue; + } + if ($subType === 'application') { + $service = $service->applications()->where('id', $subId)->first(); + } else { + $service = $service->databases()->where('id', $subId)->first(); + } + if ($service) { + $foundServices[] = "$service->id-$service->name"; + $statusFromDb = $service->status; + if ($statusFromDb !== $containerStatus) { + // ray('Updating status: ' . $containerStatus); + $service->update(['status' => $containerStatus]); + } + } + } + } + $exitedServices = collect([]); + foreach ($services as $service) { + $apps = $service->applications()->get(); + $dbs = $service->databases()->get(); + foreach ($apps as $app) { + if (in_array("$app->id-$app->name", $foundServices)) { + continue; + } else { + $exitedServices->push($app); + } + } + foreach ($dbs as $db) { + if (in_array("$db->id-$db->name", $foundServices)) { + continue; + } else { + $exitedServices->push($db); + } + } + } + $exitedServices = $exitedServices->unique('id'); + foreach ($exitedServices as $exitedService) { + if (str($exitedService->status)->startsWith('exited')) { + continue; + } + $name = data_get($exitedService, 'name'); + $fqdn = data_get($exitedService, 'fqdn'); + $containerName = $name ? "$name, available at $fqdn" : $fqdn; + $projectUuid = data_get($service, 'environment.project.uuid'); + $serviceUuid = data_get($service, 'uuid'); + $environmentName = data_get($service, 'environment.name'); + + if ($projectUuid && $serviceUuid && $environmentName) { + $url = base_url() . '/project/' . $projectUuid . "/" . $environmentName . "/service/" . $serviceUuid; + } else { + $url = null; + } + $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); + $exitedService->update(['status' => 'exited']); + } + + $notRunningApplications = $this->applications->pluck('id')->diff($foundApplications); + foreach ($notRunningApplications as $applicationId) { + $application = $this->applications->where('id', $applicationId)->first(); + if (str($application->status)->startsWith('exited')) { + continue; + } + $application->update(['status' => 'exited']); + + $name = data_get($application, 'name'); + $fqdn = data_get($application, 'fqdn'); + + $containerName = $name ? "$name ($fqdn)" : $fqdn; + + $projectUuid = data_get($application, 'environment.project.uuid'); + $applicationUuid = data_get($application, 'uuid'); + $environment = data_get($application, 'environment.name'); + + if ($projectUuid && $applicationUuid && $environment) { + $url = base_url() . '/project/' . $projectUuid . "/" . $environment . "/application/" . $applicationUuid; + } else { + $url = null; + } + + $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); + } + $notRunningApplicationPreviews = $previews->pluck('id')->diff($foundApplicationPreviews); + foreach ($notRunningApplicationPreviews as $previewId) { + $preview = $previews->where('id', $previewId)->first(); + if (str($preview->status)->startsWith('exited')) { + continue; + } + $preview->update(['status' => 'exited']); + + $name = data_get($preview, 'name'); + $fqdn = data_get($preview, 'fqdn'); + + $containerName = $name ? "$name ($fqdn)" : $fqdn; + + $projectUuid = data_get($preview, 'application.environment.project.uuid'); + $environmentName = data_get($preview, 'application.environment.name'); + $applicationUuid = data_get($preview, 'application.uuid'); + + if ($projectUuid && $applicationUuid && $environmentName) { + $url = base_url() . '/project/' . $projectUuid . "/" . $environmentName . "/application/" . $applicationUuid; + } else { + $url = null; + } + + $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); + } + $notRunningDatabases = $databases->pluck('id')->diff($foundDatabases); + foreach ($notRunningDatabases as $database) { + $database = $databases->where('id', $database)->first(); + if (str($database->status)->startsWith('exited')) { + continue; + } + $database->update(['status' => 'exited']); + + $name = data_get($database, 'name'); + $fqdn = data_get($database, 'fqdn'); + + $containerName = $name; + + $projectUuid = data_get($database, 'environment.project.uuid'); + $environmentName = data_get($database, 'environment.name'); + $databaseUuid = data_get($database, 'uuid'); + + if ($projectUuid && $databaseUuid && $environmentName) { + $url = base_url() . '/project/' . $projectUuid . "/" . $environmentName . "/database/" . $databaseUuid; + } else { + $url = null; + } + $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); + } + + // Check if proxy is running + $this->server->proxyType(); + $foundProxyContainer = $containers->filter(function ($value, $key) { + if ($this->server->isSwarm()) { + return data_get($value, 'Spec.Name') === 'coolify-proxy_traefik'; + } else { + return data_get($value, 'Name') === '/coolify-proxy'; + } + })->first(); + if (!$foundProxyContainer) { + try { + $shouldStart = CheckProxy::run($this->server); + if ($shouldStart) { + StartProxy::run($this->server, false); + $this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server)); + } + } catch (\Throwable $e) { + ray($e); + } + } else { + $this->server->proxy->status = data_get($foundProxyContainer, 'State.Status'); + $this->server->save(); + $connectProxyToDockerNetworks = connectProxyToNetworks($this->server); + instant_remote_process($connectProxyToDockerNetworks, $this->server, false); + } + } +} diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index e3221e091..54b96ee68 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -2,6 +2,7 @@ namespace App\Jobs; +use App\Actions\Docker\GetContainersStatus; use App\Enums\ApplicationDeploymentStatus; use App\Enums\ProcessStatus; use App\Events\ApplicationStatusChanged; @@ -302,7 +303,8 @@ class ApplicationDeploymentJob implements ShouldQueue, ShouldBeEncrypted { if ($this->server->isProxyShouldRun()) { - dispatch(new ContainerStatusJob($this->server)); + GetContainersStatus::dispatch($this->server); + // dispatch(new ContainerStatusJob($this->server)); } $this->next(ApplicationDeploymentStatus::FINISHED->value); if ($this->pull_request_id !== 0) { diff --git a/app/Jobs/ContainerStatusJob.php b/app/Jobs/ContainerStatusJob.php index a9cec009b..96caf5dba 100644 --- a/app/Jobs/ContainerStatusJob.php +++ b/app/Jobs/ContainerStatusJob.php @@ -2,15 +2,8 @@ namespace App\Jobs; -use App\Actions\Database\StartDatabaseProxy; -use App\Actions\Proxy\CheckProxy; -use App\Actions\Proxy\StartProxy; -use App\Actions\Shared\ComplexStatusCheck; -use App\Models\ApplicationPreview; +use App\Actions\Docker\GetContainersStatus; use App\Models\Server; -use App\Models\ServiceDatabase; -use App\Notifications\Container\ContainerRestarted; -use App\Notifications\Container\ContainerStopped; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldQueue; @@ -18,7 +11,6 @@ use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\Middleware\WithoutOverlapping; use Illuminate\Queue\SerializesModels; -use Illuminate\Support\Arr; class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted { @@ -44,335 +36,337 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted public function handle() { - if (!$this->server->isFunctional()) { - return 'Server is not ready.'; - }; - $applications = $this->server->applications(); - $skip_these_applications = collect([]); - foreach ($applications as $application) { - if ($application->additional_servers->count() > 0) { - $skip_these_applications->push($application); - ComplexStatusCheck::run($application); - $applications = $applications->filter(function ($value, $key) use ($application) { - return $value->id !== $application->id; - }); - } - } - $applications = $applications->filter(function ($value, $key) use ($skip_these_applications) { - return !$skip_these_applications->pluck('id')->contains($value->id); - }); - try { - if ($this->server->isSwarm()) { - $containers = instant_remote_process(["docker service inspect $(docker service ls -q) --format '{{json .}}'"], $this->server, false); - $containerReplicates = instant_remote_process(["docker service ls --format '{{json .}}'"], $this->server, false); - } else { - // Precheck for containers - $containers = instant_remote_process(["docker container ls -q"], $this->server, false); - if (!$containers) { - return; - } - $containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this->server, false); - $containerReplicates = null; - } - if (is_null($containers)) { - return; - } + GetContainersStatus::run(); + return; + // if (!$this->server->isFunctional()) { + // return 'Server is not ready.'; + // }; + // $applications = $this->server->applications(); + // $skip_these_applications = collect([]); + // foreach ($applications as $application) { + // if ($application->additional_servers->count() > 0) { + // $skip_these_applications->push($application); + // ComplexStatusCheck::run($application); + // $applications = $applications->filter(function ($value, $key) use ($application) { + // return $value->id !== $application->id; + // }); + // } + // } + // $applications = $applications->filter(function ($value, $key) use ($skip_these_applications) { + // return !$skip_these_applications->pluck('id')->contains($value->id); + // }); + // try { + // if ($this->server->isSwarm()) { + // $containers = instant_remote_process(["docker service inspect $(docker service ls -q) --format '{{json .}}'"], $this->server, false); + // $containerReplicates = instant_remote_process(["docker service ls --format '{{json .}}'"], $this->server, false); + // } else { + // // Precheck for containers + // $containers = instant_remote_process(["docker container ls -q"], $this->server, false); + // if (!$containers) { + // return; + // } + // $containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this->server, false); + // $containerReplicates = null; + // } + // if (is_null($containers)) { + // return; + // } - $containers = format_docker_command_output_to_json($containers); - if ($containerReplicates) { - $containerReplicates = format_docker_command_output_to_json($containerReplicates); - foreach ($containerReplicates as $containerReplica) { - $name = data_get($containerReplica, 'Name'); - $containers = $containers->map(function ($container) use ($name, $containerReplica) { - if (data_get($container, 'Spec.Name') === $name) { - $replicas = data_get($containerReplica, 'Replicas'); - $running = str($replicas)->explode('/')[0]; - $total = str($replicas)->explode('/')[1]; - if ($running === $total) { - data_set($container, 'State.Status', 'running'); - data_set($container, 'State.Health.Status', 'healthy'); - } else { - data_set($container, 'State.Status', 'starting'); - data_set($container, 'State.Health.Status', 'unhealthy'); - } - } - return $container; - }); - } - } - $databases = $this->server->databases(); - $services = $this->server->services()->get(); - $previews = $this->server->previews(); - $foundApplications = []; - $foundApplicationPreviews = []; - $foundDatabases = []; - $foundServices = []; + // $containers = format_docker_command_output_to_json($containers); + // if ($containerReplicates) { + // $containerReplicates = format_docker_command_output_to_json($containerReplicates); + // foreach ($containerReplicates as $containerReplica) { + // $name = data_get($containerReplica, 'Name'); + // $containers = $containers->map(function ($container) use ($name, $containerReplica) { + // if (data_get($container, 'Spec.Name') === $name) { + // $replicas = data_get($containerReplica, 'Replicas'); + // $running = str($replicas)->explode('/')[0]; + // $total = str($replicas)->explode('/')[1]; + // if ($running === $total) { + // data_set($container, 'State.Status', 'running'); + // data_set($container, 'State.Health.Status', 'healthy'); + // } else { + // data_set($container, 'State.Status', 'starting'); + // data_set($container, 'State.Health.Status', 'unhealthy'); + // } + // } + // return $container; + // }); + // } + // } + // $databases = $this->server->databases(); + // $services = $this->server->services()->get(); + // $previews = $this->server->previews(); + // $foundApplications = []; + // $foundApplicationPreviews = []; + // $foundDatabases = []; + // $foundServices = []; - foreach ($containers as $container) { - if ($this->server->isSwarm()) { - $labels = data_get($container, 'Spec.Labels'); - $uuid = data_get($labels, 'coolify.name'); - } else { - $labels = data_get($container, 'Config.Labels'); - } - $containerStatus = data_get($container, 'State.Status'); - $containerHealth = data_get($container, 'State.Health.Status', 'unhealthy'); - $containerStatus = "$containerStatus ($containerHealth)"; - $labels = Arr::undot(format_docker_labels_to_json($labels)); - $applicationId = data_get($labels, 'coolify.applicationId'); - if ($applicationId) { - $pullRequestId = data_get($labels, 'coolify.pullRequestId'); - if ($pullRequestId) { - if (str($applicationId)->contains('-')) { - $applicationId = str($applicationId)->before('-'); - } - $preview = ApplicationPreview::where('application_id', $applicationId)->where('pull_request_id', $pullRequestId)->first(); - if ($preview) { - $foundApplicationPreviews[] = $preview->id; - $statusFromDb = $preview->status; - if ($statusFromDb !== $containerStatus) { - $preview->update(['status' => $containerStatus]); - } - } else { - //Notify user that this container should not be there. - } - } else { - $application = $applications->where('id', $applicationId)->first(); - if ($application) { - $foundApplications[] = $application->id; - $statusFromDb = $application->status; - if ($statusFromDb !== $containerStatus) { - $application->update(['status' => $containerStatus]); - } - } else { - //Notify user that this container should not be there. - } - } - } else { - $uuid = data_get($labels, 'com.docker.compose.service'); - $type = data_get($labels, 'coolify.type'); + // foreach ($containers as $container) { + // if ($this->server->isSwarm()) { + // $labels = data_get($container, 'Spec.Labels'); + // $uuid = data_get($labels, 'coolify.name'); + // } else { + // $labels = data_get($container, 'Config.Labels'); + // } + // $containerStatus = data_get($container, 'State.Status'); + // $containerHealth = data_get($container, 'State.Health.Status', 'unhealthy'); + // $containerStatus = "$containerStatus ($containerHealth)"; + // $labels = Arr::undot(format_docker_labels_to_json($labels)); + // $applicationId = data_get($labels, 'coolify.applicationId'); + // if ($applicationId) { + // $pullRequestId = data_get($labels, 'coolify.pullRequestId'); + // if ($pullRequestId) { + // if (str($applicationId)->contains('-')) { + // $applicationId = str($applicationId)->before('-'); + // } + // $preview = ApplicationPreview::where('application_id', $applicationId)->where('pull_request_id', $pullRequestId)->first(); + // if ($preview) { + // $foundApplicationPreviews[] = $preview->id; + // $statusFromDb = $preview->status; + // if ($statusFromDb !== $containerStatus) { + // $preview->update(['status' => $containerStatus]); + // } + // } else { + // //Notify user that this container should not be there. + // } + // } else { + // $application = $applications->where('id', $applicationId)->first(); + // if ($application) { + // $foundApplications[] = $application->id; + // $statusFromDb = $application->status; + // if ($statusFromDb !== $containerStatus) { + // $application->update(['status' => $containerStatus]); + // } + // } else { + // //Notify user that this container should not be there. + // } + // } + // } else { + // $uuid = data_get($labels, 'com.docker.compose.service'); + // $type = data_get($labels, 'coolify.type'); - if ($uuid) { - if ($type === 'service') { - $database_id = data_get($labels, 'coolify.service.subId'); - if ($database_id) { - $service_db = ServiceDatabase::where('id', $database_id)->first(); - if ($service_db) { - $uuid = $service_db->service->uuid; - $isPublic = data_get($service_db, 'is_public'); - if ($isPublic) { - $foundTcpProxy = $containers->filter(function ($value, $key) use ($uuid) { - if ($this->server->isSwarm()) { - return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid"; - } else { - return data_get($value, 'Name') === "/$uuid-proxy"; - } - })->first(); - if (!$foundTcpProxy) { - StartDatabaseProxy::run($service_db); - // $this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$service_db->service->name}", $this->server)); - } - } - } - } - } else { - $database = $databases->where('uuid', $uuid)->first(); - if ($database) { - $isPublic = data_get($database, 'is_public'); - $foundDatabases[] = $database->id; - $statusFromDb = $database->status; - if ($statusFromDb !== $containerStatus) { - $database->update(['status' => $containerStatus]); - } - if ($isPublic) { - $foundTcpProxy = $containers->filter(function ($value, $key) use ($uuid) { - if ($this->server->isSwarm()) { - return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid"; - } else { - return data_get($value, 'Name') === "/$uuid-proxy"; - } - })->first(); - if (!$foundTcpProxy) { - StartDatabaseProxy::run($database); - $this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$database->name}", $this->server)); - } - } - } else { - // Notify user that this container should not be there. - } - } - } - if (data_get($container, 'Name') === '/coolify-db') { - $foundDatabases[] = 0; - } - } - $serviceLabelId = data_get($labels, 'coolify.serviceId'); - if ($serviceLabelId) { - $subType = data_get($labels, 'coolify.service.subType'); - $subId = data_get($labels, 'coolify.service.subId'); - $service = $services->where('id', $serviceLabelId)->first(); - if (!$service) { - continue; - } - if ($subType === 'application') { - $service = $service->applications()->where('id', $subId)->first(); - } else { - $service = $service->databases()->where('id', $subId)->first(); - } - if ($service) { - $foundServices[] = "$service->id-$service->name"; - $statusFromDb = $service->status; - if ($statusFromDb !== $containerStatus) { - // ray('Updating status: ' . $containerStatus); - $service->update(['status' => $containerStatus]); - } - } - } - } - $exitedServices = collect([]); - foreach ($services as $service) { - $apps = $service->applications()->get(); - $dbs = $service->databases()->get(); - foreach ($apps as $app) { - if (in_array("$app->id-$app->name", $foundServices)) { - continue; - } else { - $exitedServices->push($app); - } - } - foreach ($dbs as $db) { - if (in_array("$db->id-$db->name", $foundServices)) { - continue; - } else { - $exitedServices->push($db); - } - } - } - $exitedServices = $exitedServices->unique('id'); - foreach ($exitedServices as $exitedService) { - if (str($exitedService->status)->startsWith('exited')) { - continue; - } - $name = data_get($exitedService, 'name'); - $fqdn = data_get($exitedService, 'fqdn'); - $containerName = $name ? "$name, available at $fqdn" : $fqdn; - $projectUuid = data_get($service, 'environment.project.uuid'); - $serviceUuid = data_get($service, 'uuid'); - $environmentName = data_get($service, 'environment.name'); + // if ($uuid) { + // if ($type === 'service') { + // $database_id = data_get($labels, 'coolify.service.subId'); + // if ($database_id) { + // $service_db = ServiceDatabase::where('id', $database_id)->first(); + // if ($service_db) { + // $uuid = $service_db->service->uuid; + // $isPublic = data_get($service_db, 'is_public'); + // if ($isPublic) { + // $foundTcpProxy = $containers->filter(function ($value, $key) use ($uuid) { + // if ($this->server->isSwarm()) { + // return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid"; + // } else { + // return data_get($value, 'Name') === "/$uuid-proxy"; + // } + // })->first(); + // if (!$foundTcpProxy) { + // StartDatabaseProxy::run($service_db); + // // $this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$service_db->service->name}", $this->server)); + // } + // } + // } + // } + // } else { + // $database = $databases->where('uuid', $uuid)->first(); + // if ($database) { + // $isPublic = data_get($database, 'is_public'); + // $foundDatabases[] = $database->id; + // $statusFromDb = $database->status; + // if ($statusFromDb !== $containerStatus) { + // $database->update(['status' => $containerStatus]); + // } + // if ($isPublic) { + // $foundTcpProxy = $containers->filter(function ($value, $key) use ($uuid) { + // if ($this->server->isSwarm()) { + // return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid"; + // } else { + // return data_get($value, 'Name') === "/$uuid-proxy"; + // } + // })->first(); + // if (!$foundTcpProxy) { + // StartDatabaseProxy::run($database); + // $this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$database->name}", $this->server)); + // } + // } + // } else { + // // Notify user that this container should not be there. + // } + // } + // } + // if (data_get($container, 'Name') === '/coolify-db') { + // $foundDatabases[] = 0; + // } + // } + // $serviceLabelId = data_get($labels, 'coolify.serviceId'); + // if ($serviceLabelId) { + // $subType = data_get($labels, 'coolify.service.subType'); + // $subId = data_get($labels, 'coolify.service.subId'); + // $service = $services->where('id', $serviceLabelId)->first(); + // if (!$service) { + // continue; + // } + // if ($subType === 'application') { + // $service = $service->applications()->where('id', $subId)->first(); + // } else { + // $service = $service->databases()->where('id', $subId)->first(); + // } + // if ($service) { + // $foundServices[] = "$service->id-$service->name"; + // $statusFromDb = $service->status; + // if ($statusFromDb !== $containerStatus) { + // // ray('Updating status: ' . $containerStatus); + // $service->update(['status' => $containerStatus]); + // } + // } + // } + // } + // $exitedServices = collect([]); + // foreach ($services as $service) { + // $apps = $service->applications()->get(); + // $dbs = $service->databases()->get(); + // foreach ($apps as $app) { + // if (in_array("$app->id-$app->name", $foundServices)) { + // continue; + // } else { + // $exitedServices->push($app); + // } + // } + // foreach ($dbs as $db) { + // if (in_array("$db->id-$db->name", $foundServices)) { + // continue; + // } else { + // $exitedServices->push($db); + // } + // } + // } + // $exitedServices = $exitedServices->unique('id'); + // foreach ($exitedServices as $exitedService) { + // if (str($exitedService->status)->startsWith('exited')) { + // continue; + // } + // $name = data_get($exitedService, 'name'); + // $fqdn = data_get($exitedService, 'fqdn'); + // $containerName = $name ? "$name, available at $fqdn" : $fqdn; + // $projectUuid = data_get($service, 'environment.project.uuid'); + // $serviceUuid = data_get($service, 'uuid'); + // $environmentName = data_get($service, 'environment.name'); - if ($projectUuid && $serviceUuid && $environmentName) { - $url = base_url() . '/project/' . $projectUuid . "/" . $environmentName . "/service/" . $serviceUuid; - } else { - $url = null; - } - $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); - $exitedService->update(['status' => 'exited']); - } + // if ($projectUuid && $serviceUuid && $environmentName) { + // $url = base_url() . '/project/' . $projectUuid . "/" . $environmentName . "/service/" . $serviceUuid; + // } else { + // $url = null; + // } + // $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); + // $exitedService->update(['status' => 'exited']); + // } - $notRunningApplications = $applications->pluck('id')->diff($foundApplications); - foreach ($notRunningApplications as $applicationId) { - $application = $applications->where('id', $applicationId)->first(); - if (str($application->status)->startsWith('exited')) { - continue; - } - $application->update(['status' => 'exited']); + // $notRunningApplications = $applications->pluck('id')->diff($foundApplications); + // foreach ($notRunningApplications as $applicationId) { + // $application = $applications->where('id', $applicationId)->first(); + // if (str($application->status)->startsWith('exited')) { + // continue; + // } + // $application->update(['status' => 'exited']); - $name = data_get($application, 'name'); - $fqdn = data_get($application, 'fqdn'); + // $name = data_get($application, 'name'); + // $fqdn = data_get($application, 'fqdn'); - $containerName = $name ? "$name ($fqdn)" : $fqdn; + // $containerName = $name ? "$name ($fqdn)" : $fqdn; - $projectUuid = data_get($application, 'environment.project.uuid'); - $applicationUuid = data_get($application, 'uuid'); - $environment = data_get($application, 'environment.name'); + // $projectUuid = data_get($application, 'environment.project.uuid'); + // $applicationUuid = data_get($application, 'uuid'); + // $environment = data_get($application, 'environment.name'); - if ($projectUuid && $applicationUuid && $environment) { - $url = base_url() . '/project/' . $projectUuid . "/" . $environment . "/application/" . $applicationUuid; - } else { - $url = null; - } + // if ($projectUuid && $applicationUuid && $environment) { + // $url = base_url() . '/project/' . $projectUuid . "/" . $environment . "/application/" . $applicationUuid; + // } else { + // $url = null; + // } - $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); - } - $notRunningApplicationPreviews = $previews->pluck('id')->diff($foundApplicationPreviews); - foreach ($notRunningApplicationPreviews as $previewId) { - $preview = $previews->where('id', $previewId)->first(); - if (str($preview->status)->startsWith('exited')) { - continue; - } - $preview->update(['status' => 'exited']); + // $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); + // } + // $notRunningApplicationPreviews = $previews->pluck('id')->diff($foundApplicationPreviews); + // foreach ($notRunningApplicationPreviews as $previewId) { + // $preview = $previews->where('id', $previewId)->first(); + // if (str($preview->status)->startsWith('exited')) { + // continue; + // } + // $preview->update(['status' => 'exited']); - $name = data_get($preview, 'name'); - $fqdn = data_get($preview, 'fqdn'); + // $name = data_get($preview, 'name'); + // $fqdn = data_get($preview, 'fqdn'); - $containerName = $name ? "$name ($fqdn)" : $fqdn; + // $containerName = $name ? "$name ($fqdn)" : $fqdn; - $projectUuid = data_get($preview, 'application.environment.project.uuid'); - $environmentName = data_get($preview, 'application.environment.name'); - $applicationUuid = data_get($preview, 'application.uuid'); + // $projectUuid = data_get($preview, 'application.environment.project.uuid'); + // $environmentName = data_get($preview, 'application.environment.name'); + // $applicationUuid = data_get($preview, 'application.uuid'); - if ($projectUuid && $applicationUuid && $environmentName) { - $url = base_url() . '/project/' . $projectUuid . "/" . $environmentName . "/application/" . $applicationUuid; - } else { - $url = null; - } + // if ($projectUuid && $applicationUuid && $environmentName) { + // $url = base_url() . '/project/' . $projectUuid . "/" . $environmentName . "/application/" . $applicationUuid; + // } else { + // $url = null; + // } - $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); - } - $notRunningDatabases = $databases->pluck('id')->diff($foundDatabases); - foreach ($notRunningDatabases as $database) { - $database = $databases->where('id', $database)->first(); - if (str($database->status)->startsWith('exited')) { - continue; - } - $database->update(['status' => 'exited']); + // $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); + // } + // $notRunningDatabases = $databases->pluck('id')->diff($foundDatabases); + // foreach ($notRunningDatabases as $database) { + // $database = $databases->where('id', $database)->first(); + // if (str($database->status)->startsWith('exited')) { + // continue; + // } + // $database->update(['status' => 'exited']); - $name = data_get($database, 'name'); - $fqdn = data_get($database, 'fqdn'); + // $name = data_get($database, 'name'); + // $fqdn = data_get($database, 'fqdn'); - $containerName = $name; + // $containerName = $name; - $projectUuid = data_get($database, 'environment.project.uuid'); - $environmentName = data_get($database, 'environment.name'); - $databaseUuid = data_get($database, 'uuid'); + // $projectUuid = data_get($database, 'environment.project.uuid'); + // $environmentName = data_get($database, 'environment.name'); + // $databaseUuid = data_get($database, 'uuid'); - if ($projectUuid && $databaseUuid && $environmentName) { - $url = base_url() . '/project/' . $projectUuid . "/" . $environmentName . "/database/" . $databaseUuid; - } else { - $url = null; - } - $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); - } + // if ($projectUuid && $databaseUuid && $environmentName) { + // $url = base_url() . '/project/' . $projectUuid . "/" . $environmentName . "/database/" . $databaseUuid; + // } else { + // $url = null; + // } + // $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url)); + // } - // Check if proxy is running - $this->server->proxyType(); - $foundProxyContainer = $containers->filter(function ($value, $key) { - if ($this->server->isSwarm()) { - return data_get($value, 'Spec.Name') === 'coolify-proxy_traefik'; - } else { - return data_get($value, 'Name') === '/coolify-proxy'; - } - })->first(); - if (!$foundProxyContainer) { - try { - $shouldStart = CheckProxy::run($this->server); - if ($shouldStart) { - StartProxy::run($this->server, false); - $this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server)); - } - } catch (\Throwable $e) { - ray($e); - } - } else { - $this->server->proxy->status = data_get($foundProxyContainer, 'State.Status'); - $this->server->save(); - $connectProxyToDockerNetworks = connectProxyToNetworks($this->server); - instant_remote_process($connectProxyToDockerNetworks, $this->server, false); - } - } catch (\Throwable $e) { - send_internal_notification("ContainerStatusJob failed on ({$this->server->id}) with: " . $e->getMessage()); - ray($e->getMessage()); - return handleError($e); - } + // // Check if proxy is running + // $this->server->proxyType(); + // $foundProxyContainer = $containers->filter(function ($value, $key) { + // if ($this->server->isSwarm()) { + // return data_get($value, 'Spec.Name') === 'coolify-proxy_traefik'; + // } else { + // return data_get($value, 'Name') === '/coolify-proxy'; + // } + // })->first(); + // if (!$foundProxyContainer) { + // try { + // $shouldStart = CheckProxy::run($this->server); + // if ($shouldStart) { + // StartProxy::run($this->server, false); + // $this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server)); + // } + // } catch (\Throwable $e) { + // ray($e); + // } + // } else { + // $this->server->proxy->status = data_get($foundProxyContainer, 'State.Status'); + // $this->server->save(); + // $connectProxyToDockerNetworks = connectProxyToNetworks($this->server); + // instant_remote_process($connectProxyToDockerNetworks, $this->server, false); + // } + // } catch (\Throwable $e) { + // send_internal_notification("ContainerStatusJob failed on ({$this->server->id}) with: " . $e->getMessage()); + // ray($e->getMessage()); + // return handleError($e); + // } } } diff --git a/app/Livewire/Project/Application/Heading.php b/app/Livewire/Project/Application/Heading.php index 9adee7003..06b9a3c3c 100644 --- a/app/Livewire/Project/Application/Heading.php +++ b/app/Livewire/Project/Application/Heading.php @@ -3,6 +3,7 @@ namespace App\Livewire\Project\Application; use App\Actions\Application\StopApplication; +use App\Actions\Docker\GetContainersStatus; use App\Events\ApplicationStatusChanged; use App\Jobs\ContainerStatusJob; use App\Jobs\ServerStatusJob; @@ -32,7 +33,8 @@ class Heading extends Component public function check_status($showNotification = false) { if ($this->application->destination->server->isFunctional()) { - dispatch(new ContainerStatusJob($this->application->destination->server)); + GetContainersStatus::dispatch($this->application->destination->server); + // dispatch(new ContainerStatusJob($this->application->destination->server)); } else { dispatch(new ServerStatusJob($this->application->destination->server)); } diff --git a/app/Livewire/Project/Database/Heading.php b/app/Livewire/Project/Database/Heading.php index 960ff2689..b454a683c 100644 --- a/app/Livewire/Project/Database/Heading.php +++ b/app/Livewire/Project/Database/Heading.php @@ -11,6 +11,7 @@ use App\Actions\Database\StartMysql; use App\Actions\Database\StartPostgresql; use App\Actions\Database\StartRedis; use App\Actions\Database\StopDatabase; +use App\Actions\Docker\GetContainersStatus; use App\Jobs\ContainerStatusJob; use Livewire\Component; @@ -44,7 +45,8 @@ class Heading extends Component public function check_status($showNotification = false) { - dispatch_sync(new ContainerStatusJob($this->database->destination->server)); + GetContainersStatus::run($this->application->destination->server); + // dispatch_sync(new ContainerStatusJob($this->database->destination->server)); $this->database->refresh(); if ($showNotification) $this->dispatch('success', 'Database status updated.'); } diff --git a/app/Livewire/Project/Service/Configuration.php b/app/Livewire/Project/Service/Configuration.php index 2cbda4e02..0b26af22f 100644 --- a/app/Livewire/Project/Service/Configuration.php +++ b/app/Livewire/Project/Service/Configuration.php @@ -2,6 +2,7 @@ namespace App\Livewire\Project\Service; +use App\Actions\Docker\GetContainersStatus; use App\Jobs\ContainerStatusJob; use App\Models\Service; use Livewire\Component; @@ -64,7 +65,8 @@ class Configuration extends Component public function check_status() { try { - dispatch_sync(new ContainerStatusJob($this->service->server)); + GetContainersStatus::run($this->service->server); + // dispatch_sync(new ContainerStatusJob($this->service->server)); $this->dispatch('refresh')->self(); $this->dispatch('updateStatus'); } catch (\Exception $e) { diff --git a/app/Livewire/Project/Shared/Destination.php b/app/Livewire/Project/Shared/Destination.php index fa19e8c42..2ccae47fd 100644 --- a/app/Livewire/Project/Shared/Destination.php +++ b/app/Livewire/Project/Shared/Destination.php @@ -3,6 +3,7 @@ namespace App\Livewire\Project\Shared; use App\Actions\Application\StopApplicationOneServer; +use App\Actions\Docker\GetContainersStatus; use App\Events\ApplicationStatusChanged; use App\Jobs\ContainerStatusJob; use App\Models\Server; @@ -90,7 +91,8 @@ class Destination extends Component } public function refreshServers() { - ContainerStatusJob::dispatchSync($this->resource->destination->server); + GetContainersStatus::run($this->resource->destination->server); + // ContainerStatusJob::dispatchSync($this->resource->destination->server); $this->loadData(); $this->dispatch('refresh'); ApplicationStatusChanged::dispatch(data_get($this->resource, 'environment.project.team.id')); diff --git a/app/Livewire/Server/Proxy/Status.php b/app/Livewire/Server/Proxy/Status.php index bd0ffe431..fbc16fde4 100644 --- a/app/Livewire/Server/Proxy/Status.php +++ b/app/Livewire/Server/Proxy/Status.php @@ -2,6 +2,7 @@ namespace App\Livewire\Server\Proxy; +use App\Actions\Docker\GetContainersStatus; use App\Actions\Proxy\CheckProxy; use App\Jobs\ContainerStatusJob; use App\Models\Server; @@ -49,7 +50,8 @@ class Status extends Component public function getProxyStatus() { try { - dispatch_sync(new ContainerStatusJob($this->server)); + GetContainersStatus::run($this->server); + // dispatch_sync(new ContainerStatusJob($this->server)); $this->dispatch('proxyStatusUpdated'); } catch (\Throwable $e) { return handleError($e, $this); diff --git a/app/Notifications/Server/Revived.php b/app/Notifications/Server/Revived.php index c670ded9a..36775976b 100644 --- a/app/Notifications/Server/Revived.php +++ b/app/Notifications/Server/Revived.php @@ -2,6 +2,7 @@ namespace App\Notifications\Server; +use App\Actions\Docker\GetContainersStatus; use App\Jobs\ContainerStatusJob; use App\Models\Server; use Illuminate\Bus\Queueable; @@ -22,7 +23,8 @@ class Revived extends Notification implements ShouldQueue if ($this->server->unreachable_notification_sent === false) { return; } - dispatch(new ContainerStatusJob($server)); + GetContainersStatus::dispatch($server); + // dispatch(new ContainerStatusJob($server)); } public function via(object $notifiable): array diff --git a/database/migrations/2024_05_07_124019_add_server_metrics.php b/database/migrations/2024_05_07_124019_add_server_metrics.php new file mode 100644 index 000000000..40c74850b --- /dev/null +++ b/database/migrations/2024_05_07_124019_add_server_metrics.php @@ -0,0 +1,28 @@ +boolean('is_metrics_enabled')->default(true); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('servers', function (Blueprint $table) { + $table->dropColumn('is_metrics_enabled'); + }); + } +}; diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 91e90b989..2594822ce 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -19,6 +19,12 @@ services: PUSHER_APP_SECRET: "${PUSHER_APP_SECRET:-coolify}" volumes: - .:/var/www/html/:cached + sentinel: + ports: + - "127.0.0.1:8888:8888" + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - .:/var/www/html/:cached postgres: pull_policy: always ports: diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index f3dda9748..57dedea92 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -103,6 +103,15 @@ services: condition: service_healthy redis: condition: service_healthy + sentinel: + image: "ghcr.io/coollabsio/sentinel:${LATEST_IMAGE:-latest}" + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - /data/coolify/metrics:/var/www/html/storage/app/metrics + ports: + - "127.0.0.1:8888:8888" + healthcheck: + test: curl --fail http://127.0.0.1:8888/api/health || exit 1 postgres: volumes: - coolify-db:/var/lib/postgresql/data diff --git a/docker-compose.yml b/docker-compose.yml index 6f1cf8e08..f292cc880 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,12 @@ services: depends_on: - postgres - redis - + sentinel: + image: "ghcr.io/coollabsio/sentinel:latest" + container_name: coolify-sentinel + restart: always + networks: + - coolify postgres: image: postgres:15-alpine container_name: coolify-db