feat: experimental sentinel
This commit is contained in:
parent
2b422a542a
commit
f6f959a897
655
app/Actions/Docker/GetContainersStatus.php
Normal file
655
app/Actions/Docker/GetContainersStatus.php
Normal file
@ -0,0 +1,655 @@
|
||||
<?php
|
||||
|
||||
namespace App\Actions\Docker;
|
||||
|
||||
use App\Actions\Database\StartDatabaseProxy;
|
||||
use App\Actions\Proxy\CheckProxy;
|
||||
use App\Actions\Proxy\StartProxy;
|
||||
use App\Actions\Shared\ComplexStatusCheck;
|
||||
use App\Models\Application;
|
||||
use App\Models\ApplicationPreview;
|
||||
use App\Models\Server;
|
||||
use App\Models\ServiceDatabase;
|
||||
use App\Notifications\Container\ContainerRestarted;
|
||||
use App\Notifications\Container\ContainerStopped;
|
||||
use Illuminate\Support\Arr;
|
||||
use Lorisleiva\Actions\Concerns\AsAction;
|
||||
|
||||
class GetContainersStatus
|
||||
{
|
||||
use AsAction;
|
||||
public $applications;
|
||||
public function __construct(public Server $server)
|
||||
{
|
||||
if (isDev()) {
|
||||
$this->server = Server::find(0);
|
||||
}
|
||||
}
|
||||
public function handle()
|
||||
{
|
||||
if (!$this->server->isFunctional()) {
|
||||
return 'Server is not ready.';
|
||||
};
|
||||
$this->applications = $this->server->applications();
|
||||
$skip_these_applications = collect([]);
|
||||
foreach ($this->applications as $application) {
|
||||
if ($application->additional_servers->count() > 0) {
|
||||
$skip_these_applications->push($application);
|
||||
ComplexStatusCheck::run($application);
|
||||
$this->applications = $this->applications->filter(function ($value, $key) use ($application) {
|
||||
return $value->id !== $application->id;
|
||||
});
|
||||
}
|
||||
}
|
||||
$this->applications = $this->applications->filter(function ($value, $key) use ($skip_these_applications) {
|
||||
return !$skip_these_applications->pluck('id')->contains($value->id);
|
||||
});
|
||||
if ($this->server->isSwarm()) {
|
||||
$this->old_way();
|
||||
} else {
|
||||
if (!$this->server->is_metrics_enabled) {
|
||||
$this->old_way();
|
||||
return;
|
||||
}
|
||||
$sentinel_found = instant_remote_process(["docker inspect coolify-sentinel"], $this->server, false);
|
||||
if ($sentinel_found) {
|
||||
raY('Sentinel');
|
||||
$this->sentinel();
|
||||
} else {
|
||||
raY('Old way');
|
||||
$this->old_way();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private function sentinel()
|
||||
{
|
||||
try {
|
||||
$containers = instant_remote_process(['docker exec coolify-sentinel sh -c "curl http://127.0.0.1:8888/api/containers"'], $this->server, false);
|
||||
if (is_null($containers)) {
|
||||
return;
|
||||
}
|
||||
$containers = data_get(json_decode($containers, true), 'containers', []);
|
||||
$containers = collect($containers);
|
||||
|
||||
$databases = $this->server->databases();
|
||||
$services = $this->server->services()->get();
|
||||
$previews = $this->server->previews();
|
||||
$foundApplications = [];
|
||||
$foundApplicationPreviews = [];
|
||||
$foundDatabases = [];
|
||||
$foundServices = [];
|
||||
|
||||
foreach ($containers as $container) {
|
||||
$labels = Arr::undot(data_get($container, 'labels'));
|
||||
$containerStatus = data_get($container, 'state');
|
||||
$containerHealth = data_get($container, 'health_status', 'unhealthy');
|
||||
$containerStatus = "$containerStatus ($containerHealth)";
|
||||
$applicationId = data_get($labels, 'coolify.applicationId');
|
||||
if ($applicationId) {
|
||||
$pullRequestId = data_get($labels, 'coolify.pullRequestId');
|
||||
if ($pullRequestId) {
|
||||
if (str($applicationId)->contains('-')) {
|
||||
$applicationId = str($applicationId)->before('-');
|
||||
}
|
||||
$preview = ApplicationPreview::where('application_id', $applicationId)->where('pull_request_id', $pullRequestId)->first();
|
||||
if ($preview) {
|
||||
$foundApplicationPreviews[] = $preview->id;
|
||||
$statusFromDb = $preview->status;
|
||||
if ($statusFromDb !== $containerStatus) {
|
||||
$preview->update(['status' => $containerStatus]);
|
||||
}
|
||||
} else {
|
||||
//Notify user that this container should not be there.
|
||||
}
|
||||
} else {
|
||||
$application = $this->applications->where('id', $applicationId)->first();
|
||||
if ($application) {
|
||||
$foundApplications[] = $application->id;
|
||||
$statusFromDb = $application->status;
|
||||
ray($statusFromDb, $containerStatus);
|
||||
if ($statusFromDb !== $containerStatus) {
|
||||
$application->update(['status' => $containerStatus]);
|
||||
}
|
||||
} else {
|
||||
//Notify user that this container should not be there.
|
||||
}
|
||||
}
|
||||
} else {
|
||||
$uuid = data_get($labels, 'com.docker.compose.service');
|
||||
$type = data_get($labels, 'coolify.type');
|
||||
|
||||
if ($uuid) {
|
||||
if ($type === 'service') {
|
||||
$database_id = data_get($labels, 'coolify.service.subId');
|
||||
if ($database_id) {
|
||||
$service_db = ServiceDatabase::where('id', $database_id)->first();
|
||||
if ($service_db) {
|
||||
$uuid = $service_db->service->uuid;
|
||||
$isPublic = data_get($service_db, 'is_public');
|
||||
if ($isPublic) {
|
||||
$foundTcpProxy = $containers->filter(function ($value, $key) use ($uuid) {
|
||||
if ($this->server->isSwarm()) {
|
||||
return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid";
|
||||
} else {
|
||||
return data_get($value, 'Name') === "/$uuid-proxy";
|
||||
}
|
||||
})->first();
|
||||
if (!$foundTcpProxy) {
|
||||
StartDatabaseProxy::run($service_db);
|
||||
// $this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$service_db->service->name}", $this->server));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
$database = $databases->where('uuid', $uuid)->first();
|
||||
if ($database) {
|
||||
$isPublic = data_get($database, 'is_public');
|
||||
$foundDatabases[] = $database->id;
|
||||
$statusFromDb = $database->status;
|
||||
if ($statusFromDb !== $containerStatus) {
|
||||
$database->update(['status' => $containerStatus]);
|
||||
}
|
||||
if ($isPublic) {
|
||||
$foundTcpProxy = $containers->filter(function ($value, $key) use ($uuid) {
|
||||
if ($this->server->isSwarm()) {
|
||||
return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid";
|
||||
} else {
|
||||
return data_get($value, 'Name') === "/$uuid-proxy";
|
||||
}
|
||||
})->first();
|
||||
if (!$foundTcpProxy) {
|
||||
StartDatabaseProxy::run($database);
|
||||
$this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$database->name}", $this->server));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Notify user that this container should not be there.
|
||||
}
|
||||
}
|
||||
}
|
||||
if (data_get($container, 'Name') === '/coolify-db') {
|
||||
$foundDatabases[] = 0;
|
||||
}
|
||||
}
|
||||
$serviceLabelId = data_get($labels, 'coolify.serviceId');
|
||||
if ($serviceLabelId) {
|
||||
$subType = data_get($labels, 'coolify.service.subType');
|
||||
$subId = data_get($labels, 'coolify.service.subId');
|
||||
$service = $services->where('id', $serviceLabelId)->first();
|
||||
if (!$service) {
|
||||
continue;
|
||||
}
|
||||
if ($subType === 'application') {
|
||||
$service = $service->applications()->where('id', $subId)->first();
|
||||
} else {
|
||||
$service = $service->databases()->where('id', $subId)->first();
|
||||
}
|
||||
if ($service) {
|
||||
$foundServices[] = "$service->id-$service->name";
|
||||
$statusFromDb = $service->status;
|
||||
if ($statusFromDb !== $containerStatus) {
|
||||
// ray('Updating status: ' . $containerStatus);
|
||||
$service->update(['status' => $containerStatus]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
$exitedServices = collect([]);
|
||||
foreach ($services as $service) {
|
||||
$apps = $service->applications()->get();
|
||||
$dbs = $service->databases()->get();
|
||||
foreach ($apps as $app) {
|
||||
if (in_array("$app->id-$app->name", $foundServices)) {
|
||||
continue;
|
||||
} else {
|
||||
$exitedServices->push($app);
|
||||
}
|
||||
}
|
||||
foreach ($dbs as $db) {
|
||||
if (in_array("$db->id-$db->name", $foundServices)) {
|
||||
continue;
|
||||
} else {
|
||||
$exitedServices->push($db);
|
||||
}
|
||||
}
|
||||
}
|
||||
$exitedServices = $exitedServices->unique('id');
|
||||
foreach ($exitedServices as $exitedService) {
|
||||
if (str($exitedService->status)->startsWith('exited')) {
|
||||
continue;
|
||||
}
|
||||
$name = data_get($exitedService, 'name');
|
||||
$fqdn = data_get($exitedService, 'fqdn');
|
||||
$containerName = $name ? "$name, available at $fqdn" : $fqdn;
|
||||
$projectUuid = data_get($service, 'environment.project.uuid');
|
||||
$serviceUuid = data_get($service, 'uuid');
|
||||
$environmentName = data_get($service, 'environment.name');
|
||||
|
||||
if ($projectUuid && $serviceUuid && $environmentName) {
|
||||
$url = base_url() . '/project/' . $projectUuid . "/" . $environmentName . "/service/" . $serviceUuid;
|
||||
} else {
|
||||
$url = null;
|
||||
}
|
||||
$this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url));
|
||||
$exitedService->update(['status' => 'exited']);
|
||||
}
|
||||
|
||||
$notRunningApplications = $this->applications->pluck('id')->diff($foundApplications);
|
||||
foreach ($notRunningApplications as $applicationId) {
|
||||
$application = $this->applications->where('id', $applicationId)->first();
|
||||
if (str($application->status)->startsWith('exited')) {
|
||||
continue;
|
||||
}
|
||||
$application->update(['status' => 'exited']);
|
||||
|
||||
$name = data_get($application, 'name');
|
||||
$fqdn = data_get($application, 'fqdn');
|
||||
|
||||
$containerName = $name ? "$name ($fqdn)" : $fqdn;
|
||||
|
||||
$projectUuid = data_get($application, 'environment.project.uuid');
|
||||
$applicationUuid = data_get($application, 'uuid');
|
||||
$environment = data_get($application, 'environment.name');
|
||||
|
||||
if ($projectUuid && $applicationUuid && $environment) {
|
||||
$url = base_url() . '/project/' . $projectUuid . "/" . $environment . "/application/" . $applicationUuid;
|
||||
} else {
|
||||
$url = null;
|
||||
}
|
||||
|
||||
$this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url));
|
||||
}
|
||||
$notRunningApplicationPreviews = $previews->pluck('id')->diff($foundApplicationPreviews);
|
||||
foreach ($notRunningApplicationPreviews as $previewId) {
|
||||
$preview = $previews->where('id', $previewId)->first();
|
||||
if (str($preview->status)->startsWith('exited')) {
|
||||
continue;
|
||||
}
|
||||
$preview->update(['status' => 'exited']);
|
||||
|
||||
$name = data_get($preview, 'name');
|
||||
$fqdn = data_get($preview, 'fqdn');
|
||||
|
||||
$containerName = $name ? "$name ($fqdn)" : $fqdn;
|
||||
|
||||
$projectUuid = data_get($preview, 'application.environment.project.uuid');
|
||||
$environmentName = data_get($preview, 'application.environment.name');
|
||||
$applicationUuid = data_get($preview, 'application.uuid');
|
||||
|
||||
if ($projectUuid && $applicationUuid && $environmentName) {
|
||||
$url = base_url() . '/project/' . $projectUuid . "/" . $environmentName . "/application/" . $applicationUuid;
|
||||
} else {
|
||||
$url = null;
|
||||
}
|
||||
|
||||
$this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url));
|
||||
}
|
||||
$notRunningDatabases = $databases->pluck('id')->diff($foundDatabases);
|
||||
foreach ($notRunningDatabases as $database) {
|
||||
$database = $databases->where('id', $database)->first();
|
||||
if (str($database->status)->startsWith('exited')) {
|
||||
continue;
|
||||
}
|
||||
$database->update(['status' => 'exited']);
|
||||
|
||||
$name = data_get($database, 'name');
|
||||
$fqdn = data_get($database, 'fqdn');
|
||||
|
||||
$containerName = $name;
|
||||
|
||||
$projectUuid = data_get($database, 'environment.project.uuid');
|
||||
$environmentName = data_get($database, 'environment.name');
|
||||
$databaseUuid = data_get($database, 'uuid');
|
||||
|
||||
if ($projectUuid && $databaseUuid && $environmentName) {
|
||||
$url = base_url() . '/project/' . $projectUuid . "/" . $environmentName . "/database/" . $databaseUuid;
|
||||
} else {
|
||||
$url = null;
|
||||
}
|
||||
$this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url));
|
||||
}
|
||||
|
||||
// Check if proxy is running
|
||||
$this->server->proxyType();
|
||||
$foundProxyContainer = $containers->filter(function ($value, $key) {
|
||||
if ($this->server->isSwarm()) {
|
||||
return data_get($value, 'Spec.Name') === 'coolify-proxy_traefik';
|
||||
} else {
|
||||
return data_get($value, 'Name') === '/coolify-proxy';
|
||||
}
|
||||
})->first();
|
||||
if (!$foundProxyContainer) {
|
||||
try {
|
||||
$shouldStart = CheckProxy::run($this->server);
|
||||
if ($shouldStart) {
|
||||
StartProxy::run($this->server, false);
|
||||
$this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server));
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
ray($e);
|
||||
}
|
||||
} else {
|
||||
$this->server->proxy->status = data_get($foundProxyContainer, 'State.Status');
|
||||
$this->server->save();
|
||||
$connectProxyToDockerNetworks = connectProxyToNetworks($this->server);
|
||||
instant_remote_process($connectProxyToDockerNetworks, $this->server, false);
|
||||
}
|
||||
} catch (\Exception $e) {
|
||||
send_internal_notification("ContainerStatusJob failed on ({$this->server->id}) with: " . $e->getMessage());
|
||||
ray($e->getMessage());
|
||||
return handleError($e);
|
||||
}
|
||||
}
|
||||
private function old_way()
|
||||
{
|
||||
if ($this->server->isSwarm()) {
|
||||
$containers = instant_remote_process(["docker service inspect $(docker service ls -q) --format '{{json .}}'"], $this->server, false);
|
||||
$containerReplicates = instant_remote_process(["docker service ls --format '{{json .}}'"], $this->server, false);
|
||||
} else {
|
||||
// Precheck for containers
|
||||
$containers = instant_remote_process(["docker container ls -q"], $this->server, false);
|
||||
if (!$containers) {
|
||||
return;
|
||||
}
|
||||
$containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this->server, false);
|
||||
$containerReplicates = null;
|
||||
}
|
||||
if (is_null($containers)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$containers = format_docker_command_output_to_json($containers);
|
||||
if ($containerReplicates) {
|
||||
$containerReplicates = format_docker_command_output_to_json($containerReplicates);
|
||||
foreach ($containerReplicates as $containerReplica) {
|
||||
$name = data_get($containerReplica, 'Name');
|
||||
$containers = $containers->map(function ($container) use ($name, $containerReplica) {
|
||||
if (data_get($container, 'Spec.Name') === $name) {
|
||||
$replicas = data_get($containerReplica, 'Replicas');
|
||||
$running = str($replicas)->explode('/')[0];
|
||||
$total = str($replicas)->explode('/')[1];
|
||||
if ($running === $total) {
|
||||
data_set($container, 'State.Status', 'running');
|
||||
data_set($container, 'State.Health.Status', 'healthy');
|
||||
} else {
|
||||
data_set($container, 'State.Status', 'starting');
|
||||
data_set($container, 'State.Health.Status', 'unhealthy');
|
||||
}
|
||||
}
|
||||
return $container;
|
||||
});
|
||||
}
|
||||
}
|
||||
$databases = $this->server->databases();
|
||||
$services = $this->server->services()->get();
|
||||
$previews = $this->server->previews();
|
||||
$foundApplications = [];
|
||||
$foundApplicationPreviews = [];
|
||||
$foundDatabases = [];
|
||||
$foundServices = [];
|
||||
|
||||
foreach ($containers as $container) {
|
||||
if ($this->server->isSwarm()) {
|
||||
$labels = data_get($container, 'Spec.Labels');
|
||||
$uuid = data_get($labels, 'coolify.name');
|
||||
} else {
|
||||
$labels = data_get($container, 'Config.Labels');
|
||||
}
|
||||
$containerStatus = data_get($container, 'State.Status');
|
||||
$containerHealth = data_get($container, 'State.Health.Status', 'unhealthy');
|
||||
$containerStatus = "$containerStatus ($containerHealth)";
|
||||
$labels = Arr::undot(format_docker_labels_to_json($labels));
|
||||
$applicationId = data_get($labels, 'coolify.applicationId');
|
||||
if ($applicationId) {
|
||||
$pullRequestId = data_get($labels, 'coolify.pullRequestId');
|
||||
if ($pullRequestId) {
|
||||
if (str($applicationId)->contains('-')) {
|
||||
$applicationId = str($applicationId)->before('-');
|
||||
}
|
||||
$preview = ApplicationPreview::where('application_id', $applicationId)->where('pull_request_id', $pullRequestId)->first();
|
||||
if ($preview) {
|
||||
$foundApplicationPreviews[] = $preview->id;
|
||||
$statusFromDb = $preview->status;
|
||||
if ($statusFromDb !== $containerStatus) {
|
||||
$preview->update(['status' => $containerStatus]);
|
||||
}
|
||||
} else {
|
||||
//Notify user that this container should not be there.
|
||||
}
|
||||
} else {
|
||||
$application = $this->applications->where('id', $applicationId)->first();
|
||||
if ($application) {
|
||||
$foundApplications[] = $application->id;
|
||||
$statusFromDb = $application->status;
|
||||
if ($statusFromDb !== $containerStatus) {
|
||||
$application->update(['status' => $containerStatus]);
|
||||
}
|
||||
} else {
|
||||
//Notify user that this container should not be there.
|
||||
}
|
||||
}
|
||||
} else {
|
||||
$uuid = data_get($labels, 'com.docker.compose.service');
|
||||
$type = data_get($labels, 'coolify.type');
|
||||
|
||||
if ($uuid) {
|
||||
if ($type === 'service') {
|
||||
$database_id = data_get($labels, 'coolify.service.subId');
|
||||
if ($database_id) {
|
||||
$service_db = ServiceDatabase::where('id', $database_id)->first();
|
||||
if ($service_db) {
|
||||
$uuid = $service_db->service->uuid;
|
||||
$isPublic = data_get($service_db, 'is_public');
|
||||
if ($isPublic) {
|
||||
$foundTcpProxy = $containers->filter(function ($value, $key) use ($uuid) {
|
||||
if ($this->server->isSwarm()) {
|
||||
return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid";
|
||||
} else {
|
||||
return data_get($value, 'Name') === "/$uuid-proxy";
|
||||
}
|
||||
})->first();
|
||||
if (!$foundTcpProxy) {
|
||||
StartDatabaseProxy::run($service_db);
|
||||
// $this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$service_db->service->name}", $this->server));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
$database = $databases->where('uuid', $uuid)->first();
|
||||
if ($database) {
|
||||
$isPublic = data_get($database, 'is_public');
|
||||
$foundDatabases[] = $database->id;
|
||||
$statusFromDb = $database->status;
|
||||
if ($statusFromDb !== $containerStatus) {
|
||||
$database->update(['status' => $containerStatus]);
|
||||
}
|
||||
if ($isPublic) {
|
||||
$foundTcpProxy = $containers->filter(function ($value, $key) use ($uuid) {
|
||||
if ($this->server->isSwarm()) {
|
||||
return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid";
|
||||
} else {
|
||||
return data_get($value, 'Name') === "/$uuid-proxy";
|
||||
}
|
||||
})->first();
|
||||
if (!$foundTcpProxy) {
|
||||
StartDatabaseProxy::run($database);
|
||||
$this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$database->name}", $this->server));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Notify user that this container should not be there.
|
||||
}
|
||||
}
|
||||
}
|
||||
if (data_get($container, 'Name') === '/coolify-db') {
|
||||
$foundDatabases[] = 0;
|
||||
}
|
||||
}
|
||||
$serviceLabelId = data_get($labels, 'coolify.serviceId');
|
||||
if ($serviceLabelId) {
|
||||
$subType = data_get($labels, 'coolify.service.subType');
|
||||
$subId = data_get($labels, 'coolify.service.subId');
|
||||
$service = $services->where('id', $serviceLabelId)->first();
|
||||
if (!$service) {
|
||||
continue;
|
||||
}
|
||||
if ($subType === 'application') {
|
||||
$service = $service->applications()->where('id', $subId)->first();
|
||||
} else {
|
||||
$service = $service->databases()->where('id', $subId)->first();
|
||||
}
|
||||
if ($service) {
|
||||
$foundServices[] = "$service->id-$service->name";
|
||||
$statusFromDb = $service->status;
|
||||
if ($statusFromDb !== $containerStatus) {
|
||||
// ray('Updating status: ' . $containerStatus);
|
||||
$service->update(['status' => $containerStatus]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
$exitedServices = collect([]);
|
||||
foreach ($services as $service) {
|
||||
$apps = $service->applications()->get();
|
||||
$dbs = $service->databases()->get();
|
||||
foreach ($apps as $app) {
|
||||
if (in_array("$app->id-$app->name", $foundServices)) {
|
||||
continue;
|
||||
} else {
|
||||
$exitedServices->push($app);
|
||||
}
|
||||
}
|
||||
foreach ($dbs as $db) {
|
||||
if (in_array("$db->id-$db->name", $foundServices)) {
|
||||
continue;
|
||||
} else {
|
||||
$exitedServices->push($db);
|
||||
}
|
||||
}
|
||||
}
|
||||
$exitedServices = $exitedServices->unique('id');
|
||||
foreach ($exitedServices as $exitedService) {
|
||||
if (str($exitedService->status)->startsWith('exited')) {
|
||||
continue;
|
||||
}
|
||||
$name = data_get($exitedService, 'name');
|
||||
$fqdn = data_get($exitedService, 'fqdn');
|
||||
$containerName = $name ? "$name, available at $fqdn" : $fqdn;
|
||||
$projectUuid = data_get($service, 'environment.project.uuid');
|
||||
$serviceUuid = data_get($service, 'uuid');
|
||||
$environmentName = data_get($service, 'environment.name');
|
||||
|
||||
if ($projectUuid && $serviceUuid && $environmentName) {
|
||||
$url = base_url() . '/project/' . $projectUuid . "/" . $environmentName . "/service/" . $serviceUuid;
|
||||
} else {
|
||||
$url = null;
|
||||
}
|
||||
$this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url));
|
||||
$exitedService->update(['status' => 'exited']);
|
||||
}
|
||||
|
||||
$notRunningApplications = $this->applications->pluck('id')->diff($foundApplications);
|
||||
foreach ($notRunningApplications as $applicationId) {
|
||||
$application = $this->applications->where('id', $applicationId)->first();
|
||||
if (str($application->status)->startsWith('exited')) {
|
||||
continue;
|
||||
}
|
||||
$application->update(['status' => 'exited']);
|
||||
|
||||
$name = data_get($application, 'name');
|
||||
$fqdn = data_get($application, 'fqdn');
|
||||
|
||||
$containerName = $name ? "$name ($fqdn)" : $fqdn;
|
||||
|
||||
$projectUuid = data_get($application, 'environment.project.uuid');
|
||||
$applicationUuid = data_get($application, 'uuid');
|
||||
$environment = data_get($application, 'environment.name');
|
||||
|
||||
if ($projectUuid && $applicationUuid && $environment) {
|
||||
$url = base_url() . '/project/' . $projectUuid . "/" . $environment . "/application/" . $applicationUuid;
|
||||
} else {
|
||||
$url = null;
|
||||
}
|
||||
|
||||
$this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url));
|
||||
}
|
||||
$notRunningApplicationPreviews = $previews->pluck('id')->diff($foundApplicationPreviews);
|
||||
foreach ($notRunningApplicationPreviews as $previewId) {
|
||||
$preview = $previews->where('id', $previewId)->first();
|
||||
if (str($preview->status)->startsWith('exited')) {
|
||||
continue;
|
||||
}
|
||||
$preview->update(['status' => 'exited']);
|
||||
|
||||
$name = data_get($preview, 'name');
|
||||
$fqdn = data_get($preview, 'fqdn');
|
||||
|
||||
$containerName = $name ? "$name ($fqdn)" : $fqdn;
|
||||
|
||||
$projectUuid = data_get($preview, 'application.environment.project.uuid');
|
||||
$environmentName = data_get($preview, 'application.environment.name');
|
||||
$applicationUuid = data_get($preview, 'application.uuid');
|
||||
|
||||
if ($projectUuid && $applicationUuid && $environmentName) {
|
||||
$url = base_url() . '/project/' . $projectUuid . "/" . $environmentName . "/application/" . $applicationUuid;
|
||||
} else {
|
||||
$url = null;
|
||||
}
|
||||
|
||||
$this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url));
|
||||
}
|
||||
$notRunningDatabases = $databases->pluck('id')->diff($foundDatabases);
|
||||
foreach ($notRunningDatabases as $database) {
|
||||
$database = $databases->where('id', $database)->first();
|
||||
if (str($database->status)->startsWith('exited')) {
|
||||
continue;
|
||||
}
|
||||
$database->update(['status' => 'exited']);
|
||||
|
||||
$name = data_get($database, 'name');
|
||||
$fqdn = data_get($database, 'fqdn');
|
||||
|
||||
$containerName = $name;
|
||||
|
||||
$projectUuid = data_get($database, 'environment.project.uuid');
|
||||
$environmentName = data_get($database, 'environment.name');
|
||||
$databaseUuid = data_get($database, 'uuid');
|
||||
|
||||
if ($projectUuid && $databaseUuid && $environmentName) {
|
||||
$url = base_url() . '/project/' . $projectUuid . "/" . $environmentName . "/database/" . $databaseUuid;
|
||||
} else {
|
||||
$url = null;
|
||||
}
|
||||
$this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url));
|
||||
}
|
||||
|
||||
// Check if proxy is running
|
||||
$this->server->proxyType();
|
||||
$foundProxyContainer = $containers->filter(function ($value, $key) {
|
||||
if ($this->server->isSwarm()) {
|
||||
return data_get($value, 'Spec.Name') === 'coolify-proxy_traefik';
|
||||
} else {
|
||||
return data_get($value, 'Name') === '/coolify-proxy';
|
||||
}
|
||||
})->first();
|
||||
if (!$foundProxyContainer) {
|
||||
try {
|
||||
$shouldStart = CheckProxy::run($this->server);
|
||||
if ($shouldStart) {
|
||||
StartProxy::run($this->server, false);
|
||||
$this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server));
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
ray($e);
|
||||
}
|
||||
} else {
|
||||
$this->server->proxy->status = data_get($foundProxyContainer, 'State.Status');
|
||||
$this->server->save();
|
||||
$connectProxyToDockerNetworks = connectProxyToNetworks($this->server);
|
||||
instant_remote_process($connectProxyToDockerNetworks, $this->server, false);
|
||||
}
|
||||
}
|
||||
}
|
@ -2,6 +2,7 @@
|
||||
|
||||
namespace App\Jobs;
|
||||
|
||||
use App\Actions\Docker\GetContainersStatus;
|
||||
use App\Enums\ApplicationDeploymentStatus;
|
||||
use App\Enums\ProcessStatus;
|
||||
use App\Events\ApplicationStatusChanged;
|
||||
@ -302,7 +303,8 @@ private function post_deployment()
|
||||
{
|
||||
|
||||
if ($this->server->isProxyShouldRun()) {
|
||||
dispatch(new ContainerStatusJob($this->server));
|
||||
GetContainersStatus::dispatch($this->server);
|
||||
// dispatch(new ContainerStatusJob($this->server));
|
||||
}
|
||||
$this->next(ApplicationDeploymentStatus::FINISHED->value);
|
||||
if ($this->pull_request_id !== 0) {
|
||||
|
@ -2,15 +2,8 @@
|
||||
|
||||
namespace App\Jobs;
|
||||
|
||||
use App\Actions\Database\StartDatabaseProxy;
|
||||
use App\Actions\Proxy\CheckProxy;
|
||||
use App\Actions\Proxy\StartProxy;
|
||||
use App\Actions\Shared\ComplexStatusCheck;
|
||||
use App\Models\ApplicationPreview;
|
||||
use App\Actions\Docker\GetContainersStatus;
|
||||
use App\Models\Server;
|
||||
use App\Models\ServiceDatabase;
|
||||
use App\Notifications\Container\ContainerRestarted;
|
||||
use App\Notifications\Container\ContainerStopped;
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
@ -18,7 +11,6 @@
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\Middleware\WithoutOverlapping;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Arr;
|
||||
|
||||
class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted
|
||||
{
|
||||
@ -44,335 +36,337 @@ public function uniqueId(): int
|
||||
|
||||
public function handle()
|
||||
{
|
||||
if (!$this->server->isFunctional()) {
|
||||
return 'Server is not ready.';
|
||||
};
|
||||
$applications = $this->server->applications();
|
||||
$skip_these_applications = collect([]);
|
||||
foreach ($applications as $application) {
|
||||
if ($application->additional_servers->count() > 0) {
|
||||
$skip_these_applications->push($application);
|
||||
ComplexStatusCheck::run($application);
|
||||
$applications = $applications->filter(function ($value, $key) use ($application) {
|
||||
return $value->id !== $application->id;
|
||||
});
|
||||
}
|
||||
}
|
||||
$applications = $applications->filter(function ($value, $key) use ($skip_these_applications) {
|
||||
return !$skip_these_applications->pluck('id')->contains($value->id);
|
||||
});
|
||||
try {
|
||||
if ($this->server->isSwarm()) {
|
||||
$containers = instant_remote_process(["docker service inspect $(docker service ls -q) --format '{{json .}}'"], $this->server, false);
|
||||
$containerReplicates = instant_remote_process(["docker service ls --format '{{json .}}'"], $this->server, false);
|
||||
} else {
|
||||
// Precheck for containers
|
||||
$containers = instant_remote_process(["docker container ls -q"], $this->server, false);
|
||||
if (!$containers) {
|
||||
return;
|
||||
}
|
||||
$containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this->server, false);
|
||||
$containerReplicates = null;
|
||||
}
|
||||
if (is_null($containers)) {
|
||||
return;
|
||||
}
|
||||
GetContainersStatus::run();
|
||||
return;
|
||||
// if (!$this->server->isFunctional()) {
|
||||
// return 'Server is not ready.';
|
||||
// };
|
||||
// $applications = $this->server->applications();
|
||||
// $skip_these_applications = collect([]);
|
||||
// foreach ($applications as $application) {
|
||||
// if ($application->additional_servers->count() > 0) {
|
||||
// $skip_these_applications->push($application);
|
||||
// ComplexStatusCheck::run($application);
|
||||
// $applications = $applications->filter(function ($value, $key) use ($application) {
|
||||
// return $value->id !== $application->id;
|
||||
// });
|
||||
// }
|
||||
// }
|
||||
// $applications = $applications->filter(function ($value, $key) use ($skip_these_applications) {
|
||||
// return !$skip_these_applications->pluck('id')->contains($value->id);
|
||||
// });
|
||||
// try {
|
||||
// if ($this->server->isSwarm()) {
|
||||
// $containers = instant_remote_process(["docker service inspect $(docker service ls -q) --format '{{json .}}'"], $this->server, false);
|
||||
// $containerReplicates = instant_remote_process(["docker service ls --format '{{json .}}'"], $this->server, false);
|
||||
// } else {
|
||||
// // Precheck for containers
|
||||
// $containers = instant_remote_process(["docker container ls -q"], $this->server, false);
|
||||
// if (!$containers) {
|
||||
// return;
|
||||
// }
|
||||
// $containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this->server, false);
|
||||
// $containerReplicates = null;
|
||||
// }
|
||||
// if (is_null($containers)) {
|
||||
// return;
|
||||
// }
|
||||
|
||||
$containers = format_docker_command_output_to_json($containers);
|
||||
if ($containerReplicates) {
|
||||
$containerReplicates = format_docker_command_output_to_json($containerReplicates);
|
||||
foreach ($containerReplicates as $containerReplica) {
|
||||
$name = data_get($containerReplica, 'Name');
|
||||
$containers = $containers->map(function ($container) use ($name, $containerReplica) {
|
||||
if (data_get($container, 'Spec.Name') === $name) {
|
||||
$replicas = data_get($containerReplica, 'Replicas');
|
||||
$running = str($replicas)->explode('/')[0];
|
||||
$total = str($replicas)->explode('/')[1];
|
||||
if ($running === $total) {
|
||||
data_set($container, 'State.Status', 'running');
|
||||
data_set($container, 'State.Health.Status', 'healthy');
|
||||
} else {
|
||||
data_set($container, 'State.Status', 'starting');
|
||||
data_set($container, 'State.Health.Status', 'unhealthy');
|
||||
}
|
||||
}
|
||||
return $container;
|
||||
});
|
||||
}
|
||||
}
|
||||
$databases = $this->server->databases();
|
||||
$services = $this->server->services()->get();
|
||||
$previews = $this->server->previews();
|
||||
$foundApplications = [];
|
||||
$foundApplicationPreviews = [];
|
||||
$foundDatabases = [];
|
||||
$foundServices = [];
|
||||
// $containers = format_docker_command_output_to_json($containers);
|
||||
// if ($containerReplicates) {
|
||||
// $containerReplicates = format_docker_command_output_to_json($containerReplicates);
|
||||
// foreach ($containerReplicates as $containerReplica) {
|
||||
// $name = data_get($containerReplica, 'Name');
|
||||
// $containers = $containers->map(function ($container) use ($name, $containerReplica) {
|
||||
// if (data_get($container, 'Spec.Name') === $name) {
|
||||
// $replicas = data_get($containerReplica, 'Replicas');
|
||||
// $running = str($replicas)->explode('/')[0];
|
||||
// $total = str($replicas)->explode('/')[1];
|
||||
// if ($running === $total) {
|
||||
// data_set($container, 'State.Status', 'running');
|
||||
// data_set($container, 'State.Health.Status', 'healthy');
|
||||
// } else {
|
||||
// data_set($container, 'State.Status', 'starting');
|
||||
// data_set($container, 'State.Health.Status', 'unhealthy');
|
||||
// }
|
||||
// }
|
||||
// return $container;
|
||||
// });
|
||||
// }
|
||||
// }
|
||||
// $databases = $this->server->databases();
|
||||
// $services = $this->server->services()->get();
|
||||
// $previews = $this->server->previews();
|
||||
// $foundApplications = [];
|
||||
// $foundApplicationPreviews = [];
|
||||
// $foundDatabases = [];
|
||||
// $foundServices = [];
|
||||
|
||||
foreach ($containers as $container) {
|
||||
if ($this->server->isSwarm()) {
|
||||
$labels = data_get($container, 'Spec.Labels');
|
||||
$uuid = data_get($labels, 'coolify.name');
|
||||
} else {
|
||||
$labels = data_get($container, 'Config.Labels');
|
||||
}
|
||||
$containerStatus = data_get($container, 'State.Status');
|
||||
$containerHealth = data_get($container, 'State.Health.Status', 'unhealthy');
|
||||
$containerStatus = "$containerStatus ($containerHealth)";
|
||||
$labels = Arr::undot(format_docker_labels_to_json($labels));
|
||||
$applicationId = data_get($labels, 'coolify.applicationId');
|
||||
if ($applicationId) {
|
||||
$pullRequestId = data_get($labels, 'coolify.pullRequestId');
|
||||
if ($pullRequestId) {
|
||||
if (str($applicationId)->contains('-')) {
|
||||
$applicationId = str($applicationId)->before('-');
|
||||
}
|
||||
$preview = ApplicationPreview::where('application_id', $applicationId)->where('pull_request_id', $pullRequestId)->first();
|
||||
if ($preview) {
|
||||
$foundApplicationPreviews[] = $preview->id;
|
||||
$statusFromDb = $preview->status;
|
||||
if ($statusFromDb !== $containerStatus) {
|
||||
$preview->update(['status' => $containerStatus]);
|
||||
}
|
||||
} else {
|
||||
//Notify user that this container should not be there.
|
||||
}
|
||||
} else {
|
||||
$application = $applications->where('id', $applicationId)->first();
|
||||
if ($application) {
|
||||
$foundApplications[] = $application->id;
|
||||
$statusFromDb = $application->status;
|
||||
if ($statusFromDb !== $containerStatus) {
|
||||
$application->update(['status' => $containerStatus]);
|
||||
}
|
||||
} else {
|
||||
//Notify user that this container should not be there.
|
||||
}
|
||||
}
|
||||
} else {
|
||||
$uuid = data_get($labels, 'com.docker.compose.service');
|
||||
$type = data_get($labels, 'coolify.type');
|
||||
// foreach ($containers as $container) {
|
||||
// if ($this->server->isSwarm()) {
|
||||
// $labels = data_get($container, 'Spec.Labels');
|
||||
// $uuid = data_get($labels, 'coolify.name');
|
||||
// } else {
|
||||
// $labels = data_get($container, 'Config.Labels');
|
||||
// }
|
||||
// $containerStatus = data_get($container, 'State.Status');
|
||||
// $containerHealth = data_get($container, 'State.Health.Status', 'unhealthy');
|
||||
// $containerStatus = "$containerStatus ($containerHealth)";
|
||||
// $labels = Arr::undot(format_docker_labels_to_json($labels));
|
||||
// $applicationId = data_get($labels, 'coolify.applicationId');
|
||||
// if ($applicationId) {
|
||||
// $pullRequestId = data_get($labels, 'coolify.pullRequestId');
|
||||
// if ($pullRequestId) {
|
||||
// if (str($applicationId)->contains('-')) {
|
||||
// $applicationId = str($applicationId)->before('-');
|
||||
// }
|
||||
// $preview = ApplicationPreview::where('application_id', $applicationId)->where('pull_request_id', $pullRequestId)->first();
|
||||
// if ($preview) {
|
||||
// $foundApplicationPreviews[] = $preview->id;
|
||||
// $statusFromDb = $preview->status;
|
||||
// if ($statusFromDb !== $containerStatus) {
|
||||
// $preview->update(['status' => $containerStatus]);
|
||||
// }
|
||||
// } else {
|
||||
// //Notify user that this container should not be there.
|
||||
// }
|
||||
// } else {
|
||||
// $application = $applications->where('id', $applicationId)->first();
|
||||
// if ($application) {
|
||||
// $foundApplications[] = $application->id;
|
||||
// $statusFromDb = $application->status;
|
||||
// if ($statusFromDb !== $containerStatus) {
|
||||
// $application->update(['status' => $containerStatus]);
|
||||
// }
|
||||
// } else {
|
||||
// //Notify user that this container should not be there.
|
||||
// }
|
||||
// }
|
||||
// } else {
|
||||
// $uuid = data_get($labels, 'com.docker.compose.service');
|
||||
// $type = data_get($labels, 'coolify.type');
|
||||
|
||||
if ($uuid) {
|
||||
if ($type === 'service') {
|
||||
$database_id = data_get($labels, 'coolify.service.subId');
|
||||
if ($database_id) {
|
||||
$service_db = ServiceDatabase::where('id', $database_id)->first();
|
||||
if ($service_db) {
|
||||
$uuid = $service_db->service->uuid;
|
||||
$isPublic = data_get($service_db, 'is_public');
|
||||
if ($isPublic) {
|
||||
$foundTcpProxy = $containers->filter(function ($value, $key) use ($uuid) {
|
||||
if ($this->server->isSwarm()) {
|
||||
return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid";
|
||||
} else {
|
||||
return data_get($value, 'Name') === "/$uuid-proxy";
|
||||
}
|
||||
})->first();
|
||||
if (!$foundTcpProxy) {
|
||||
StartDatabaseProxy::run($service_db);
|
||||
// $this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$service_db->service->name}", $this->server));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
$database = $databases->where('uuid', $uuid)->first();
|
||||
if ($database) {
|
||||
$isPublic = data_get($database, 'is_public');
|
||||
$foundDatabases[] = $database->id;
|
||||
$statusFromDb = $database->status;
|
||||
if ($statusFromDb !== $containerStatus) {
|
||||
$database->update(['status' => $containerStatus]);
|
||||
}
|
||||
if ($isPublic) {
|
||||
$foundTcpProxy = $containers->filter(function ($value, $key) use ($uuid) {
|
||||
if ($this->server->isSwarm()) {
|
||||
return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid";
|
||||
} else {
|
||||
return data_get($value, 'Name') === "/$uuid-proxy";
|
||||
}
|
||||
})->first();
|
||||
if (!$foundTcpProxy) {
|
||||
StartDatabaseProxy::run($database);
|
||||
$this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$database->name}", $this->server));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Notify user that this container should not be there.
|
||||
}
|
||||
}
|
||||
}
|
||||
if (data_get($container, 'Name') === '/coolify-db') {
|
||||
$foundDatabases[] = 0;
|
||||
}
|
||||
}
|
||||
$serviceLabelId = data_get($labels, 'coolify.serviceId');
|
||||
if ($serviceLabelId) {
|
||||
$subType = data_get($labels, 'coolify.service.subType');
|
||||
$subId = data_get($labels, 'coolify.service.subId');
|
||||
$service = $services->where('id', $serviceLabelId)->first();
|
||||
if (!$service) {
|
||||
continue;
|
||||
}
|
||||
if ($subType === 'application') {
|
||||
$service = $service->applications()->where('id', $subId)->first();
|
||||
} else {
|
||||
$service = $service->databases()->where('id', $subId)->first();
|
||||
}
|
||||
if ($service) {
|
||||
$foundServices[] = "$service->id-$service->name";
|
||||
$statusFromDb = $service->status;
|
||||
if ($statusFromDb !== $containerStatus) {
|
||||
// ray('Updating status: ' . $containerStatus);
|
||||
$service->update(['status' => $containerStatus]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
$exitedServices = collect([]);
|
||||
foreach ($services as $service) {
|
||||
$apps = $service->applications()->get();
|
||||
$dbs = $service->databases()->get();
|
||||
foreach ($apps as $app) {
|
||||
if (in_array("$app->id-$app->name", $foundServices)) {
|
||||
continue;
|
||||
} else {
|
||||
$exitedServices->push($app);
|
||||
}
|
||||
}
|
||||
foreach ($dbs as $db) {
|
||||
if (in_array("$db->id-$db->name", $foundServices)) {
|
||||
continue;
|
||||
} else {
|
||||
$exitedServices->push($db);
|
||||
}
|
||||
}
|
||||
}
|
||||
$exitedServices = $exitedServices->unique('id');
|
||||
foreach ($exitedServices as $exitedService) {
|
||||
if (str($exitedService->status)->startsWith('exited')) {
|
||||
continue;
|
||||
}
|
||||
$name = data_get($exitedService, 'name');
|
||||
$fqdn = data_get($exitedService, 'fqdn');
|
||||
$containerName = $name ? "$name, available at $fqdn" : $fqdn;
|
||||
$projectUuid = data_get($service, 'environment.project.uuid');
|
||||
$serviceUuid = data_get($service, 'uuid');
|
||||
$environmentName = data_get($service, 'environment.name');
|
||||
// if ($uuid) {
|
||||
// if ($type === 'service') {
|
||||
// $database_id = data_get($labels, 'coolify.service.subId');
|
||||
// if ($database_id) {
|
||||
// $service_db = ServiceDatabase::where('id', $database_id)->first();
|
||||
// if ($service_db) {
|
||||
// $uuid = $service_db->service->uuid;
|
||||
// $isPublic = data_get($service_db, 'is_public');
|
||||
// if ($isPublic) {
|
||||
// $foundTcpProxy = $containers->filter(function ($value, $key) use ($uuid) {
|
||||
// if ($this->server->isSwarm()) {
|
||||
// return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid";
|
||||
// } else {
|
||||
// return data_get($value, 'Name') === "/$uuid-proxy";
|
||||
// }
|
||||
// })->first();
|
||||
// if (!$foundTcpProxy) {
|
||||
// StartDatabaseProxy::run($service_db);
|
||||
// // $this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$service_db->service->name}", $this->server));
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// } else {
|
||||
// $database = $databases->where('uuid', $uuid)->first();
|
||||
// if ($database) {
|
||||
// $isPublic = data_get($database, 'is_public');
|
||||
// $foundDatabases[] = $database->id;
|
||||
// $statusFromDb = $database->status;
|
||||
// if ($statusFromDb !== $containerStatus) {
|
||||
// $database->update(['status' => $containerStatus]);
|
||||
// }
|
||||
// if ($isPublic) {
|
||||
// $foundTcpProxy = $containers->filter(function ($value, $key) use ($uuid) {
|
||||
// if ($this->server->isSwarm()) {
|
||||
// return data_get($value, 'Spec.Name') === "coolify-proxy_$uuid";
|
||||
// } else {
|
||||
// return data_get($value, 'Name') === "/$uuid-proxy";
|
||||
// }
|
||||
// })->first();
|
||||
// if (!$foundTcpProxy) {
|
||||
// StartDatabaseProxy::run($database);
|
||||
// $this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$database->name}", $this->server));
|
||||
// }
|
||||
// }
|
||||
// } else {
|
||||
// // Notify user that this container should not be there.
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// if (data_get($container, 'Name') === '/coolify-db') {
|
||||
// $foundDatabases[] = 0;
|
||||
// }
|
||||
// }
|
||||
// $serviceLabelId = data_get($labels, 'coolify.serviceId');
|
||||
// if ($serviceLabelId) {
|
||||
// $subType = data_get($labels, 'coolify.service.subType');
|
||||
// $subId = data_get($labels, 'coolify.service.subId');
|
||||
// $service = $services->where('id', $serviceLabelId)->first();
|
||||
// if (!$service) {
|
||||
// continue;
|
||||
// }
|
||||
// if ($subType === 'application') {
|
||||
// $service = $service->applications()->where('id', $subId)->first();
|
||||
// } else {
|
||||
// $service = $service->databases()->where('id', $subId)->first();
|
||||
// }
|
||||
// if ($service) {
|
||||
// $foundServices[] = "$service->id-$service->name";
|
||||
// $statusFromDb = $service->status;
|
||||
// if ($statusFromDb !== $containerStatus) {
|
||||
// // ray('Updating status: ' . $containerStatus);
|
||||
// $service->update(['status' => $containerStatus]);
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// $exitedServices = collect([]);
|
||||
// foreach ($services as $service) {
|
||||
// $apps = $service->applications()->get();
|
||||
// $dbs = $service->databases()->get();
|
||||
// foreach ($apps as $app) {
|
||||
// if (in_array("$app->id-$app->name", $foundServices)) {
|
||||
// continue;
|
||||
// } else {
|
||||
// $exitedServices->push($app);
|
||||
// }
|
||||
// }
|
||||
// foreach ($dbs as $db) {
|
||||
// if (in_array("$db->id-$db->name", $foundServices)) {
|
||||
// continue;
|
||||
// } else {
|
||||
// $exitedServices->push($db);
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// $exitedServices = $exitedServices->unique('id');
|
||||
// foreach ($exitedServices as $exitedService) {
|
||||
// if (str($exitedService->status)->startsWith('exited')) {
|
||||
// continue;
|
||||
// }
|
||||
// $name = data_get($exitedService, 'name');
|
||||
// $fqdn = data_get($exitedService, 'fqdn');
|
||||
// $containerName = $name ? "$name, available at $fqdn" : $fqdn;
|
||||
// $projectUuid = data_get($service, 'environment.project.uuid');
|
||||
// $serviceUuid = data_get($service, 'uuid');
|
||||
// $environmentName = data_get($service, 'environment.name');
|
||||
|
||||
if ($projectUuid && $serviceUuid && $environmentName) {
|
||||
$url = base_url() . '/project/' . $projectUuid . "/" . $environmentName . "/service/" . $serviceUuid;
|
||||
} else {
|
||||
$url = null;
|
||||
}
|
||||
$this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url));
|
||||
$exitedService->update(['status' => 'exited']);
|
||||
}
|
||||
// if ($projectUuid && $serviceUuid && $environmentName) {
|
||||
// $url = base_url() . '/project/' . $projectUuid . "/" . $environmentName . "/service/" . $serviceUuid;
|
||||
// } else {
|
||||
// $url = null;
|
||||
// }
|
||||
// $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url));
|
||||
// $exitedService->update(['status' => 'exited']);
|
||||
// }
|
||||
|
||||
$notRunningApplications = $applications->pluck('id')->diff($foundApplications);
|
||||
foreach ($notRunningApplications as $applicationId) {
|
||||
$application = $applications->where('id', $applicationId)->first();
|
||||
if (str($application->status)->startsWith('exited')) {
|
||||
continue;
|
||||
}
|
||||
$application->update(['status' => 'exited']);
|
||||
// $notRunningApplications = $applications->pluck('id')->diff($foundApplications);
|
||||
// foreach ($notRunningApplications as $applicationId) {
|
||||
// $application = $applications->where('id', $applicationId)->first();
|
||||
// if (str($application->status)->startsWith('exited')) {
|
||||
// continue;
|
||||
// }
|
||||
// $application->update(['status' => 'exited']);
|
||||
|
||||
$name = data_get($application, 'name');
|
||||
$fqdn = data_get($application, 'fqdn');
|
||||
// $name = data_get($application, 'name');
|
||||
// $fqdn = data_get($application, 'fqdn');
|
||||
|
||||
$containerName = $name ? "$name ($fqdn)" : $fqdn;
|
||||
// $containerName = $name ? "$name ($fqdn)" : $fqdn;
|
||||
|
||||
$projectUuid = data_get($application, 'environment.project.uuid');
|
||||
$applicationUuid = data_get($application, 'uuid');
|
||||
$environment = data_get($application, 'environment.name');
|
||||
// $projectUuid = data_get($application, 'environment.project.uuid');
|
||||
// $applicationUuid = data_get($application, 'uuid');
|
||||
// $environment = data_get($application, 'environment.name');
|
||||
|
||||
if ($projectUuid && $applicationUuid && $environment) {
|
||||
$url = base_url() . '/project/' . $projectUuid . "/" . $environment . "/application/" . $applicationUuid;
|
||||
} else {
|
||||
$url = null;
|
||||
}
|
||||
// if ($projectUuid && $applicationUuid && $environment) {
|
||||
// $url = base_url() . '/project/' . $projectUuid . "/" . $environment . "/application/" . $applicationUuid;
|
||||
// } else {
|
||||
// $url = null;
|
||||
// }
|
||||
|
||||
$this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url));
|
||||
}
|
||||
$notRunningApplicationPreviews = $previews->pluck('id')->diff($foundApplicationPreviews);
|
||||
foreach ($notRunningApplicationPreviews as $previewId) {
|
||||
$preview = $previews->where('id', $previewId)->first();
|
||||
if (str($preview->status)->startsWith('exited')) {
|
||||
continue;
|
||||
}
|
||||
$preview->update(['status' => 'exited']);
|
||||
// $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url));
|
||||
// }
|
||||
// $notRunningApplicationPreviews = $previews->pluck('id')->diff($foundApplicationPreviews);
|
||||
// foreach ($notRunningApplicationPreviews as $previewId) {
|
||||
// $preview = $previews->where('id', $previewId)->first();
|
||||
// if (str($preview->status)->startsWith('exited')) {
|
||||
// continue;
|
||||
// }
|
||||
// $preview->update(['status' => 'exited']);
|
||||
|
||||
$name = data_get($preview, 'name');
|
||||
$fqdn = data_get($preview, 'fqdn');
|
||||
// $name = data_get($preview, 'name');
|
||||
// $fqdn = data_get($preview, 'fqdn');
|
||||
|
||||
$containerName = $name ? "$name ($fqdn)" : $fqdn;
|
||||
// $containerName = $name ? "$name ($fqdn)" : $fqdn;
|
||||
|
||||
$projectUuid = data_get($preview, 'application.environment.project.uuid');
|
||||
$environmentName = data_get($preview, 'application.environment.name');
|
||||
$applicationUuid = data_get($preview, 'application.uuid');
|
||||
// $projectUuid = data_get($preview, 'application.environment.project.uuid');
|
||||
// $environmentName = data_get($preview, 'application.environment.name');
|
||||
// $applicationUuid = data_get($preview, 'application.uuid');
|
||||
|
||||
if ($projectUuid && $applicationUuid && $environmentName) {
|
||||
$url = base_url() . '/project/' . $projectUuid . "/" . $environmentName . "/application/" . $applicationUuid;
|
||||
} else {
|
||||
$url = null;
|
||||
}
|
||||
// if ($projectUuid && $applicationUuid && $environmentName) {
|
||||
// $url = base_url() . '/project/' . $projectUuid . "/" . $environmentName . "/application/" . $applicationUuid;
|
||||
// } else {
|
||||
// $url = null;
|
||||
// }
|
||||
|
||||
$this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url));
|
||||
}
|
||||
$notRunningDatabases = $databases->pluck('id')->diff($foundDatabases);
|
||||
foreach ($notRunningDatabases as $database) {
|
||||
$database = $databases->where('id', $database)->first();
|
||||
if (str($database->status)->startsWith('exited')) {
|
||||
continue;
|
||||
}
|
||||
$database->update(['status' => 'exited']);
|
||||
// $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url));
|
||||
// }
|
||||
// $notRunningDatabases = $databases->pluck('id')->diff($foundDatabases);
|
||||
// foreach ($notRunningDatabases as $database) {
|
||||
// $database = $databases->where('id', $database)->first();
|
||||
// if (str($database->status)->startsWith('exited')) {
|
||||
// continue;
|
||||
// }
|
||||
// $database->update(['status' => 'exited']);
|
||||
|
||||
$name = data_get($database, 'name');
|
||||
$fqdn = data_get($database, 'fqdn');
|
||||
// $name = data_get($database, 'name');
|
||||
// $fqdn = data_get($database, 'fqdn');
|
||||
|
||||
$containerName = $name;
|
||||
// $containerName = $name;
|
||||
|
||||
$projectUuid = data_get($database, 'environment.project.uuid');
|
||||
$environmentName = data_get($database, 'environment.name');
|
||||
$databaseUuid = data_get($database, 'uuid');
|
||||
// $projectUuid = data_get($database, 'environment.project.uuid');
|
||||
// $environmentName = data_get($database, 'environment.name');
|
||||
// $databaseUuid = data_get($database, 'uuid');
|
||||
|
||||
if ($projectUuid && $databaseUuid && $environmentName) {
|
||||
$url = base_url() . '/project/' . $projectUuid . "/" . $environmentName . "/database/" . $databaseUuid;
|
||||
} else {
|
||||
$url = null;
|
||||
}
|
||||
$this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url));
|
||||
}
|
||||
// if ($projectUuid && $databaseUuid && $environmentName) {
|
||||
// $url = base_url() . '/project/' . $projectUuid . "/" . $environmentName . "/database/" . $databaseUuid;
|
||||
// } else {
|
||||
// $url = null;
|
||||
// }
|
||||
// $this->server->team?->notify(new ContainerStopped($containerName, $this->server, $url));
|
||||
// }
|
||||
|
||||
// Check if proxy is running
|
||||
$this->server->proxyType();
|
||||
$foundProxyContainer = $containers->filter(function ($value, $key) {
|
||||
if ($this->server->isSwarm()) {
|
||||
return data_get($value, 'Spec.Name') === 'coolify-proxy_traefik';
|
||||
} else {
|
||||
return data_get($value, 'Name') === '/coolify-proxy';
|
||||
}
|
||||
})->first();
|
||||
if (!$foundProxyContainer) {
|
||||
try {
|
||||
$shouldStart = CheckProxy::run($this->server);
|
||||
if ($shouldStart) {
|
||||
StartProxy::run($this->server, false);
|
||||
$this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server));
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
ray($e);
|
||||
}
|
||||
} else {
|
||||
$this->server->proxy->status = data_get($foundProxyContainer, 'State.Status');
|
||||
$this->server->save();
|
||||
$connectProxyToDockerNetworks = connectProxyToNetworks($this->server);
|
||||
instant_remote_process($connectProxyToDockerNetworks, $this->server, false);
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
send_internal_notification("ContainerStatusJob failed on ({$this->server->id}) with: " . $e->getMessage());
|
||||
ray($e->getMessage());
|
||||
return handleError($e);
|
||||
}
|
||||
// // Check if proxy is running
|
||||
// $this->server->proxyType();
|
||||
// $foundProxyContainer = $containers->filter(function ($value, $key) {
|
||||
// if ($this->server->isSwarm()) {
|
||||
// return data_get($value, 'Spec.Name') === 'coolify-proxy_traefik';
|
||||
// } else {
|
||||
// return data_get($value, 'Name') === '/coolify-proxy';
|
||||
// }
|
||||
// })->first();
|
||||
// if (!$foundProxyContainer) {
|
||||
// try {
|
||||
// $shouldStart = CheckProxy::run($this->server);
|
||||
// if ($shouldStart) {
|
||||
// StartProxy::run($this->server, false);
|
||||
// $this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server));
|
||||
// }
|
||||
// } catch (\Throwable $e) {
|
||||
// ray($e);
|
||||
// }
|
||||
// } else {
|
||||
// $this->server->proxy->status = data_get($foundProxyContainer, 'State.Status');
|
||||
// $this->server->save();
|
||||
// $connectProxyToDockerNetworks = connectProxyToNetworks($this->server);
|
||||
// instant_remote_process($connectProxyToDockerNetworks, $this->server, false);
|
||||
// }
|
||||
// } catch (\Throwable $e) {
|
||||
// send_internal_notification("ContainerStatusJob failed on ({$this->server->id}) with: " . $e->getMessage());
|
||||
// ray($e->getMessage());
|
||||
// return handleError($e);
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
@ -3,6 +3,7 @@
|
||||
namespace App\Livewire\Project\Application;
|
||||
|
||||
use App\Actions\Application\StopApplication;
|
||||
use App\Actions\Docker\GetContainersStatus;
|
||||
use App\Events\ApplicationStatusChanged;
|
||||
use App\Jobs\ContainerStatusJob;
|
||||
use App\Jobs\ServerStatusJob;
|
||||
@ -32,7 +33,8 @@ public function mount()
|
||||
public function check_status($showNotification = false)
|
||||
{
|
||||
if ($this->application->destination->server->isFunctional()) {
|
||||
dispatch(new ContainerStatusJob($this->application->destination->server));
|
||||
GetContainersStatus::dispatch($this->application->destination->server);
|
||||
// dispatch(new ContainerStatusJob($this->application->destination->server));
|
||||
} else {
|
||||
dispatch(new ServerStatusJob($this->application->destination->server));
|
||||
}
|
||||
|
@ -11,6 +11,7 @@
|
||||
use App\Actions\Database\StartPostgresql;
|
||||
use App\Actions\Database\StartRedis;
|
||||
use App\Actions\Database\StopDatabase;
|
||||
use App\Actions\Docker\GetContainersStatus;
|
||||
use App\Jobs\ContainerStatusJob;
|
||||
use Livewire\Component;
|
||||
|
||||
@ -44,7 +45,8 @@ public function activityFinished()
|
||||
|
||||
public function check_status($showNotification = false)
|
||||
{
|
||||
dispatch_sync(new ContainerStatusJob($this->database->destination->server));
|
||||
GetContainersStatus::run($this->application->destination->server);
|
||||
// dispatch_sync(new ContainerStatusJob($this->database->destination->server));
|
||||
$this->database->refresh();
|
||||
if ($showNotification) $this->dispatch('success', 'Database status updated.');
|
||||
}
|
||||
|
@ -2,6 +2,7 @@
|
||||
|
||||
namespace App\Livewire\Project\Service;
|
||||
|
||||
use App\Actions\Docker\GetContainersStatus;
|
||||
use App\Jobs\ContainerStatusJob;
|
||||
use App\Models\Service;
|
||||
use Livewire\Component;
|
||||
@ -64,7 +65,8 @@ public function restartDatabase($id)
|
||||
public function check_status()
|
||||
{
|
||||
try {
|
||||
dispatch_sync(new ContainerStatusJob($this->service->server));
|
||||
GetContainersStatus::run($this->service->server);
|
||||
// dispatch_sync(new ContainerStatusJob($this->service->server));
|
||||
$this->dispatch('refresh')->self();
|
||||
$this->dispatch('updateStatus');
|
||||
} catch (\Exception $e) {
|
||||
|
@ -3,6 +3,7 @@
|
||||
namespace App\Livewire\Project\Shared;
|
||||
|
||||
use App\Actions\Application\StopApplicationOneServer;
|
||||
use App\Actions\Docker\GetContainersStatus;
|
||||
use App\Events\ApplicationStatusChanged;
|
||||
use App\Jobs\ContainerStatusJob;
|
||||
use App\Models\Server;
|
||||
@ -90,7 +91,8 @@ public function promote(int $network_id, int $server_id)
|
||||
}
|
||||
public function refreshServers()
|
||||
{
|
||||
ContainerStatusJob::dispatchSync($this->resource->destination->server);
|
||||
GetContainersStatus::run($this->resource->destination->server);
|
||||
// ContainerStatusJob::dispatchSync($this->resource->destination->server);
|
||||
$this->loadData();
|
||||
$this->dispatch('refresh');
|
||||
ApplicationStatusChanged::dispatch(data_get($this->resource, 'environment.project.team.id'));
|
||||
|
@ -2,6 +2,7 @@
|
||||
|
||||
namespace App\Livewire\Server\Proxy;
|
||||
|
||||
use App\Actions\Docker\GetContainersStatus;
|
||||
use App\Actions\Proxy\CheckProxy;
|
||||
use App\Jobs\ContainerStatusJob;
|
||||
use App\Models\Server;
|
||||
@ -49,7 +50,8 @@ public function checkProxy(bool $notification = false)
|
||||
public function getProxyStatus()
|
||||
{
|
||||
try {
|
||||
dispatch_sync(new ContainerStatusJob($this->server));
|
||||
GetContainersStatus::run($this->server);
|
||||
// dispatch_sync(new ContainerStatusJob($this->server));
|
||||
$this->dispatch('proxyStatusUpdated');
|
||||
} catch (\Throwable $e) {
|
||||
return handleError($e, $this);
|
||||
|
@ -2,6 +2,7 @@
|
||||
|
||||
namespace App\Notifications\Server;
|
||||
|
||||
use App\Actions\Docker\GetContainersStatus;
|
||||
use App\Jobs\ContainerStatusJob;
|
||||
use App\Models\Server;
|
||||
use Illuminate\Bus\Queueable;
|
||||
@ -22,7 +23,8 @@ public function __construct(public Server $server)
|
||||
if ($this->server->unreachable_notification_sent === false) {
|
||||
return;
|
||||
}
|
||||
dispatch(new ContainerStatusJob($server));
|
||||
GetContainersStatus::dispatch($server);
|
||||
// dispatch(new ContainerStatusJob($server));
|
||||
}
|
||||
|
||||
public function via(object $notifiable): array
|
||||
|
28
database/migrations/2024_05_07_124019_add_server_metrics.php
Normal file
28
database/migrations/2024_05_07_124019_add_server_metrics.php
Normal file
@ -0,0 +1,28 @@
|
||||
<?php
|
||||
|
||||
use Illuminate\Database\Migrations\Migration;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
use Illuminate\Support\Facades\Schema;
|
||||
|
||||
return new class extends Migration
|
||||
{
|
||||
/**
|
||||
* Run the migrations.
|
||||
*/
|
||||
public function up(): void
|
||||
{
|
||||
Schema::table('servers', function (Blueprint $table) {
|
||||
$table->boolean('is_metrics_enabled')->default(true);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Reverse the migrations.
|
||||
*/
|
||||
public function down(): void
|
||||
{
|
||||
Schema::table('servers', function (Blueprint $table) {
|
||||
$table->dropColumn('is_metrics_enabled');
|
||||
});
|
||||
}
|
||||
};
|
@ -19,6 +19,12 @@ services:
|
||||
PUSHER_APP_SECRET: "${PUSHER_APP_SECRET:-coolify}"
|
||||
volumes:
|
||||
- .:/var/www/html/:cached
|
||||
sentinel:
|
||||
ports:
|
||||
- "127.0.0.1:8888:8888"
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
- .:/var/www/html/:cached
|
||||
postgres:
|
||||
pull_policy: always
|
||||
ports:
|
||||
|
@ -103,6 +103,15 @@ services:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
sentinel:
|
||||
image: "ghcr.io/coollabsio/sentinel:${LATEST_IMAGE:-latest}"
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
- /data/coolify/metrics:/var/www/html/storage/app/metrics
|
||||
ports:
|
||||
- "127.0.0.1:8888:8888"
|
||||
healthcheck:
|
||||
test: curl --fail http://127.0.0.1:8888/api/health || exit 1
|
||||
postgres:
|
||||
volumes:
|
||||
- coolify-db:/var/lib/postgresql/data
|
||||
|
@ -10,7 +10,12 @@ services:
|
||||
depends_on:
|
||||
- postgres
|
||||
- redis
|
||||
|
||||
sentinel:
|
||||
image: "ghcr.io/coollabsio/sentinel:latest"
|
||||
container_name: coolify-sentinel
|
||||
restart: always
|
||||
networks:
|
||||
- coolify
|
||||
postgres:
|
||||
image: postgres:15-alpine
|
||||
container_name: coolify-db
|
||||
|
Loading…
Reference in New Issue
Block a user