diff --git a/app/Actions/Proxy/StartProxy.php b/app/Actions/Proxy/StartProxy.php
index d8a14d27e..7e2878b7e 100644
--- a/app/Actions/Proxy/StartProxy.php
+++ b/app/Actions/Proxy/StartProxy.php
@@ -8,7 +8,7 @@
class StartProxy
{
- public function __invoke(Server $server): Activity
+ public function __invoke(Server $server, bool $async = true): Activity|string
{
$proxy_path = get_proxy_path();
$networks = collect($server->standaloneDockers)->map(function ($docker) {
@@ -26,8 +26,7 @@ public function __invoke(Server $server): Activity
$docker_compose_yml_base64 = base64_encode($configuration);
$server->proxy->last_applied_settings = Str::of($docker_compose_yml_base64)->pipe('md5')->value;
$server->save();
-
- $activity = remote_process([
+ $commands = [
"echo '####### Creating required Docker networks...'",
...$create_networks_command,
"cd $proxy_path",
@@ -44,8 +43,13 @@ public function __invoke(Server $server): Activity
"echo '####### Starting coolify-proxy...'",
'docker compose up -d --remove-orphans',
"echo '####### Proxy installed successfully...'"
- ], $server);
-
- return $activity;
+ ];
+ if (!$async) {
+ instant_remote_process($commands, $server);
+ return 'OK';
+ } else {
+ $activity = remote_process($commands, $server);
+ return $activity;
+ }
}
}
diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php
index ae40fec49..d15107c19 100644
--- a/app/Console/Kernel.php
+++ b/app/Console/Kernel.php
@@ -2,21 +2,15 @@
namespace App\Console;
-use App\Enums\ProxyTypes;
-use App\Jobs\ApplicationContainerStatusJob;
use App\Jobs\CheckResaleLicenseJob;
use App\Jobs\CleanupInstanceStuffsJob;
use App\Jobs\DatabaseBackupJob;
-use App\Jobs\DatabaseContainerStatusJob;
use App\Jobs\DockerCleanupJob;
use App\Jobs\InstanceAutoUpdateJob;
-use App\Jobs\ProxyContainerStatusJob;
-use App\Jobs\ServerDetailsCheckJob;
-use App\Models\Application;
+use App\Jobs\ContainerStatusJob;
use App\Models\InstanceSettings;
use App\Models\ScheduledDatabaseBackup;
use App\Models\Server;
-use App\Models\StandalonePostgresql;
use Illuminate\Console\Scheduling\Schedule;
use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
@@ -25,15 +19,14 @@ class Kernel extends ConsoleKernel
protected function schedule(Schedule $schedule): void
{
if (isDev()) {
- $schedule->job(new ServerDetailsCheckJob(Server::find(0)))->everyTenMinutes()->onOneServer();
+ // $schedule->job(new ContainerStatusJob(Server::find(0)))->everyTenMinutes()->onOneServer();
// $schedule->command('horizon:snapshot')->everyMinute();
// $schedule->job(new CleanupInstanceStuffsJob)->everyMinute();
// $schedule->job(new CheckResaleLicenseJob)->hourly();
// $schedule->job(new DockerCleanupJob)->everyOddHour();
// $this->instance_auto_update($schedule);
// $this->check_scheduled_backups($schedule);
- // $this->check_resources($schedule);
- // $this->check_proxies($schedule);
+ $this->check_resources($schedule);
} else {
$schedule->command('horizon:snapshot')->everyFiveMinutes();
$schedule->job(new CleanupInstanceStuffsJob)->everyTenMinutes()->onOneServer();
@@ -42,26 +35,15 @@ protected function schedule(Schedule $schedule): void
$this->instance_auto_update($schedule);
$this->check_scheduled_backups($schedule);
$this->check_resources($schedule);
- $this->check_proxies($schedule);
- }
- }
- private function check_proxies($schedule)
- {
- $servers = Server::all()->where('settings.is_usable', true)->where('settings.is_reachable', true)->whereNotNull('proxy.type')->where('proxy.type', '!=', ProxyTypes::NONE->value);
- foreach ($servers as $server) {
- $schedule->job(new ProxyContainerStatusJob($server))->everyMinute()->onOneServer();
}
}
private function check_resources($schedule)
{
- $applications = Application::all();
- foreach ($applications as $application) {
- $schedule->job(new ApplicationContainerStatusJob($application))->everyMinute()->onOneServer();
- }
+ $servers = Server::all()->where('settings.is_usable', true)->where('settings.is_reachable', true);
+ ray($servers);
- $postgresqls = StandalonePostgresql::all();
- foreach ($postgresqls as $postgresql) {
- $schedule->job(new DatabaseContainerStatusJob($postgresql))->everyMinute()->onOneServer();
+ foreach ($servers as $server) {
+ $schedule->job(new ContainerStatusJob($server))->everyMinute()->onOneServer();
}
}
private function instance_auto_update($schedule)
diff --git a/app/Http/Livewire/Project/Application/Heading.php b/app/Http/Livewire/Project/Application/Heading.php
index 5b9560007..0774ad441 100644
--- a/app/Http/Livewire/Project/Application/Heading.php
+++ b/app/Http/Livewire/Project/Application/Heading.php
@@ -3,6 +3,7 @@
namespace App\Http\Livewire\Project\Application;
use App\Jobs\ApplicationContainerStatusJob;
+use App\Jobs\ContainerStatusJob;
use App\Models\Application;
use App\Notifications\Application\StatusChanged;
use Livewire\Component;
@@ -22,9 +23,8 @@ public function mount()
public function check_status()
{
- dispatch_sync(new ApplicationContainerStatusJob(
- application: $this->application,
- ));
+ ray($this->application->destination->server);
+ dispatch_sync(new ContainerStatusJob($this->application->destination->server));
$this->application->refresh();
}
diff --git a/app/Http/Livewire/Project/Database/Heading.php b/app/Http/Livewire/Project/Database/Heading.php
index 178cbfca5..f2fe8f033 100644
--- a/app/Http/Livewire/Project/Database/Heading.php
+++ b/app/Http/Livewire/Project/Database/Heading.php
@@ -3,8 +3,7 @@
namespace App\Http\Livewire\Project\Database;
use App\Actions\Database\StartPostgresql;
-use App\Jobs\DatabaseContainerStatusJob;
-use App\Notifications\Application\StatusChanged;
+use App\Jobs\ContainerStatusJob;
use Livewire\Component;
class Heading extends Component
@@ -25,9 +24,7 @@ public function activityFinished()
public function check_status()
{
- dispatch_sync(new DatabaseContainerStatusJob(
- database: $this->database,
- ));
+ dispatch_sync(new ContainerStatusJob($this->database->destination->server));
$this->database->refresh();
}
diff --git a/app/Http/Livewire/Server/Form.php b/app/Http/Livewire/Server/Form.php
index 9d5c892b3..b288d3bfa 100644
--- a/app/Http/Livewire/Server/Form.php
+++ b/app/Http/Livewire/Server/Form.php
@@ -88,17 +88,11 @@ public function delete()
public function submit()
{
$this->validate();
- // $validation = Validator::make($this->server->toArray(), [
- // 'ip' => [
- // 'ip'
- // ],
- // ]);
- // if ($validation->fails()) {
- // foreach ($validation->errors()->getMessages() as $key => $value) {
- // $this->addError("server.{$key}", $value[0]);
- // }
- // return;
- // }
+ $uniqueIPs = Server::all()->pluck('ip')->toArray();
+ if (in_array($this->server->ip, $uniqueIPs)) {
+ $this->emit('error', 'IP address is already in use by another team.');
+ return;
+ }
$this->server->settings->wildcard_domain = $this->wildcard_domain;
$this->server->settings->cleanup_after_percentage = $this->cleanup_after_percentage;
$this->server->settings->save();
diff --git a/app/Http/Livewire/Settings/Configuration.php b/app/Http/Livewire/Settings/Configuration.php
index 6801823fe..9bc7c20cb 100644
--- a/app/Http/Livewire/Settings/Configuration.php
+++ b/app/Http/Livewire/Settings/Configuration.php
@@ -2,7 +2,7 @@
namespace App\Http\Livewire\Settings;
-use App\Jobs\ProxyContainerStatusJob;
+use App\Jobs\ContainerStatusJob;
use App\Models\InstanceSettings as ModelsInstanceSettings;
use App\Models\Server;
use Livewire\Component;
@@ -124,7 +124,7 @@ private function setup_instance_fqdn()
];
}
$this->save_configuration_to_disk($traefik_dynamic_conf, $file);
- dispatch(new ProxyContainerStatusJob($this->server));
+ dispatch(new ContainerStatusJob($this->server));
}
}
diff --git a/app/Jobs/ApplicationContainerStatusJob.php b/app/Jobs/ApplicationContainerStatusJob.php
index f1d2ed7c9..ae41a5c41 100644
--- a/app/Jobs/ApplicationContainerStatusJob.php
+++ b/app/Jobs/ApplicationContainerStatusJob.php
@@ -4,7 +4,6 @@
use App\Models\Application;
use App\Models\ApplicationPreview;
-use App\Notifications\Application\StatusChanged;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldBeUnique;
diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php
index d224afd10..ddd64b089 100644
--- a/app/Jobs/ApplicationDeploymentJob.php
+++ b/app/Jobs/ApplicationDeploymentJob.php
@@ -132,7 +132,9 @@ public function handle(): void
$this->deploy();
}
}
- if ($this->application->fqdn) dispatch(new ProxyContainerStatusJob($this->server));
+ if ($this->server->isProxyShouldRun()) {
+ dispatch(new ContainerStatusJob($this->server));
+ }
$this->next(ApplicationDeploymentStatus::FINISHED->value);
} catch (Exception $e) {
ray($e);
@@ -267,6 +269,7 @@ private function health_check()
"echo 'Rolling update completed.'"
],
);
+ $this->application->update(['status' => 'running']);
break;
}
$counter++;
diff --git a/app/Jobs/ContainerStatusJob.php b/app/Jobs/ContainerStatusJob.php
new file mode 100644
index 000000000..f2c82f4bf
--- /dev/null
+++ b/app/Jobs/ContainerStatusJob.php
@@ -0,0 +1,139 @@
+server->uuid)];
+ }
+
+ public function uniqueId(): string
+ {
+ return $this->server->uuid;
+ }
+
+ private function checkServerConnection() {
+ ray("Checking server connection to {$this->server->ip}");
+ $uptime = instant_remote_process(['uptime'], $this->server, false);
+ if (!is_null($uptime)) {
+ ray('Server is up');
+ return true;
+ }
+ }
+ public function handle(): void
+ {
+ try {
+ ray()->clearAll();
+ $serverUptimeCheckNumber = 0;
+ $serverUptimeCheckNumberMax = 5;
+ while (true) {
+ if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) {
+ $this->server->settings()->update(['is_reachable' => false]);
+ $this->server->team->notify(new Unreachable($this->server));
+ return;
+ }
+ $result = $this->checkServerConnection();
+ if ($result) {
+ break;
+ }
+ $serverUptimeCheckNumber++;
+ sleep(5);
+ }
+ $containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this->server);
+ $containers = format_docker_command_output_to_json($containers);
+ $applications = $this->server->applications();
+ $databases = $this->server->databases();
+ if ($this->server->isProxyShouldRun()) {
+ $foundProxyContainer = $containers->filter(function ($value, $key) {
+ return data_get($value, 'Name') === '/coolify-proxy';
+ })->first();
+ if (!$foundProxyContainer) {
+ resolve(StartProxy::class)($this->server, false);
+ $this->server->team->notify(new ContainerRestarted('coolify-proxy', $this->server));
+ }
+ }
+ foreach ($applications as $application) {
+ $uuid = data_get($application, 'uuid');
+ $foundContainer = $containers->filter(function ($value, $key) use ($uuid) {
+ return Str::startsWith(data_get($value, 'Name'), "/$uuid");
+ })->first();
+
+ if ($foundContainer) {
+ $containerStatus = data_get($foundContainer, 'State.Status');
+ $databaseStatus = data_get($application, 'status');
+ if ($containerStatus !== $databaseStatus) {
+ $application->update(['status' => $containerStatus]);
+ }
+ } else {
+ $databaseStatus = data_get($application, 'status');
+ if ($databaseStatus !== 'exited') {
+ $application->update(['status' => 'exited']);
+ $name = data_get($application, 'name');
+ $fqdn = data_get($application, 'fqdn');
+ $containerName = $name ? "$name ($fqdn)" : $fqdn;
+ $project = data_get($application, 'environment.project');
+ $environment = data_get($application, 'environment');
+ $url = base_url() . '/project/' . $project->uuid . "/" . $environment->name . "/application/" . $application->uuid;
+ $this->server->team->notify(new ContainerStopped($containerName, $this->server, $url));
+ }
+ }
+ }
+ foreach ($databases as $database) {
+ $uuid = data_get($database, 'uuid');
+ $foundContainer = $containers->filter(function ($value, $key) use ($uuid) {
+ return Str::startsWith(data_get($value, 'Name'), "/$uuid");
+ })->first();
+
+ if ($foundContainer) {
+ $containerStatus = data_get($foundContainer, 'State.Status');
+ $databaseStatus = data_get($database, 'status');
+ if ($containerStatus !== $databaseStatus) {
+ $database->update(['status' => $containerStatus]);
+ }
+ } else {
+ $databaseStatus = data_get($database, 'status');
+ if ($databaseStatus !== 'exited') {
+ $database->update(['status' => 'exited']);
+ $name = data_get($database, 'name');
+ $containerName = $name;
+ $project = data_get($database, 'environment.project');
+ $environment = data_get($database, 'environment');
+ $url = base_url() . '/project/' . $project->uuid . "/" . $environment->name . "/database/" . $database->uuid;
+ $this->server->team->notify(new ContainerStopped($containerName, $this->server, $url));
+ }
+ }
+ }
+ // TODO Monitor other containers not managed by Coolify
+ } catch (\Throwable $e) {
+ send_internal_notification('ContainerStatusJob failed with: ' . $e->getMessage());
+ ray($e->getMessage());
+ throw $e;
+ }
+ }
+}
diff --git a/app/Jobs/ServerDetailsCheckJob.php b/app/Jobs/ServerDetailsCheckJob.php
deleted file mode 100644
index 8a68c66ae..000000000
--- a/app/Jobs/ServerDetailsCheckJob.php
+++ /dev/null
@@ -1,80 +0,0 @@
-server->uuid)];
- }
-
- public function uniqueId(): string
- {
- return $this->server->uuid;
- }
-
- public function handle(): void
- {
- try {
- ray()->clearAll();
- $containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this->server);
- $containers = format_docker_command_output_to_json($containers);
- $applications = $this->server->applications();
- // ray($applications);
- // ray(format_docker_command_output_to_json($containers));
- foreach ($applications as $application) {
- $uuid = data_get($application, 'uuid');
- $foundContainer = $containers->filter(function ($value, $key) use ($uuid) {
- $image = data_get($value, 'Config.Image');
- return Str::startsWith($image, $uuid);
- })->first();
-
- if ($foundContainer) {
- $containerStatus = data_get($foundContainer, 'State.Status');
- $databaseStatus = data_get($application, 'status');
- ray($containerStatus, $databaseStatus);
- if ($containerStatus !== $databaseStatus) {
- // $application->update(['status' => $containerStatus]);
- }
- }
- }
- // foreach ($containers as $container) {
- // $labels = format_docker_labels_to_json(data_get($container,'Config.Labels'));
- // $foundLabel = $labels->filter(fn ($value, $key) => Str::startsWith($key, 'coolify.applicationId'));
- // if ($foundLabel->count() > 0) {
- // $appFound = $applications->where('id', $foundLabel['coolify.applicationId'])->first();
- // if ($appFound) {
- // $containerStatus = data_get($container, 'State.Status');
- // $databaseStatus = data_get($appFound, 'status');
- // ray($containerStatus, $databaseStatus);
- // }
- // }
- // }
- } catch (\Throwable $e) {
- // send_internal_notification('ServerDetailsCheckJob failed with: ' . $e->getMessage());
- ray($e->getMessage());
- throw $e;
- }
- }
-}
diff --git a/app/Notifications/Application/DeploymentSuccess.php b/app/Notifications/Application/DeploymentSuccess.php
index 0705e11ae..ea789b69a 100644
--- a/app/Notifications/Application/DeploymentSuccess.php
+++ b/app/Notifications/Application/DeploymentSuccess.php
@@ -53,7 +53,7 @@ public function toMail(): MailMessage
$pull_request_id = data_get($this->preview, 'pull_request_id', 0);
$fqdn = $this->fqdn;
if ($pull_request_id === 0) {
- $mail->subject("✅New version is deployed of {$this->application_name}");
+ $mail->subject("✅ New version is deployed of {$this->application_name}");
} else {
$fqdn = $this->preview->fqdn;
$mail->subject("✅ Pull request #{$pull_request_id} of {$this->application_name} deployed successfully");
diff --git a/app/Notifications/Container/ContainerRestarted.php b/app/Notifications/Container/ContainerRestarted.php
new file mode 100644
index 000000000..480fe2cb0
--- /dev/null
+++ b/app/Notifications/Container/ContainerRestarted.php
@@ -0,0 +1,62 @@
+subject("✅ Container ({$this->name}) has been restarted automatically on {$this->server->name}");
+ $mail->view('emails.container-restarted', [
+ 'containerName' => $this->name,
+ 'serverName' => $this->server->name,
+ 'url' => $this->url ,
+ ]);
+ return $mail;
+ }
+
+ public function toDiscord(): string
+ {
+ $message = "✅ Container ({$this->name}) has been restarted automatically on {$this->server->name}";
+ return $message;
+ }
+ public function toTelegram(): array
+ {
+ $message = "✅ Container ({$this->name}) has been restarted automatically on {$this->server->name}";
+ $payload = [
+ "message" => $message,
+ ];
+ if ($this->url) {
+ $payload['buttons'] = [
+ [
+ [
+ "text" => "Check Proxy in Coolify",
+ "url" => $this->url
+ ]
+ ]
+ ];
+ };
+ return $payload;
+ }
+}
diff --git a/app/Notifications/Container/ContainerStopped.php b/app/Notifications/Container/ContainerStopped.php
new file mode 100644
index 000000000..eec287751
--- /dev/null
+++ b/app/Notifications/Container/ContainerStopped.php
@@ -0,0 +1,61 @@
+subject("⛔ Container ({$this->name}) has been stopped on {$this->server->name}");
+ $mail->view('emails.container-stopped', [
+ 'containerName' => $this->name,
+ 'serverName' => $this->server->name,
+ 'url' => $this->url,
+ ]);
+ return $mail;
+ }
+
+ public function toDiscord(): string
+ {
+ $message = "⛔ Container ({$this->name}) has been stopped on {$this->server->name}";
+ return $message;
+ }
+ public function toTelegram(): array
+ {
+ $message = "⛔ Container ({$this->name}) has been stopped on {$this->server->name}";
+ $payload = [
+ "message" => $message,
+ ];
+ if ($this->url) {
+ $payload['buttons'] = [
+ [
+ [
+ "text" => "Open Application in Coolify",
+ "url" => $this->url
+ ]
+ ]
+ ];
+ }
+ return $payload;
+ }
+}
diff --git a/app/Notifications/Server/NotReachable.php b/app/Notifications/Server/NotReachable.php
deleted file mode 100644
index 672636c57..000000000
--- a/app/Notifications/Server/NotReachable.php
+++ /dev/null
@@ -1,53 +0,0 @@
-fqdn;
- $mail->subject("⛔ Server '{$this->server->name}' is unreachable");
- // $mail->view('emails.application-status-changes', [
- // 'name' => $this->application_name,
- // 'fqdn' => $fqdn,
- // 'application_url' => $this->application_url,
- // ]);
- return $mail;
- }
-
- public function toDiscord(): string
- {
- $message = '⛔ Server \'' . $this->server->name . '\' is unreachable (could be a temporary issue). If you receive this more than twice in a row, please check your server.';
- return $message;
- }
- public function toTelegram(): array
- {
- return [
- "message" => '⛔ Server \'' . $this->server->name . '\' is unreachable (could be a temporary issue). If you receive this more than twice in a row, please check your server.'
- ];
- }
-}
diff --git a/app/Notifications/Server/Unreachable.php b/app/Notifications/Server/Unreachable.php
new file mode 100644
index 000000000..ec9c11d11
--- /dev/null
+++ b/app/Notifications/Server/Unreachable.php
@@ -0,0 +1,47 @@
+subject("⛔ Server ({$this->server->name}) is unreachable after trying to connect to it 5 times");
+ $mail->view('emails.server-lost-connection', [
+ 'name' => $this->server->name,
+ ]);
+ return $mail;
+ }
+
+ public function toDiscord(): string
+ {
+ $message = "⛔ Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: You have to validate your server again after you fix the issue.";
+ return $message;
+ }
+ public function toTelegram(): array
+ {
+ return [
+ "message" => "⛔ Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: You have to validate your server again after you fix the issue."
+ ];
+ }
+}
diff --git a/resources/views/emails/container-restarted.blade.php b/resources/views/emails/container-restarted.blade.php
new file mode 100644
index 000000000..e3ec3371c
--- /dev/null
+++ b/resources/views/emails/container-restarted.blade.php
@@ -0,0 +1,11 @@
+
+
+Container ({{ $containerName }}) has been restarted automatically on {{$serverName}}, because it was stopped unexpected.
+
+@if ($containerName === 'coolify-proxy')
+Coolify Proxy should run on your server as you have FQDN set up in one of your resources. If you don't want to use Coolify Proxy, please remove FQDN from your resources.
+
+Note: The proxy should not stop unexpectedly, so please check what is going on your server.
+@endif
+
+
diff --git a/resources/views/emails/container-stopped.blade.php b/resources/views/emails/container-stopped.blade.php
new file mode 100644
index 000000000..77e8a2597
--- /dev/null
+++ b/resources/views/emails/container-stopped.blade.php
@@ -0,0 +1,9 @@
+
+
+Container {{ $containerName }} has been stopped unexpected on {{$serverName}}.
+
+@if ($url)
+Please check what is going on [here]({{ $url }}).
+@endif
+
+
diff --git a/resources/views/emails/server-lost-connection.blade.php b/resources/views/emails/server-lost-connection.blade.php
index 67aa0816a..83b5e2db5 100644
--- a/resources/views/emails/server-lost-connection.blade.php
+++ b/resources/views/emails/server-lost-connection.blade.php
@@ -1,5 +1,10 @@
-Coolify Cloud cannot connect to your server ({{$name}}). Please check your server and make sure it is running.
+
+Coolify cannot connect to your server ({{$name}}). Please check your server and make sure it is running.
+
+All automations & integrations are turned off!
+
+IMPORTANT: You have to validate your server again after you fix the issue.
If you have any questions, please contact us.