diff --git a/app/Actions/Server/InstallLogDrain.php b/app/Actions/Server/InstallLogDrain.php index 4db355aa8..11487d633 100644 --- a/app/Actions/Server/InstallLogDrain.php +++ b/app/Actions/Server/InstallLogDrain.php @@ -8,8 +8,17 @@ use App\Models\Server; class InstallLogDrain { use AsAction; - public function handle(Server $server, string $type) + public function handle(Server $server) { + if ($server->settings->is_logdrain_newrelic_enabled) { + $type = 'newrelic'; + } else if ($server->settings->is_logdrain_highlight_enabled) { + $type = 'highlight'; + } else if ($server->settings->is_logdrain_axiom_enabled) { + $type = 'axiom'; + } else { + $type = 'none'; + } try { if ($type === 'none') { $command = [ diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index fa195aafd..e24d657a9 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -2,6 +2,7 @@ namespace App\Console; +use App\Jobs\CheckLogDrainContainerJob; use App\Jobs\CleanupInstanceStuffsJob; use App\Jobs\DatabaseBackupJob; use App\Jobs\InstanceAutoUpdateJob; @@ -61,6 +62,9 @@ class Kernel extends ConsoleKernel foreach ($servers as $server) { $schedule->job(new ServerStatusJob($server))->everyTenMinutes()->onOneServer(); $schedule->job(new ContainerStatusJob($server))->everyMinute()->onOneServer(); + if ($server->isLogDrainEnabled()) { + $schedule->job(new CheckLogDrainContainerJob($server))->everyMinute()->onOneServer(); + } } } private function instance_auto_update($schedule) diff --git a/app/Http/Livewire/Server/LogDrains.php b/app/Http/Livewire/Server/LogDrains.php index cb7a5138a..056beb584 100644 --- a/app/Http/Livewire/Server/LogDrains.php +++ b/app/Http/Livewire/Server/LogDrains.php @@ -2,6 +2,7 @@ namespace App\Http\Livewire\Server; +use App\Actions\Server\InstallLogDrain; use App\Models\Server; use Livewire\Component; @@ -46,14 +47,8 @@ class LogDrains extends Component public function configureLogDrain() { try { - if ($this->server->settings->is_logdrain_newrelic_enabled) { - $this->server->logDrain('newrelic'); - } else if ($this->server->settings->is_logdrain_highlight_enabled) { - $this->server->logDrain('highlight'); - } else if ($this->server->settings->is_logdrain_axiom_enabled) { - $this->server->logDrain('axiom'); - } else { - $this->server->logDrain('none'); + InstallLogDrain::run($this->server); + if (!$this->server->isLogDrainEnabled()) { $this->emit('serverRefresh'); $this->emit('success', 'Log drain service stopped.'); return; diff --git a/app/Jobs/CheckLogDrainContainerJob.php b/app/Jobs/CheckLogDrainContainerJob.php new file mode 100644 index 000000000..2bf59913a --- /dev/null +++ b/app/Jobs/CheckLogDrainContainerJob.php @@ -0,0 +1,83 @@ +server->id))->dontRelease()]; + } + + public function uniqueId(): int + { + return $this->server->id; + } + public function healthcheck() + { + $status = instant_remote_process(["docker inspect --format='{{json .State.Status}}' coolify-log-drain"], $this->server, false); + if (str($status)->contains('running')) { + return true; + } else { + return false; + } + } + public function handle(): void + { + // ray("checking log drain statuses for {$this->server->id}"); + try { + if (!$this->server->isServerReady()) { + return; + }; + $containers = instant_remote_process(["docker container ls -q"], $this->server); + if (!$containers) { + return; + } + $containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this->server); + $containers = format_docker_command_output_to_json($containers); + + $foundLogDrainContainer = $containers->filter(function ($value, $key) { + return data_get($value, 'Name') === '/coolify-log-drain'; + })->first(); + if (!$foundLogDrainContainer || !$this->healthcheck()) { + ray('Log drain container not found or unhealthy. Restarting...'); + InstallLogDrain::run($this->server); + Sleep::for(10)->seconds(); + if ($this->healthcheck()) { + if ($this->server->log_drain_notification_sent) { + $this->server->team->notify(new ContainerRestarted('Coolify Log Drainer', $this->server)); + $this->server->update(['log_drain_notification_sent' => false]); + } + return; + } + if (!$this->server->log_drain_notification_sent) { + ray('Log drain container still unhealthy. Sending notification...'); + $this->server->team->notify(new ContainerStopped('Coolify Log Drainer', $this->server, null)); + $this->server->update(['log_drain_notification_sent' => true]); + } + } + } catch (\Throwable $e) { + send_internal_notification("CheckLogDrainContainerJob failed on ({$this->server->id}) with: " . $e->getMessage()); + ray($e->getMessage()); + handleError($e); + } + } +} diff --git a/app/Jobs/ContainerStatusJob.php b/app/Jobs/ContainerStatusJob.php index 1c66ef53f..34678b18a 100644 --- a/app/Jobs/ContainerStatusJob.php +++ b/app/Jobs/ContainerStatusJob.php @@ -52,29 +52,7 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted $databases = $this->server->databases(); $services = $this->server->services()->get(); $previews = $this->server->previews(); - $this->server->proxyType(); - /// Check if proxy is running - $foundProxyContainer = $containers->filter(function ($value, $key) { - return data_get($value, 'Name') === '/coolify-proxy'; - })->first(); - if (!$foundProxyContainer) { - try { - $shouldStart = CheckProxy::run($this->server); - if ($shouldStart) { - StartProxy::run($this->server, false); - $this->server->team->notify(new ContainerRestarted('coolify-proxy', $this->server)); - } else { - ray('Proxy could not be started.'); - } - } catch (\Throwable $e) { - ray($e); - } - } else { - $this->server->proxy->status = data_get($foundProxyContainer, 'State.Status'); - $this->server->save(); - $connectProxyToDockerNetworks = connectProxyToNetworks($this->server); - instant_remote_process($connectProxyToDockerNetworks, $this->server, false); - } + $foundApplications = []; $foundApplicationPreviews = []; $foundDatabases = []; @@ -267,6 +245,30 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted } $this->server->team->notify(new ContainerStopped($containerName, $this->server, $url)); } + + // Check if proxy is running + $this->server->proxyType(); + $foundProxyContainer = $containers->filter(function ($value, $key) { + return data_get($value, 'Name') === '/coolify-proxy'; + })->first(); + if (!$foundProxyContainer) { + try { + $shouldStart = CheckProxy::run($this->server); + if ($shouldStart) { + StartProxy::run($this->server, false); + $this->server->team->notify(new ContainerRestarted('coolify-proxy', $this->server)); + } else { + ray('Proxy could not be started.'); + } + } catch (\Throwable $e) { + ray($e); + } + } else { + $this->server->proxy->status = data_get($foundProxyContainer, 'State.Status'); + $this->server->save(); + $connectProxyToDockerNetworks = connectProxyToNetworks($this->server); + instant_remote_process($connectProxyToDockerNetworks, $this->server, false); + } } catch (\Throwable $e) { send_internal_notification("ContainerStatusJob failed on ({$this->server->id}) with: " . $e->getMessage()); ray($e->getMessage()); diff --git a/app/Models/Server.php b/app/Models/Server.php index 9d91ea123..02c3186c6 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -296,10 +296,6 @@ class Server extends BaseModel // } return true; } - public function logDrain($type) - { - InstallLogDrain::run($this, $type); - } public function isFunctional() { return $this->settings->is_reachable && $this->settings->is_usable; diff --git a/app/Notifications/Container/ContainerRestarted.php b/app/Notifications/Container/ContainerRestarted.php index dc55c05d2..723cad8f6 100644 --- a/app/Notifications/Container/ContainerRestarted.php +++ b/app/Notifications/Container/ContainerRestarted.php @@ -27,7 +27,7 @@ class ContainerRestarted extends Notification implements ShouldQueue public function toMail(): MailMessage { $mail = new MailMessage(); - $mail->subject("Coolify: Container ({$this->name}) has been restarted automatically on {$this->server->name}"); + $mail->subject("Coolify: A service ({$this->name}) has been restarted automatically on {$this->server->name}"); $mail->view('emails.container-restarted', [ 'containerName' => $this->name, 'serverName' => $this->server->name, @@ -38,12 +38,12 @@ class ContainerRestarted extends Notification implements ShouldQueue public function toDiscord(): string { - $message = "Coolify: Container ({$this->name}) has been restarted automatically on {$this->server->name}"; + $message = "Coolify: A service ({$this->name}) has been restarted automatically on {$this->server->name}"; return $message; } public function toTelegram(): array { - $message = "Coolify: Container ({$this->name}) has been restarted automatically on {$this->server->name}"; + $message = "Coolify: A service ({$this->name}) has been restarted automatically on {$this->server->name}"; $payload = [ "message" => $message, ]; diff --git a/app/Notifications/Container/ContainerStopped.php b/app/Notifications/Container/ContainerStopped.php index 4518698fb..ea474a3e9 100644 --- a/app/Notifications/Container/ContainerStopped.php +++ b/app/Notifications/Container/ContainerStopped.php @@ -26,7 +26,7 @@ class ContainerStopped extends Notification implements ShouldQueue public function toMail(): MailMessage { $mail = new MailMessage(); - $mail->subject("Coolify: Container ({$this->name}) has been stopped on {$this->server->name}"); + $mail->subject("Coolify: A service ({$this->name}) has been stopped on {$this->server->name}"); $mail->view('emails.container-stopped', [ 'containerName' => $this->name, 'serverName' => $this->server->name, @@ -37,12 +37,12 @@ class ContainerStopped extends Notification implements ShouldQueue public function toDiscord(): string { - $message = "Coolify: Container ({$this->name}) has been stopped on {$this->server->name}"; + $message = "Coolify: A service ({$this->name}) has been stopped on {$this->server->name}"; return $message; } public function toTelegram(): array { - $message = "Coolify: Container ($this->name} has been stopped on {$this->server->name}"; + $message = "Coolify: A service ($this->name} has been stopped on {$this->server->name}"; $payload = [ "message" => $message, ]; diff --git a/database/migrations/2023_11_17_160437_add_drain_log_enable_by_service.php b/database/migrations/2023_11_17_160437_add_drain_log_enable_by_service.php index f68a268b1..8d0853be3 100644 --- a/database/migrations/2023_11_17_160437_add_drain_log_enable_by_service.php +++ b/database/migrations/2023_11_17_160437_add_drain_log_enable_by_service.php @@ -35,7 +35,9 @@ return new class extends Migration Schema::table('service_databases', function (Blueprint $table) { $table->boolean('is_log_drain_enabled')->default(false); }); - + Schema::table('servers', function (Blueprint $table) { + $table->boolean('log_drain_notification_sent')->default(false); + }); } /** @@ -67,6 +69,8 @@ return new class extends Migration Schema::table('service_databases', function (Blueprint $table) { $table->dropColumn('is_log_drain_enabled'); }); - + Schema::table('servers', function (Blueprint $table) { + $table->dropColumn('log_drain_notification_sent'); + }); } }; diff --git a/resources/views/emails/container-restarted.blade.php b/resources/views/emails/container-restarted.blade.php index ebc7f99fb..085aafa53 100644 --- a/resources/views/emails/container-restarted.blade.php +++ b/resources/views/emails/container-restarted.blade.php @@ -1,6 +1,6 @@ -Container ({{ $containerName }}) has been restarted automatically on {{$serverName}}, because it was stopped unexpectedly. +A service ({{ $containerName }}) has been restarted automatically on {{$serverName}}, because it was stopped unexpectedly. @if ($containerName === 'coolify-proxy') Coolify Proxy should run on your server as you have FQDNs set up in one of your resources. diff --git a/resources/views/emails/container-stopped.blade.php b/resources/views/emails/container-stopped.blade.php index 575e740b2..bf7cc086a 100644 --- a/resources/views/emails/container-stopped.blade.php +++ b/resources/views/emails/container-stopped.blade.php @@ -1,6 +1,6 @@ -Container {{ $containerName }} has been stopped unexpectedly on {{$serverName}}. +A service ({{ $containerName }}) has been stopped unexpectedly on {{$serverName}}. @if ($url) Please check what is going on [here]({{ $url }}). diff --git a/resources/views/livewire/server/log-drains.blade.php b/resources/views/livewire/server/log-drains.blade.php index 60f51665c..2e1109dc7 100644 --- a/resources/views/livewire/server/log-drains.blade.php +++ b/resources/views/livewire/server/log-drains.blade.php @@ -1,7 +1,7 @@

Log Drains

-
Sends resource logs to external services.
+
Sends service logs to 3rd party tools.