Shiloh/lasthourcloud
Shiloh/lasthourcloud
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 4 Wiki Activity

feat: log drainer container check

Browse Source
This commit is contained in:
Andras Bacsai 2023-11-17 21:16:25 +01:00
parent 3b20eee909
commit 689480003a
12 changed files with 140 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
app/Actions/Server/InstallLogDrain.php
View File

@ -8,8 +8,17 @@ use App\Models\Server;
class InstallLogDrain
{
use AsAction;
public function handle(Server $server, string $type)
public function handle(Server $server)
{
if ($server->settings->is_logdrain_newrelic_enabled) {
$type = 'newrelic';
} else if ($server->settings->is_logdrain_highlight_enabled) {
$type = 'highlight';
} else if ($server->settings->is_logdrain_axiom_enabled) {
$type = 'axiom';
} else {
$type = 'none';
}
try {
if ($type === 'none') {
$command = [

4
app/Console/Kernel.php
View File

@ -2,6 +2,7 @@
namespace App\Console;
use App\Jobs\CheckLogDrainContainerJob;
use App\Jobs\CleanupInstanceStuffsJob;
use App\Jobs\DatabaseBackupJob;
use App\Jobs\InstanceAutoUpdateJob;
@ -61,6 +62,9 @@ class Kernel extends ConsoleKernel
foreach ($servers as $server) {
$schedule->job(new ServerStatusJob($server))->everyTenMinutes()->onOneServer();
$schedule->job(new ContainerStatusJob($server))->everyMinute()->onOneServer();
if ($server->isLogDrainEnabled()) {
$schedule->job(new CheckLogDrainContainerJob($server))->everyMinute()->onOneServer();
}
}
}
private function instance_auto_update($schedule)

11
app/Http/Livewire/Server/LogDrains.php
View File

@ -2,6 +2,7 @@
namespace App\Http\Livewire\Server;
use App\Actions\Server\InstallLogDrain;
use App\Models\Server;
use Livewire\Component;
@ -46,14 +47,8 @@ class LogDrains extends Component
public function configureLogDrain()
{
try {
if ($this->server->settings->is_logdrain_newrelic_enabled) {
$this->server->logDrain('newrelic');
} else if ($this->server->settings->is_logdrain_highlight_enabled) {
$this->server->logDrain('highlight');
} else if ($this->server->settings->is_logdrain_axiom_enabled) {
$this->server->logDrain('axiom');
} else {
$this->server->logDrain('none');
InstallLogDrain::run($this->server);
if (!$this->server->isLogDrainEnabled()) {
$this->emit('serverRefresh');
$this->emit('success', 'Log drain service stopped.');
return;

83
app/Jobs/CheckLogDrainContainerJob.php Normal file
View File

@ -0,0 +1,83 @@
<?php
namespace App\Jobs;
use App\Actions\Server\InstallLogDrain;
use App\Models\Server;
use App\Notifications\Container\ContainerRestarted;
use App\Notifications\Container\ContainerStopped;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Sleep;
class CheckLogDrainContainerJob implements ShouldQueue, ShouldBeEncrypted
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public function __construct(public Server $server)
{
}
public function middleware(): array
{
return [(new WithoutOverlapping($this->server->id))->dontRelease()];
}
public function uniqueId(): int
{
return $this->server->id;
}
public function healthcheck()
{
$status = instant_remote_process(["docker inspect --format='{{json .State.Status}}' coolify-log-drain"], $this->server, false);
if (str($status)->contains('running')) {
return true;
} else {
return false;
}
}
public function handle(): void
{
// ray("checking log drain statuses for {$this->server->id}");
try {
if (!$this->server->isServerReady()) {
return;
};
$containers = instant_remote_process(["docker container ls -q"], $this->server);
if (!$containers) {
return;
}
$containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this->server);
$containers = format_docker_command_output_to_json($containers);
$foundLogDrainContainer = $containers->filter(function ($value, $key) {
return data_get($value, 'Name') === '/coolify-log-drain';
})->first();
if (!$foundLogDrainContainer || !$this->healthcheck()) {
ray('Log drain container not found or unhealthy. Restarting...');
InstallLogDrain::run($this->server);
Sleep::for(10)->seconds();
if ($this->healthcheck()) {
if ($this->server->log_drain_notification_sent) {
$this->server->team->notify(new ContainerRestarted('Coolify Log Drainer', $this->server));
$this->server->update(['log_drain_notification_sent' => false]);
}
return;
}
if (!$this->server->log_drain_notification_sent) {
ray('Log drain container still unhealthy. Sending notification...');
$this->server->team->notify(new ContainerStopped('Coolify Log Drainer', $this->server, null));
$this->server->update(['log_drain_notification_sent' => true]);
}
}
} catch (\Throwable $e) {
send_internal_notification("CheckLogDrainContainerJob failed on ({$this->server->id}) with: " . $e->getMessage());
ray($e->getMessage());
handleError($e);
}
}
}

48
app/Jobs/ContainerStatusJob.php
View File

@ -52,29 +52,7 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted
$databases = $this->server->databases();
$services = $this->server->services()->get();
$previews = $this->server->previews();
$this->server->proxyType();
/// Check if proxy is running
$foundProxyContainer = $containers->filter(function ($value, $key) {
return data_get($value, 'Name') === '/coolify-proxy';
})->first();
if (!$foundProxyContainer) {
try {
$shouldStart = CheckProxy::run($this->server);
if ($shouldStart) {
StartProxy::run($this->server, false);
$this->server->team->notify(new ContainerRestarted('coolify-proxy', $this->server));
} else {
ray('Proxy could not be started.');
}
} catch (\Throwable $e) {
ray($e);
}
} else {
$this->server->proxy->status = data_get($foundProxyContainer, 'State.Status');
$this->server->save();
$connectProxyToDockerNetworks = connectProxyToNetworks($this->server);
instant_remote_process($connectProxyToDockerNetworks, $this->server, false);
}
$foundApplications = [];
$foundApplicationPreviews = [];
$foundDatabases = [];
@ -267,6 +245,30 @@ class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted
}
$this->server->team->notify(new ContainerStopped($containerName, $this->server, $url));
}
// Check if proxy is running
$this->server->proxyType();
$foundProxyContainer = $containers->filter(function ($value, $key) {
return data_get($value, 'Name') === '/coolify-proxy';
})->first();
if (!$foundProxyContainer) {
try {
$shouldStart = CheckProxy::run($this->server);
if ($shouldStart) {
StartProxy::run($this->server, false);
$this->server->team->notify(new ContainerRestarted('coolify-proxy', $this->server));
} else {
ray('Proxy could not be started.');
}
} catch (\Throwable $e) {
ray($e);
}
} else {
$this->server->proxy->status = data_get($foundProxyContainer, 'State.Status');
$this->server->save();
$connectProxyToDockerNetworks = connectProxyToNetworks($this->server);
instant_remote_process($connectProxyToDockerNetworks, $this->server, false);
}
} catch (\Throwable $e) {
send_internal_notification("ContainerStatusJob failed on ({$this->server->id}) with: " . $e->getMessage());
ray($e->getMessage());

4
app/Models/Server.php
View File

@ -296,10 +296,6 @@ class Server extends BaseModel
// }
return true;
}
public function logDrain($type)
{
InstallLogDrain::run($this, $type);
}
public function isFunctional()
{
return $this->settings->is_reachable && $this->settings->is_usable;

6
app/Notifications/Container/ContainerRestarted.php
View File

@ -27,7 +27,7 @@ class ContainerRestarted extends Notification implements ShouldQueue
public function toMail(): MailMessage
{
$mail = new MailMessage();
$mail->subject("Coolify: Container ({$this->name}) has been restarted automatically on {$this->server->name}");
$mail->subject("Coolify: A service ({$this->name}) has been restarted automatically on {$this->server->name}");
$mail->view('emails.container-restarted', [
'containerName' => $this->name,
'serverName' => $this->server->name,
@ -38,12 +38,12 @@ class ContainerRestarted extends Notification implements ShouldQueue
public function toDiscord(): string
{
$message = "Coolify: Container ({$this->name}) has been restarted automatically on {$this->server->name}";
$message = "Coolify: A service ({$this->name}) has been restarted automatically on {$this->server->name}";
return $message;
}
public function toTelegram(): array
{
$message = "Coolify: Container ({$this->name}) has been restarted automatically on {$this->server->name}";
$message = "Coolify: A service ({$this->name}) has been restarted automatically on {$this->server->name}";
$payload = [
"message" => $message,
];

6
app/Notifications/Container/ContainerStopped.php
View File

@ -26,7 +26,7 @@ class ContainerStopped extends Notification implements ShouldQueue
public function toMail(): MailMessage
{
$mail = new MailMessage();
$mail->subject("Coolify: Container ({$this->name}) has been stopped on {$this->server->name}");
$mail->subject("Coolify: A service ({$this->name}) has been stopped on {$this->server->name}");
$mail->view('emails.container-stopped', [
'containerName' => $this->name,
'serverName' => $this->server->name,
@ -37,12 +37,12 @@ class ContainerStopped extends Notification implements ShouldQueue
public function toDiscord(): string
{
$message = "Coolify: Container ({$this->name}) has been stopped on {$this->server->name}";
$message = "Coolify: A service ({$this->name}) has been stopped on {$this->server->name}";
return $message;
}
public function toTelegram(): array
{
$message = "Coolify: Container ($this->name} has been stopped on {$this->server->name}";
$message = "Coolify: A service ($this->name} has been stopped on {$this->server->name}";
$payload = [
"message" => $message,
];

8
database/migrations/2023_11_17_160437_add_drain_log_enable_by_service.php
View File

@ -35,7 +35,9 @@ return new class extends Migration
Schema::table('service_databases', function (Blueprint $table) {
$table->boolean('is_log_drain_enabled')->default(false);
});
Schema::table('servers', function (Blueprint $table) {
$table->boolean('log_drain_notification_sent')->default(false);
});
}
/**
@ -67,6 +69,8 @@ return new class extends Migration
Schema::table('service_databases', function (Blueprint $table) {
$table->dropColumn('is_log_drain_enabled');
});
Schema::table('servers', function (Blueprint $table) {
$table->dropColumn('log_drain_notification_sent');
});
}
};

2
resources/views/emails/container-restarted.blade.php
View File

@ -1,6 +1,6 @@
<x-emails.layout>
Container ({{ $containerName }}) has been restarted automatically on {{$serverName}}, because it was stopped unexpectedly.
A service ({{ $containerName }}) has been restarted automatically on {{$serverName}}, because it was stopped unexpectedly.
@if ($containerName === 'coolify-proxy')
Coolify Proxy should run on your server as you have FQDNs set up in one of your resources.

2
resources/views/emails/container-stopped.blade.php
View File

@ -1,6 +1,6 @@
<x-emails.layout>
Container {{ $containerName }} has been stopped unexpectedly on {{$serverName}}.
A service ({{ $containerName }}) has been stopped unexpectedly on {{$serverName}}.
@if ($url)
Please check what is going on [here]({{ $url }}).

2
resources/views/livewire/server/log-drains.blade.php
View File

@ -1,7 +1,7 @@
<div>
<x-server.navbar :server="$server" :parameters="$parameters" />
<h2>Log Drains</h2>
<div class="pb-4">Sends resource logs to external services.</div>
<div class="pb-4">Sends service logs to 3rd party tools.</div>
<div class="flex flex-col gap-4 pt-4">
<div class="p-4 border border-coolgray-500">
<form wire:submit.prevent='submit("newrelic")' class="flex flex-col">