Shiloh/lasthourcloud
Shiloh/lasthourcloud
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 4 Wiki Activity

Update server status check and notifications

Browse Source
This commit is contained in:
Andras Bacsai 2023-11-16 11:53:37 +01:00
parent f5de21a343
commit 7cec6330cf
13 changed files with 254 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/Console/Kernel.php
View File

@ -9,6 +9,7 @@
use App\Jobs\InstanceAutoUpdateJob;
use App\Jobs\ContainerStatusJob;
use App\Jobs\PullHelperImageJob;
use App\Jobs\ServerStatusJob;
use App\Models\InstanceSettings;
use App\Models\ScheduledDatabaseBackup;
use App\Models\Server;
@ -67,6 +68,7 @@ private function check_resources($schedule)
}
foreach ($servers as $server) {
$schedule->job(new ContainerStatusJob($server))->everyMinute()->onOneServer();
$schedule->job(new ServerStatusJob($server))->everyFiveMinutes()->onOneServer();
}
}
private function instance_auto_update($schedule)

71
app/Jobs/ContainerStatusJob.php
View File

@ -41,76 +41,7 @@ public function handle(): void
{
// ray("checking server status for {$this->server->id}");
try {
// ray()->clearAll();
$serverUptimeCheckNumber = $this->server->unreachable_count;
$serverUptimeCheckNumberMax = 3;
// ray('checking # ' . $serverUptimeCheckNumber);
if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) {
if ($this->server->unreachable_email_sent === false) {
ray('Server unreachable, sending notification...');
$this->server->team->notify(new Unreachable($this->server));
$this->server->update(['unreachable_email_sent' => true]);
}
$this->server->settings()->update([
'is_reachable' => false,
]);
$this->server->update([
'unreachable_count' => 0,
]);
// Update all applications, databases and services to exited
foreach ($this->server->applications() as $application) {
$application->update(['status' => 'exited']);
}
foreach ($this->server->databases() as $database) {
$database->update(['status' => 'exited']);
}
foreach ($this->server->services() as $service) {
$apps = $service->applications()->get();
$dbs = $service->databases()->get();
foreach ($apps as $app) {
$app->update(['status' => 'exited']);
}
foreach ($dbs as $db) {
$db->update(['status' => 'exited']);
}
}
return;
}
$result = $this->server->validateConnection();
if ($result) {
$this->server->settings()->update([
'is_reachable' => true,
]);
$this->server->update([
'unreachable_count' => 0,
]);
} else {
$serverUptimeCheckNumber++;
$this->server->settings()->update([
'is_reachable' => false,
]);
$this->server->update([
'unreachable_count' => $serverUptimeCheckNumber,
]);
return;
}
if (data_get($this->server, 'unreachable_email_sent') === true) {
ray('Server is reachable again, sending notification...');
$this->server->team->notify(new Revived($this->server));
$this->server->update(['unreachable_email_sent' => false]);
}
if (
data_get($this->server, 'settings.is_reachable') === false ||
data_get($this->server, 'settings.is_usable') === false
) {
$this->server->settings()->update([
'is_reachable' => true,
'is_usable' => true
]);
}
// $this->server->validateDockerEngine(true);
$this->server->checkServerRediness();
$containers = instant_remote_process(["docker container ls -q"], $this->server);
if (!$containers) {
return;

32
app/Jobs/DockerCleanupJob.php
View File

@ -3,6 +3,7 @@
namespace App\Jobs;
use App\Models\Server;
use App\Notifications\Server\HighDiskUsage;
use Exception;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
@ -18,7 +19,6 @@ class DockerCleanupJob implements ShouldQueue, ShouldBeEncrypted
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $timeout = 300;
public ?string $dockerRootFilesystem = null;
public ?int $usageBefore = null;
public function __construct(public Server $server)
@ -26,28 +26,27 @@ public function __construct(public Server $server)
}
public function handle(): void
{
$isInprogress = false;
$this->server->applications()->each(function ($application) use (&$isInprogress) {
if ($application->isDeploymentInprogress()) {
$isInprogress = true;
return;
}
});
if ($isInprogress) {
throw new Exception('DockerCleanupJob: ApplicationDeploymentQueue is not empty, skipping...');
}
try {
$isInprogress = false;
$this->server->applications()->each(function ($application) use (&$isInprogress) {
if ($application->isDeploymentInprogress()) {
$isInprogress = true;
return;
}
});
if ($isInprogress) {
throw new Exception('DockerCleanupJob: ApplicationDeploymentQueue is not empty, skipping...');
}
if (!$this->server->isFunctional()) {
return;
}
$this->dockerRootFilesystem = "/";
$this->usageBefore = $this->getFilesystemUsage();
if ($this->usageBefore >= $this->server->settings->cleanup_after_percentage) {
ray('Cleaning up ' . $this->server->name);
instant_remote_process(['docker image prune -af'], $this->server);
instant_remote_process(['docker container prune -f --filter "label=coolify.managed=true"'], $this->server);
instant_remote_process(['docker builder prune -af'], $this->server);
$usageAfter = $this->getFilesystemUsage();
$usageAfter = $this->server->getDiskUsage();
if ($usageAfter < $this->usageBefore) {
ray('Saved ' . ($this->usageBefore - $usageAfter) . '% disk space on ' . $this->server->name);
send_internal_notification('DockerCleanupJob done: Saved ' . ($this->usageBefore - $usageAfter) . '% disk space on ' . $this->server->name);
@ -65,9 +64,4 @@ public function handle(): void
throw $e;
}
}
private function getFilesystemUsage()
{
return instant_remote_process(["df '{$this->dockerRootFilesystem}'| tail -1 | awk '{ print $5}' | sed 's/%//g'"], $this->server, false);
}
}

54
app/Jobs/ServerStatusJob.php Normal file
View File

@ -0,0 +1,54 @@
<?php
namespace App\Jobs;
use App\Models\Server;
use App\Notifications\Server\HighDiskUsage;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
class ServerStatusJob implements ShouldQueue, ShouldBeEncrypted
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public function __construct(public Server $server)
{
}
public function middleware(): array
{
return [(new WithoutOverlapping($this->server->id))->dontRelease()];
}
public function uniqueId(): int
{
return $this->server->id;
}
public function handle(): void
{
ray("checking server status for {$this->server->id}");
try {
$this->server->checkServerRediness();
$disk_usage = $this->server->getDiskUsage();
ray($this->server->settings->cleanup_after_percentage);
if ($disk_usage >= $this->server->settings->cleanup_after_percentage) {
$this->server->high_disk_usage_notification_sent = true;
$this->server->save();
$this->server->team->notify(new HighDiskUsage($this->server, $disk_usage, $this->server->settings->cleanup_after_percentage));
} else {
$this->server->high_disk_usage_notification_sent = false;
$this->server->save();
}
} catch (\Throwable $e) {
send_internal_notification('ServerStatusJob failed with: ' . $e->getMessage());
ray($e->getMessage());
handleError($e);
}
}
}

72
app/Models/Server.php
View File

@ -4,8 +4,11 @@
use App\Enums\ProxyStatus;
use App\Enums\ProxyTypes;
use App\Notifications\Server\Revived;
use App\Notifications\Server\Unreachable;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Support\Sleep;
use Spatie\SchemalessAttributes\Casts\SchemalessAttributes;
use Spatie\SchemalessAttributes\SchemalessAttributesTrait;
use Illuminate\Support\Str;
@ -109,6 +112,75 @@ public function scopeWithProxy(): Builder
return $this->proxy->modelScope();
}
public function checkServerRediness()
{
$serverUptimeCheckNumber = $this->unreachable_count;
$serverUptimeCheckNumberMax = 5;
while (true) {
if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) {
if ($this->unreachable_notification_sent === false) {
ray('Server unreachable, sending notification...');
$this->team->notify(new Unreachable($this));
$this->update(['unreachable_notification_sent' => true]);
}
$this->settings()->update([
'is_reachable' => false,
]);
$this->update([
'unreachable_count' => 0,
]);
foreach ($this->applications() as $application) {
$application->update(['status' => 'exited']);
}
foreach ($this->databases() as $database) {
$database->update(['status' => 'exited']);
}
foreach ($this->services() as $service) {
$apps = $service->applications()->get();
$dbs = $service->databases()->get();
foreach ($apps as $app) {
$app->update(['status' => 'exited']);
}
foreach ($dbs as $db) {
$db->update(['status' => 'exited']);
}
}
throw new \Exception('Server is not reachable.');
}
$result = $this->validateConnection();
ray('validateConnection: ' . $result);
if (!$result) {
$serverUptimeCheckNumber++;
$this->update([
'unreachable_count' => $serverUptimeCheckNumber,
]);
Sleep::for(5)->seconds();
return;
}
$this->update([
'unreachable_count' => 0,
]);
if (data_get($this, 'unreachable_notification_sent') === true) {
ray('Server is reachable again, sending notification...');
$this->team->notify(new Revived($this));
$this->update(['unreachable_notification_sent' => false]);
}
if (
data_get($this, 'settings.is_reachable') === false ||
data_get($this, 'settings.is_usable') === false
) {
$this->settings()->update([
'is_reachable' => true,
'is_usable' => true
]);
}
break;
}
}
public function getDiskUsage()
{
return instant_remote_process(["df /| tail -1 | awk '{ print $5}' | sed 's/%//g'"], $this, false);
}
public function hasDefinedResources()
{
$applications = $this->applications()->count() > 0;

68
app/Notifications/Server/HighDiskUsage.php Normal file
View File

@ -0,0 +1,68 @@
<?php
namespace App\Notifications\Server;
use App\Models\Server;
use Illuminate\Bus\Queueable;
use App\Notifications\Channels\DiscordChannel;
use App\Notifications\Channels\EmailChannel;
use App\Notifications\Channels\TelegramChannel;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Notifications\Messages\MailMessage;
use Illuminate\Notifications\Notification;
class HighDiskUsage extends Notification implements ShouldQueue
{
use Queueable;
public $tries = 1;
public function __construct(public Server $server, public int $disk_usage, public int $cleanup_after_percentage)
{
if ($this->server->high_disk_usage_notification_sent === false) {
return;
}
}
public function via(object $notifiable): array
{
$channels = [];
$isEmailEnabled = isEmailEnabled($notifiable);
$isDiscordEnabled = data_get($notifiable, 'discord_enabled');
$isTelegramEnabled = data_get($notifiable, 'telegram_enabled');
if ($isDiscordEnabled) {
$channels[] = DiscordChannel::class;
}
if ($isEmailEnabled) {
$channels[] = EmailChannel::class;
}
if ($isTelegramEnabled) {
$channels[] = TelegramChannel::class;
}
return $channels;
}
public function toMail(): MailMessage
{
$mail = new MailMessage();
$mail->subject("Coolify: Server ({$this->server->name}) high disk usage detected!");
$mail->view('emails.high-disk-usage', [
'name' => $this->server->name,
'disk_usage' => $this->disk_usage,
'threshold' => $this->cleanup_after_percentage,
]);
return $mail;
}
public function toDiscord(): string
{
$message = "Coolify: Server '{$this->server->name}' high disk usage detected! \nDisk usage: {$this->disk_usage}";
return $message;
}
public function toTelegram(): array
{
return [
"message" => "Coolify: Server '{$this->server->name}' high disk usage detected! \n Disk usage: {$this->disk_usage}"
];
}
}

2
app/Notifications/Server/Revived.php
View File

@ -18,7 +18,7 @@ class Revived extends Notification implements ShouldQueue
public $tries = 1;
public function __construct(public Server $server)
{
if ($this->server->unreachable_email_sent === false) {
if ($this->server->unreachable_notification_sent === false) {
return;
}
}

6
bootstrap/helpers/remoteProcess.php
View File

@ -191,7 +191,7 @@ function refresh_server_connection(?PrivateKey $private_key = null)
// if (!$uptime) {
// $server->settings->is_reachable = false;
// $server->team->notify(new Unreachable($server));
// $server->unreachable_email_sent = true;
// $server->unreachable_notification_sent = true;
// $server->save();
// return [
// "uptime" => null,
@ -213,9 +213,9 @@ function refresh_server_connection(?PrivateKey $private_key = null)
// $server->settings->is_usable = false;
// } else {
// $server->settings->is_usable = true;
// if (data_get($server, 'unreachable_email_sent') === true) {
// if (data_get($server, 'unreachable_notification_sent') === true) {
// $server->team->notify(new Revived($server));
// $server->unreachable_email_sent = false;
// $server->unreachable_notification_sent = false;
// $server->save();
// }
// }

2
config/sentry.php
View File

@ -7,7 +7,7 @@
// The release version of your application
// Example with dynamic git hash: trim(exec('git --git-dir ' . base_path('.git') . ' log --pretty="%h" -n1 HEAD'))
'release' => '4.0.0-beta.136',
'release' => '4.0.0-beta.137',
// When left empty or `null` the Laravel environment will be used
'environment' => config('app.env'),

2
config/version.php
View File

@ -1,3 +1,3 @@
<?php
return '4.0.0-beta.136';
return '4.0.0-beta.137';

30
database/migrations/2023_11_16_101819_add_high_disk_usage_notification.php Normal file
View File

@ -0,0 +1,30 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('servers', function (Blueprint $table) {
$table->boolean('high_disk_usage_notification_sent')->default(false);
$table->renameColumn('unreachable_email_sent', 'unreachable_notification_sent');
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('servers', function (Blueprint $table) {
$table->dropColumn('high_disk_usage_notification_sent');
$table->renameColumn('unreachable_notification_sent', 'unreachable_email_sent');
});
}
};

7
resources/views/emails/high-disk-usage.blade.php Normal file
View File

@ -0,0 +1,7 @@
<x-emails.layout>
Your server ({{ $name }}) has high disk usage ({{ $disk_usage }}%).
Threshold is {{ $threshold }}% (you can change it in the Server Settings menu).
</x-emails.layout>

2
versions.json
View File

@ -4,7 +4,7 @@
"version": "3.12.36"
},
"v4": {
"version": "4.0.0-beta.136"
"version": "4.0.0-beta.137"
}
}
}