Shiloh/lasthourcloud
Shiloh/lasthourcloud
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 4 Wiki Activity

Refactor server and docker cleanup jobs

Browse Source
This commit is contained in:
Andras Bacsai 2023-11-16 14:29:01 +01:00
parent fb42c43953
commit 2b666ff121
6 changed files with 42 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
app/Console/Kernel.php
View File

@ -2,10 +2,8 @@
namespace App\Console; namespace App\Console;
use App\Jobs\CheckResaleLicenseJob;
use App\Jobs\CleanupInstanceStuffsJob; use App\Jobs\CleanupInstanceStuffsJob;
use App\Jobs\DatabaseBackupJob; use App\Jobs\DatabaseBackupJob;
use App\Jobs\DockerCleanupJob;
use App\Jobs\InstanceAutoUpdateJob; use App\Jobs\InstanceAutoUpdateJob;
use App\Jobs\ContainerStatusJob; use App\Jobs\ContainerStatusJob;
use App\Jobs\PullHelperImageJob; use App\Jobs\PullHelperImageJob;
@ -28,7 +26,6 @@ protected function schedule(Schedule $schedule): void
// Server Jobs // Server Jobs
$this->check_scheduled_backups($schedule); $this->check_scheduled_backups($schedule);
$this->check_resources($schedule); $this->check_resources($schedule);
$this->cleanup_servers($schedule);
$this->check_scheduled_backups($schedule); $this->check_scheduled_backups($schedule);
$this->pull_helper_image($schedule); $this->pull_helper_image($schedule);
} else { } else {
@ -41,7 +38,6 @@ protected function schedule(Schedule $schedule): void
$this->instance_auto_update($schedule); $this->instance_auto_update($schedule);
$this->check_scheduled_backups($schedule); $this->check_scheduled_backups($schedule);
$this->check_resources($schedule); $this->check_resources($schedule);
$this->cleanup_servers($schedule);
$this->pull_helper_image($schedule); $this->pull_helper_image($schedule);
} }
} }
@ -52,13 +48,6 @@ private function pull_helper_image($schedule)
$schedule->job(new PullHelperImageJob($server))->everyTenMinutes()->onOneServer(); $schedule->job(new PullHelperImageJob($server))->everyTenMinutes()->onOneServer();
} }
} }
private function cleanup_servers($schedule)
{
$servers = Server::all()->where('settings.is_usable', true)->where('settings.is_reachable', true);
foreach ($servers as $server) {
$schedule->job(new DockerCleanupJob($server))->everyTenMinutes()->onOneServer();
}
}
private function check_resources($schedule) private function check_resources($schedule)
{ {
if (isCloud()) { if (isCloud()) {
@ -67,8 +56,8 @@ private function check_resources($schedule)
$servers = Server::all(); $servers = Server::all();
} }
foreach ($servers as $server) { foreach ($servers as $server) {
$schedule->job(new ServerStatusJob($server))->everyTenMinutes()->onOneServer();
$schedule->job(new ContainerStatusJob($server))->everyMinute()->onOneServer(); $schedule->job(new ContainerStatusJob($server))->everyMinute()->onOneServer();
$schedule->job(new ServerStatusJob($server))->everyFiveMinutes()->onOneServer();
} }
} }
private function instance_auto_update($schedule) private function instance_auto_update($schedule)

9
app/Jobs/DockerCleanupJob.php
View File

@ -40,12 +40,13 @@ public function handle(): void
if (!$this->server->isFunctional()) { if (!$this->server->isFunctional()) {
return; return;
} }
$this->usageBefore = $this->server->getDiskUsage();
ray('Usage before: ' . $this->usageBefore);
if ($this->usageBefore >= $this->server->settings->cleanup_after_percentage) { if ($this->usageBefore >= $this->server->settings->cleanup_after_percentage) {
ray('Cleaning up ' . $this->server->name); ray('Cleaning up ' . $this->server->name);
instant_remote_process(['docker image prune -af'], $this->server); instant_remote_process(['docker image prune -af'], $this->server, false);
instant_remote_process(['docker container prune -f --filter "label=coolify.managed=true"'], $this->server); instant_remote_process(['docker container prune -f --filter "label=coolify.managed=true"'], $this->server, false);
instant_remote_process(['docker builder prune -af'], $this->server); instant_remote_process(['docker builder prune -af'], $this->server, false);
$usageAfter = $this->server->getDiskUsage(); $usageAfter = $this->server->getDiskUsage();
if ($usageAfter < $this->usageBefore) { if ($usageAfter < $this->usageBefore) {
ray('Saved ' . ($this->usageBefore - $usageAfter) . '% disk space on ' . $this->server->name); ray('Saved ' . ($this->usageBefore - $usageAfter) . '% disk space on ' . $this->server->name);

37
app/Jobs/ServerStatusJob.php
View File

@ -16,6 +16,7 @@ class ServerStatusJob implements ShouldQueue, ShouldBeEncrypted
{ {
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public ?int $disk_usage = null;
public function __construct(public Server $server) public function __construct(public Server $server)
{ {
} }
@ -34,23 +35,33 @@ public function handle(): void
ray("checking server status for {$this->server->id}"); ray("checking server status for {$this->server->id}");
try { try {
$this->server->checkServerRediness(); $this->server->checkServerRediness();
$disk_usage = $this->server->getDiskUsage(); $this->cleanup(notify: false);
if ($disk_usage >= $this->server->settings->cleanup_after_percentage) {
if ($this->server->high_disk_usage_notification_sent) {
ray('high disk usage notification already sent');
return;
}
$this->server->high_disk_usage_notification_sent = true;
$this->server->save();
$this->server->team->notify(new HighDiskUsage($this->server, $disk_usage, $this->server->settings->cleanup_after_percentage));
} else {
$this->server->high_disk_usage_notification_sent = false;
$this->server->save();
}
} catch (\Throwable $e) { } catch (\Throwable $e) {
send_internal_notification('ServerStatusJob failed with: ' . $e->getMessage()); send_internal_notification('ServerStatusJob failed with: ' . $e->getMessage());
ray($e->getMessage()); ray($e->getMessage());
handleError($e); handleError($e);
} }
} }
public function cleanup(bool $notify = false): void
{
$this->disk_usage = $this->server->getDiskUsage();
if ($this->disk_usage >= $this->server->settings->cleanup_after_percentage) {
if ($notify) {
if ($this->server->high_disk_usage_notification_sent) {
ray('high disk usage notification already sent');
return;
} else {
$this->server->high_disk_usage_notification_sent = true;
$this->server->save();
$this->server->team->notify(new HighDiskUsage($this->server, $this->disk_usage, $this->server->settings->cleanup_after_percentage));
}
} else {
DockerCleanupJob::dispatchSync($this->server);
$this->cleanup(notify: true);
}
} else {
$this->server->high_disk_usage_notification_sent = false;
$this->server->save();
}
}
} }

16
app/Models/Server.php
View File

@ -112,17 +112,19 @@ public function scopeWithProxy(): Builder
return $this->proxy->modelScope(); return $this->proxy->modelScope();
} }
public function isLocalhost() { public function isLocalhost()
if (isDev()) { {
return $this->ip === 'coolify-testing-host'; return $this->ip === 'host.docker.internal' || $this->id === 0;
}
return $this->ip === 'host.docker.internal';
} }
public function checkServerRediness() public function checkServerRediness()
{ {
$serverUptimeCheckNumber = $this->unreachable_count; $serverUptimeCheckNumber = $this->unreachable_count;
$serverUptimeCheckNumberMax = 5; $serverUptimeCheckNumberMax = 5;
while (true) {
$currentTime = now()->timestamp;
$runtime5Minutes = 5 * 60;
// Run for 5 minutes max and check every 5 seconds
while ($currentTime + $runtime5Minutes > now()->timestamp) {
if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) { if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) {
if ($this->unreachable_notification_sent === false) { if ($this->unreachable_notification_sent === false) {
ray('Server unreachable, sending notification...'); ray('Server unreachable, sending notification...');
@ -226,7 +228,7 @@ public function getIp(): Attribute
if (isDev()) { if (isDev()) {
return '127.0.0.1'; return '127.0.0.1';
} }
if ($this->ip === 'host.docker.internal') { if ($this->isLocalhost()) {
return base_ip(); return base_ip();
} }
return $this->ip; return $this->ip;

2
app/Models/ServiceDatabase.php
View File

@ -36,7 +36,7 @@ public function getServiceDatabaseUrl()
{ {
$port = $this->public_port; $port = $this->public_port;
$realIp = $this->service->server->ip; $realIp = $this->service->server->ip;
if ($realIp === 'host.docker.internal' || isDev()) { if ($this->service->server->isLocalhost() || isDev()) {
$realIp = base_ip(); $realIp = base_ip();
} }
$url = "{$realIp}:{$port}"; $url = "{$realIp}:{$port}";

4
app/Notifications/Server/HighDiskUsage.php
View File

@ -53,13 +53,13 @@ public function toMail(): MailMessage
public function toDiscord(): string public function toDiscord(): string
{ {
$message = "Coolify: Server '{$this->server->name}' high disk usage detected! \nDisk usage: {$this->disk_usage}. Please cleanup your disk to prevent data-loss. Here are some tips: https://coolify.io/docs/automated-cleanup."; $message = "Coolify: Server '{$this->server->name}' high disk usage detected!\nDisk usage: {$this->disk_usage}%. Threshold: {$this->cleanup_after_percentage}%.\nPlease cleanup your disk to prevent data-loss.\nHere are some tips: https://coolify.io/docs/automated-cleanup.";
return $message; return $message;
} }
public function toTelegram(): array public function toTelegram(): array
{ {
return [ return [
"message" => "Coolify: Server '{$this->server->name}' high disk usage detected! \n Disk usage: {$this->disk_usage}. Please cleanup your disk to prevent data-loss. Here are some tips: https://coolify.io/docs/automated-cleanup." "message" => "Coolify: Server '{$this->server->name}' high disk usage detected!\nDisk usage: {$this->disk_usage}%. Threshold: {$this->cleanup_after_percentage}%.\nPlease cleanup your disk to prevent data-loss.\nHere are some tips: https://coolify.io/docs/automated-cleanup."
]; ];
} }
} }