diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 3439630d5..e0d9f2752 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -2,10 +2,8 @@ namespace App\Console; -use App\Jobs\CheckResaleLicenseJob; use App\Jobs\CleanupInstanceStuffsJob; use App\Jobs\DatabaseBackupJob; -use App\Jobs\DockerCleanupJob; use App\Jobs\InstanceAutoUpdateJob; use App\Jobs\ContainerStatusJob; use App\Jobs\PullHelperImageJob; @@ -28,7 +26,6 @@ protected function schedule(Schedule $schedule): void // Server Jobs $this->check_scheduled_backups($schedule); $this->check_resources($schedule); - $this->cleanup_servers($schedule); $this->check_scheduled_backups($schedule); $this->pull_helper_image($schedule); } else { @@ -41,7 +38,6 @@ protected function schedule(Schedule $schedule): void $this->instance_auto_update($schedule); $this->check_scheduled_backups($schedule); $this->check_resources($schedule); - $this->cleanup_servers($schedule); $this->pull_helper_image($schedule); } } @@ -52,13 +48,6 @@ private function pull_helper_image($schedule) $schedule->job(new PullHelperImageJob($server))->everyTenMinutes()->onOneServer(); } } - private function cleanup_servers($schedule) - { - $servers = Server::all()->where('settings.is_usable', true)->where('settings.is_reachable', true); - foreach ($servers as $server) { - $schedule->job(new DockerCleanupJob($server))->everyTenMinutes()->onOneServer(); - } - } private function check_resources($schedule) { if (isCloud()) { @@ -67,8 +56,8 @@ private function check_resources($schedule) $servers = Server::all(); } foreach ($servers as $server) { + $schedule->job(new ServerStatusJob($server))->everyTenMinutes()->onOneServer(); $schedule->job(new ContainerStatusJob($server))->everyMinute()->onOneServer(); - $schedule->job(new ServerStatusJob($server))->everyFiveMinutes()->onOneServer(); } } private function instance_auto_update($schedule) diff --git a/app/Jobs/DockerCleanupJob.php b/app/Jobs/DockerCleanupJob.php index 081c1d863..14ca11b22 100644 --- a/app/Jobs/DockerCleanupJob.php +++ b/app/Jobs/DockerCleanupJob.php @@ -40,12 +40,13 @@ public function handle(): void if (!$this->server->isFunctional()) { return; } - + $this->usageBefore = $this->server->getDiskUsage(); + ray('Usage before: ' . $this->usageBefore); if ($this->usageBefore >= $this->server->settings->cleanup_after_percentage) { ray('Cleaning up ' . $this->server->name); - instant_remote_process(['docker image prune -af'], $this->server); - instant_remote_process(['docker container prune -f --filter "label=coolify.managed=true"'], $this->server); - instant_remote_process(['docker builder prune -af'], $this->server); + instant_remote_process(['docker image prune -af'], $this->server, false); + instant_remote_process(['docker container prune -f --filter "label=coolify.managed=true"'], $this->server, false); + instant_remote_process(['docker builder prune -af'], $this->server, false); $usageAfter = $this->server->getDiskUsage(); if ($usageAfter < $this->usageBefore) { ray('Saved ' . ($this->usageBefore - $usageAfter) . '% disk space on ' . $this->server->name); diff --git a/app/Jobs/ServerStatusJob.php b/app/Jobs/ServerStatusJob.php index d3599049d..7d591bc83 100644 --- a/app/Jobs/ServerStatusJob.php +++ b/app/Jobs/ServerStatusJob.php @@ -16,6 +16,7 @@ class ServerStatusJob implements ShouldQueue, ShouldBeEncrypted { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; + public ?int $disk_usage = null; public function __construct(public Server $server) { } @@ -34,23 +35,33 @@ public function handle(): void ray("checking server status for {$this->server->id}"); try { $this->server->checkServerRediness(); - $disk_usage = $this->server->getDiskUsage(); - if ($disk_usage >= $this->server->settings->cleanup_after_percentage) { - if ($this->server->high_disk_usage_notification_sent) { - ray('high disk usage notification already sent'); - return; - } - $this->server->high_disk_usage_notification_sent = true; - $this->server->save(); - $this->server->team->notify(new HighDiskUsage($this->server, $disk_usage, $this->server->settings->cleanup_after_percentage)); - } else { - $this->server->high_disk_usage_notification_sent = false; - $this->server->save(); - } + $this->cleanup(notify: false); } catch (\Throwable $e) { send_internal_notification('ServerStatusJob failed with: ' . $e->getMessage()); ray($e->getMessage()); handleError($e); } } + public function cleanup(bool $notify = false): void + { + $this->disk_usage = $this->server->getDiskUsage(); + if ($this->disk_usage >= $this->server->settings->cleanup_after_percentage) { + if ($notify) { + if ($this->server->high_disk_usage_notification_sent) { + ray('high disk usage notification already sent'); + return; + } else { + $this->server->high_disk_usage_notification_sent = true; + $this->server->save(); + $this->server->team->notify(new HighDiskUsage($this->server, $this->disk_usage, $this->server->settings->cleanup_after_percentage)); + } + } else { + DockerCleanupJob::dispatchSync($this->server); + $this->cleanup(notify: true); + } + } else { + $this->server->high_disk_usage_notification_sent = false; + $this->server->save(); + } + } } diff --git a/app/Models/Server.php b/app/Models/Server.php index 3ba40ecc2..ff6af7c46 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -112,17 +112,19 @@ public function scopeWithProxy(): Builder return $this->proxy->modelScope(); } - public function isLocalhost() { - if (isDev()) { - return $this->ip === 'coolify-testing-host'; - } - return $this->ip === 'host.docker.internal'; + public function isLocalhost() + { + return $this->ip === 'host.docker.internal' || $this->id === 0; } public function checkServerRediness() { $serverUptimeCheckNumber = $this->unreachable_count; $serverUptimeCheckNumberMax = 5; - while (true) { + + $currentTime = now()->timestamp; + $runtime5Minutes = 5 * 60; + // Run for 5 minutes max and check every 5 seconds + while ($currentTime + $runtime5Minutes > now()->timestamp) { if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) { if ($this->unreachable_notification_sent === false) { ray('Server unreachable, sending notification...'); @@ -226,7 +228,7 @@ public function getIp(): Attribute if (isDev()) { return '127.0.0.1'; } - if ($this->ip === 'host.docker.internal') { + if ($this->isLocalhost()) { return base_ip(); } return $this->ip; diff --git a/app/Models/ServiceDatabase.php b/app/Models/ServiceDatabase.php index c86334e39..2feaeb947 100644 --- a/app/Models/ServiceDatabase.php +++ b/app/Models/ServiceDatabase.php @@ -36,7 +36,7 @@ public function getServiceDatabaseUrl() { $port = $this->public_port; $realIp = $this->service->server->ip; - if ($realIp === 'host.docker.internal' || isDev()) { + if ($this->service->server->isLocalhost() || isDev()) { $realIp = base_ip(); } $url = "{$realIp}:{$port}"; diff --git a/app/Notifications/Server/HighDiskUsage.php b/app/Notifications/Server/HighDiskUsage.php index d23a2a0a4..d8794600d 100644 --- a/app/Notifications/Server/HighDiskUsage.php +++ b/app/Notifications/Server/HighDiskUsage.php @@ -53,13 +53,13 @@ public function toMail(): MailMessage public function toDiscord(): string { - $message = "Coolify: Server '{$this->server->name}' high disk usage detected! \nDisk usage: {$this->disk_usage}. Please cleanup your disk to prevent data-loss. Here are some tips: https://coolify.io/docs/automated-cleanup."; + $message = "Coolify: Server '{$this->server->name}' high disk usage detected!\nDisk usage: {$this->disk_usage}%. Threshold: {$this->cleanup_after_percentage}%.\nPlease cleanup your disk to prevent data-loss.\nHere are some tips: https://coolify.io/docs/automated-cleanup."; return $message; } public function toTelegram(): array { return [ - "message" => "Coolify: Server '{$this->server->name}' high disk usage detected! \n Disk usage: {$this->disk_usage}. Please cleanup your disk to prevent data-loss. Here are some tips: https://coolify.io/docs/automated-cleanup." + "message" => "Coolify: Server '{$this->server->name}' high disk usage detected!\nDisk usage: {$this->disk_usage}%. Threshold: {$this->cleanup_after_percentage}%.\nPlease cleanup your disk to prevent data-loss.\nHere are some tips: https://coolify.io/docs/automated-cleanup." ]; } }