Shiloh/lasthourcloud
Shiloh/lasthourcloud
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 4 Wiki Activity

Refactor server and docker cleanup jobs

Browse Source
This commit is contained in:
Andras Bacsai 2023-11-16 14:29:01 +01:00
parent fb42c43953
commit 2b666ff121
6 changed files with 42 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
app/Console/Kernel.php
View File

@ -2,10 +2,8 @@
namespace App\Console;
use App\Jobs\CheckResaleLicenseJob;
use App\Jobs\CleanupInstanceStuffsJob;
use App\Jobs\DatabaseBackupJob;
use App\Jobs\DockerCleanupJob;
use App\Jobs\InstanceAutoUpdateJob;
use App\Jobs\ContainerStatusJob;
use App\Jobs\PullHelperImageJob;
@ -28,7 +26,6 @@ protected function schedule(Schedule $schedule): void
// Server Jobs
$this->check_scheduled_backups($schedule);
$this->check_resources($schedule);
$this->cleanup_servers($schedule);
$this->check_scheduled_backups($schedule);
$this->pull_helper_image($schedule);
} else {
@ -41,7 +38,6 @@ protected function schedule(Schedule $schedule): void
$this->instance_auto_update($schedule);
$this->check_scheduled_backups($schedule);
$this->check_resources($schedule);
$this->cleanup_servers($schedule);
$this->pull_helper_image($schedule);
}
}
@ -52,13 +48,6 @@ private function pull_helper_image($schedule)
$schedule->job(new PullHelperImageJob($server))->everyTenMinutes()->onOneServer();
}
}
private function cleanup_servers($schedule)
{
$servers = Server::all()->where('settings.is_usable', true)->where('settings.is_reachable', true);
foreach ($servers as $server) {
$schedule->job(new DockerCleanupJob($server))->everyTenMinutes()->onOneServer();
}
}
private function check_resources($schedule)
{
if (isCloud()) {
@ -67,8 +56,8 @@ private function check_resources($schedule)
$servers = Server::all();
}
foreach ($servers as $server) {
$schedule->job(new ServerStatusJob($server))->everyTenMinutes()->onOneServer();
$schedule->job(new ContainerStatusJob($server))->everyMinute()->onOneServer();
$schedule->job(new ServerStatusJob($server))->everyFiveMinutes()->onOneServer();
}
}
private function instance_auto_update($schedule)

9
app/Jobs/DockerCleanupJob.php
View File

@ -40,12 +40,13 @@ public function handle(): void
if (!$this->server->isFunctional()) {
return;
}
$this->usageBefore = $this->server->getDiskUsage();
ray('Usage before: ' . $this->usageBefore);
if ($this->usageBefore >= $this->server->settings->cleanup_after_percentage) {
ray('Cleaning up ' . $this->server->name);
instant_remote_process(['docker image prune -af'], $this->server);
instant_remote_process(['docker container prune -f --filter "label=coolify.managed=true"'], $this->server);
instant_remote_process(['docker builder prune -af'], $this->server);
instant_remote_process(['docker image prune -af'], $this->server, false);
instant_remote_process(['docker container prune -f --filter "label=coolify.managed=true"'], $this->server, false);
instant_remote_process(['docker builder prune -af'], $this->server, false);
$usageAfter = $this->server->getDiskUsage();
if ($usageAfter < $this->usageBefore) {
ray('Saved ' . ($this->usageBefore - $usageAfter) . '% disk space on ' . $this->server->name);

37
app/Jobs/ServerStatusJob.php
View File

@ -16,6 +16,7 @@ class ServerStatusJob implements ShouldQueue, ShouldBeEncrypted
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public ?int $disk_usage = null;
public function __construct(public Server $server)
{
}
@ -34,23 +35,33 @@ public function handle(): void
ray("checking server status for {$this->server->id}");
try {
$this->server->checkServerRediness();
$disk_usage = $this->server->getDiskUsage();
if ($disk_usage >= $this->server->settings->cleanup_after_percentage) {
if ($this->server->high_disk_usage_notification_sent) {
ray('high disk usage notification already sent');
return;
}
$this->server->high_disk_usage_notification_sent = true;
$this->server->save();
$this->server->team->notify(new HighDiskUsage($this->server, $disk_usage, $this->server->settings->cleanup_after_percentage));
} else {
$this->server->high_disk_usage_notification_sent = false;
$this->server->save();
}
$this->cleanup(notify: false);
} catch (\Throwable $e) {
send_internal_notification('ServerStatusJob failed with: ' . $e->getMessage());
ray($e->getMessage());
handleError($e);
}
}
public function cleanup(bool $notify = false): void
{
$this->disk_usage = $this->server->getDiskUsage();
if ($this->disk_usage >= $this->server->settings->cleanup_after_percentage) {
if ($notify) {
if ($this->server->high_disk_usage_notification_sent) {
ray('high disk usage notification already sent');
return;
} else {
$this->server->high_disk_usage_notification_sent = true;
$this->server->save();
$this->server->team->notify(new HighDiskUsage($this->server, $this->disk_usage, $this->server->settings->cleanup_after_percentage));
}
} else {
DockerCleanupJob::dispatchSync($this->server);
$this->cleanup(notify: true);
}
} else {
$this->server->high_disk_usage_notification_sent = false;
$this->server->save();
}
}
}

16
app/Models/Server.php
View File

@ -112,17 +112,19 @@ public function scopeWithProxy(): Builder
return $this->proxy->modelScope();
}
public function isLocalhost() {
if (isDev()) {
return $this->ip === 'coolify-testing-host';
}
return $this->ip === 'host.docker.internal';
public function isLocalhost()
{
return $this->ip === 'host.docker.internal' || $this->id === 0;
}
public function checkServerRediness()
{
$serverUptimeCheckNumber = $this->unreachable_count;
$serverUptimeCheckNumberMax = 5;
while (true) {
$currentTime = now()->timestamp;
$runtime5Minutes = 5 * 60;
// Run for 5 minutes max and check every 5 seconds
while ($currentTime + $runtime5Minutes > now()->timestamp) {
if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) {
if ($this->unreachable_notification_sent === false) {
ray('Server unreachable, sending notification...');
@ -226,7 +228,7 @@ public function getIp(): Attribute
if (isDev()) {
return '127.0.0.1';
}
if ($this->ip === 'host.docker.internal') {
if ($this->isLocalhost()) {
return base_ip();
}
return $this->ip;

2
app/Models/ServiceDatabase.php
View File

@ -36,7 +36,7 @@ public function getServiceDatabaseUrl()
{
$port = $this->public_port;
$realIp = $this->service->server->ip;
if ($realIp === 'host.docker.internal' || isDev()) {
if ($this->service->server->isLocalhost() || isDev()) {
$realIp = base_ip();
}
$url = "{$realIp}:{$port}";

4
app/Notifications/Server/HighDiskUsage.php
View File

@ -53,13 +53,13 @@ public function toMail(): MailMessage
public function toDiscord(): string
{
$message = "Coolify: Server '{$this->server->name}' high disk usage detected! \nDisk usage: {$this->disk_usage}. Please cleanup your disk to prevent data-loss. Here are some tips: https://coolify.io/docs/automated-cleanup.";
$message = "Coolify: Server '{$this->server->name}' high disk usage detected!\nDisk usage: {$this->disk_usage}%. Threshold: {$this->cleanup_after_percentage}%.\nPlease cleanup your disk to prevent data-loss.\nHere are some tips: https://coolify.io/docs/automated-cleanup.";
return $message;
}
public function toTelegram(): array
{
return [
"message" => "Coolify: Server '{$this->server->name}' high disk usage detected! \n Disk usage: {$this->disk_usage}. Please cleanup your disk to prevent data-loss. Here are some tips: https://coolify.io/docs/automated-cleanup."
"message" => "Coolify: Server '{$this->server->name}' high disk usage detected!\nDisk usage: {$this->disk_usage}%. Threshold: {$this->cleanup_after_percentage}%.\nPlease cleanup your disk to prevent data-loss.\nHere are some tips: https://coolify.io/docs/automated-cleanup."
];
}
}