Shiloh/lasthourcloud
Shiloh/lasthourcloud
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 4 Wiki Activity

fix: docker cleanup should be a job by server

Browse Source
This commit is contained in:
Andras Bacsai 2023-09-29 14:26:42 +02:00
parent 4161ea7eb6
commit 23e205b6cd
2 changed files with 56 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
app/Console/Kernel.php
View File

@ -27,14 +27,23 @@ class Kernel extends ConsoleKernel
// $this->instance_auto_update($schedule); // $this->instance_auto_update($schedule);
// $this->check_scheduled_backups($schedule); // $this->check_scheduled_backups($schedule);
$this->check_resources($schedule); $this->check_resources($schedule);
$this->cleanup_servers($schedule);
} else { } else {
$schedule->command('horizon:snapshot')->everyFiveMinutes(); $schedule->command('horizon:snapshot')->everyFiveMinutes();
$schedule->job(new CleanupInstanceStuffsJob)->everyTwoMinutes()->onOneServer(); $schedule->job(new CleanupInstanceStuffsJob)->everyTwoMinutes()->onOneServer();
$schedule->job(new CheckResaleLicenseJob)->hourly()->onOneServer(); $schedule->job(new CheckResaleLicenseJob)->hourly()->onOneServer();
$schedule->job(new DockerCleanupJob)->everyTenMinutes()->onOneServer(); // $schedule->job(new DockerCleanupJob)->everyTenMinutes()->onOneServer();
$this->instance_auto_update($schedule); $this->instance_auto_update($schedule);
$this->check_scheduled_backups($schedule); $this->check_scheduled_backups($schedule);
$this->check_resources($schedule); $this->check_resources($schedule);
$this->cleanup_servers($schedule);
}
}
private function cleanup_servers($schedule)
{
$servers = Server::all()->where('settings.is_usable', true)->where('settings.is_reachable', true);
foreach ($servers as $server) {
$schedule->job(new DockerCleanupJob($server))->everyTenMinutes()->onOneServer();
} }
} }
private function check_resources($schedule) private function check_resources($schedule)

91
app/Jobs/DockerCleanupJob.php
View File

@ -16,65 +16,66 @@ class DockerCleanupJob implements ShouldQueue, ShouldBeEncrypted
{ {
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $timeout = 500; public $timeout = 1000;
public ?string $dockerRootFilesystem = null; public ?string $dockerRootFilesystem = null;
public ?int $usageBefore = null; public ?int $usageBefore = null;
public function middleware(): array public function middleware(): array
{ {
return [ return [(new WithoutOverlapping($this->server->uuid))->dontRelease()];
(new WithoutOverlapping("dockerimagejobs"))->shared(),
];
} }
public function __construct()
public function uniqueId(): string
{
return $this->server->uuid;
}
public function __construct(public Server $server)
{ {
} }
public function handle(): void public function handle(): void
{ {
$queue = ApplicationDeploymentQueue::where('status', '==', 'in_progress')->get(); $queuedCount = 0;
if ($queue->count() > 0) { $this->server->applications()->each(function ($application) use ($queuedCount) {
$count = data_get($application->deployments(), 'count', 0);
$queuedCount += $count;
});
if ($queuedCount > 0) {
ray('DockerCleanupJob: ApplicationDeploymentQueue is not empty, skipping')->color('orange'); ray('DockerCleanupJob: ApplicationDeploymentQueue is not empty, skipping')->color('orange');
return; return;
} }
try { try {
// ray()->showQueries()->color('orange'); if (!$this->server->isFunctional()) {
$servers = Server::all(); return;
foreach ($servers as $server) { }
if ( if (isDev()) {
!$server->isFunctional() $this->dockerRootFilesystem = "/";
) { } else {
continue; $this->dockerRootFilesystem = instant_remote_process(
} [
if (isDev()) { "stat --printf=%m $(docker info --format '{{json .DockerRootDir}}'' |sed 's/\"//g')"
$this->dockerRootFilesystem = "/"; ],
$this->server,
false
);
}
if (!$this->dockerRootFilesystem) {
return;
}
$this->usageBefore = $this->getFilesystemUsage();
if ($this->usageBefore >= $this->server->settings->cleanup_after_percentage) {
ray('Cleaning up ' . $this->server->name)->color('orange');
instant_remote_process(['docker image prune -af'], $this->server);
instant_remote_process(['docker container prune -f --filter "label=coolify.managed=true"'], $this->server);
instant_remote_process(['docker builder prune -af'], $this->server);
$usageAfter = $this->getFilesystemUsage();
if ($usageAfter < $this->usageBefore) {
ray('Saved ' . ($this->usageBefore - $usageAfter) . '% disk space on ' . $this->server->name)->color('orange');
send_internal_notification('DockerCleanupJob done: Saved ' . ($this->usageBefore - $usageAfter) . '% disk space on ' . $this->server->name);
} else { } else {
$this->dockerRootFilesystem = instant_remote_process( ray('DockerCleanupJob failed to save disk space on ' . $this->server->name)->color('orange');
[
"stat --printf=%m $(docker info --format '{{json .DockerRootDir}}'' |sed 's/\"//g')"
],
$server,
false
);
}
if (!$this->dockerRootFilesystem) {
continue;
}
$this->usageBefore = $this->getFilesystemUsage($server);
if ($this->usageBefore >= $server->settings->cleanup_after_percentage) {
ray('Cleaning up ' . $server->name)->color('orange');
instant_remote_process(['docker image prune -af'], $server);
instant_remote_process(['docker container prune -f --filter "label=coolify.managed=true"'], $server);
instant_remote_process(['docker builder prune -af'], $server);
$usageAfter = $this->getFilesystemUsage($server);
if ($usageAfter < $this->usageBefore) {
ray('Saved ' . ($this->usageBefore - $usageAfter) . '% disk space on ' . $server->name)->color('orange');
send_internal_notification('DockerCleanupJob done: Saved ' . ($this->usageBefore - $usageAfter) . '% disk space on ' . $server->name);
} else {
ray('DockerCleanupJob failed to save disk space on ' . $server->name)->color('orange');
}
} else {
ray('No need to clean up ' . $server->name)->color('orange');
} }
} else {
ray('No need to clean up ' . $this->server->name)->color('orange');
} }
} catch (\Throwable $e) { } catch (\Throwable $e) {
send_internal_notification('DockerCleanupJob failed with: ' . $e->getMessage()); send_internal_notification('DockerCleanupJob failed with: ' . $e->getMessage());
@ -83,8 +84,8 @@ class DockerCleanupJob implements ShouldQueue, ShouldBeEncrypted
} }
} }
private function getFilesystemUsage(Server $server) private function getFilesystemUsage()
{ {
return instant_remote_process(["df '{$this->dockerRootFilesystem}'| tail -1 | awk '{ print $5}' | sed 's/%//g'"], $server, false); return instant_remote_process(["df '{$this->dockerRootFilesystem}'| tail -1 | awk '{ print $5}' | sed 's/%//g'"], $this->server, false);
} }
} }