2023-07-07 13:50:36 +00:00
|
|
|
<?php
|
|
|
|
|
|
|
|
namespace App\Jobs;
|
|
|
|
|
2023-09-04 14:59:02 +00:00
|
|
|
use App\Models\ApplicationDeploymentQueue;
|
2023-07-07 13:50:36 +00:00
|
|
|
use App\Models\Server;
|
|
|
|
use Illuminate\Bus\Queueable;
|
|
|
|
use Illuminate\Contracts\Queue\ShouldQueue;
|
|
|
|
use Illuminate\Foundation\Bus\Dispatchable;
|
|
|
|
use Illuminate\Queue\InteractsWithQueue;
|
2023-09-01 08:11:00 +00:00
|
|
|
use Illuminate\Queue\Middleware\WithoutOverlapping;
|
2023-07-07 13:50:36 +00:00
|
|
|
use Illuminate\Queue\SerializesModels;
|
|
|
|
|
|
|
|
class DockerCleanupJob implements ShouldQueue
|
|
|
|
{
|
|
|
|
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
2023-08-08 09:51:36 +00:00
|
|
|
|
2023-07-07 13:50:36 +00:00
|
|
|
public $timeout = 500;
|
2023-08-29 08:00:29 +00:00
|
|
|
public ?string $dockerRootFilesystem = null;
|
|
|
|
public ?int $usageBefore = null;
|
2023-09-01 08:11:00 +00:00
|
|
|
|
|
|
|
public function middleware(): array
|
|
|
|
{
|
|
|
|
return [
|
|
|
|
(new WithoutOverlapping("dockerimagejobs"))->shared(),
|
|
|
|
];
|
|
|
|
}
|
2023-07-07 13:50:36 +00:00
|
|
|
public function __construct()
|
|
|
|
{
|
|
|
|
}
|
|
|
|
public function handle(): void
|
|
|
|
{
|
2023-09-04 14:59:02 +00:00
|
|
|
$queue = ApplicationDeploymentQueue::where('status', '==', 'in_progress')->get();
|
|
|
|
if ($queue->count() > 0) {
|
|
|
|
ray('DockerCleanupJob: ApplicationDeploymentQueue is not empty, skipping')->color('orange');
|
|
|
|
return;
|
|
|
|
}
|
2023-07-07 13:50:36 +00:00
|
|
|
try {
|
2023-08-29 08:00:29 +00:00
|
|
|
ray()->showQueries()->color('orange');
|
2023-07-07 13:50:36 +00:00
|
|
|
$servers = Server::all();
|
|
|
|
foreach ($servers as $server) {
|
2023-08-27 19:36:11 +00:00
|
|
|
if (
|
|
|
|
!$server->settings->is_reachable && !$server->settings->is_usable
|
|
|
|
) {
|
|
|
|
continue;
|
|
|
|
}
|
2023-08-27 13:23:47 +00:00
|
|
|
if (isDev()) {
|
2023-08-29 08:00:29 +00:00
|
|
|
$this->dockerRootFilesystem = "/";
|
2023-07-07 13:50:36 +00:00
|
|
|
} else {
|
2023-09-01 08:02:24 +00:00
|
|
|
$this->dockerRootFilesystem = instant_remote_process(
|
|
|
|
[
|
|
|
|
"stat --printf=%m $(docker info --format '{{json .DockerRootDir}}'' |sed 's/\"//g')"
|
|
|
|
],
|
|
|
|
$server,
|
|
|
|
false
|
|
|
|
);
|
2023-07-07 13:50:36 +00:00
|
|
|
}
|
2023-08-29 08:00:29 +00:00
|
|
|
if (!$this->dockerRootFilesystem) {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
$this->usageBefore = $this->getFilesystemUsage($server);
|
|
|
|
if ($this->usageBefore >= $server->settings->cleanup_after_percentage) {
|
|
|
|
ray('Cleaning up ' . $server->name)->color('orange');
|
2023-07-07 19:07:42 +00:00
|
|
|
instant_remote_process(['docker image prune -af'], $server);
|
|
|
|
instant_remote_process(['docker container prune -f --filter "label=coolify.managed=true"'], $server);
|
|
|
|
instant_remote_process(['docker builder prune -af'], $server);
|
2023-08-29 08:00:29 +00:00
|
|
|
$usageAfter = $this->getFilesystemUsage($server);
|
|
|
|
if ($usageAfter < $this->usageBefore) {
|
2023-09-01 08:02:24 +00:00
|
|
|
ray('Saved ' . ($this->usageBefore - $usageAfter) . '% disk space on ' . $server->name)->color('orange');
|
|
|
|
send_internal_notification('DockerCleanupJob done: Saved ' . ($this->usageBefore - $usageAfter) . '% disk space on ' . $server->name);
|
|
|
|
} else {
|
2023-08-29 08:00:29 +00:00
|
|
|
ray('DockerCleanupJob failed to save disk space on ' . $server->name)->color('orange');
|
2023-07-07 19:35:29 +00:00
|
|
|
}
|
2023-08-29 08:00:29 +00:00
|
|
|
} else {
|
|
|
|
ray('No need to clean up ' . $server->name)->color('orange');
|
2023-07-07 13:50:36 +00:00
|
|
|
}
|
|
|
|
}
|
2023-09-11 15:36:30 +00:00
|
|
|
} catch (\Throwable $e) {
|
2023-08-24 14:14:09 +00:00
|
|
|
send_internal_notification('DockerCleanupJob failed with: ' . $e->getMessage());
|
2023-08-29 08:00:29 +00:00
|
|
|
ray($e->getMessage())->color('orange');
|
2023-08-24 19:09:58 +00:00
|
|
|
throw $e;
|
2023-07-07 13:50:36 +00:00
|
|
|
}
|
|
|
|
}
|
2023-07-26 11:33:48 +00:00
|
|
|
|
2023-08-29 08:00:29 +00:00
|
|
|
private function getFilesystemUsage(Server $server)
|
2023-07-26 11:33:48 +00:00
|
|
|
{
|
2023-08-29 08:00:29 +00:00
|
|
|
return instant_remote_process(["df '{$this->dockerRootFilesystem}'| tail -1 | awk '{ print $5}' | sed 's/%//g'"], $server, false);
|
2023-07-07 19:35:29 +00:00
|
|
|
}
|
2023-07-26 11:33:48 +00:00
|
|
|
}
|