feat: force cleanup server

This commit is contained in:
Andras Bacsai 2024-07-18 14:38:56 +02:00
parent 8ea50dc029
commit b8e95b2099
5 changed files with 81 additions and 28 deletions

View File

@ -12,7 +12,6 @@
use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels; use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Log;
use RuntimeException;
class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue class DockerCleanupJob implements ShouldBeEncrypted, ShouldQueue
{ {
@ -27,20 +26,25 @@ public function __construct(public Server $server) {}
public function handle(): void public function handle(): void
{ {
try { try {
$isInprogress = false; // $isInprogress = false;
$this->server->applications()->each(function ($application) use (&$isInprogress) { // $this->server->applications()->each(function ($application) use (&$isInprogress) {
if ($application->isDeploymentInprogress()) { // if ($application->isDeploymentInprogress()) {
$isInprogress = true; // $isInprogress = true;
return; // return;
} // }
}); // });
// if ($isInprogress) { // if ($isInprogress) {
// throw new RuntimeException('DockerCleanupJob: ApplicationDeploymentQueue is not empty, skipping...'); // throw new RuntimeException('DockerCleanupJob: ApplicationDeploymentQueue is not empty, skipping...');
// } // }
if (! $this->server->isFunctional()) { if (! $this->server->isFunctional()) {
return; return;
} }
if ($this->server->is_force_cleanup_enabled) {
CleanupDocker::run($this->server);
return;
}
$this->usageBefore = $this->server->getDiskUsage(); $this->usageBefore = $this->server->getDiskUsage();
ray('Usage before: '.$this->usageBefore); ray('Usage before: '.$this->usageBefore);
if ($this->usageBefore >= $this->server->settings->cleanup_after_percentage) { if ($this->usageBefore >= $this->server->settings->cleanup_after_percentage) {

View File

@ -44,7 +44,7 @@ public function handle()
} }
try { try {
if ($this->server->isFunctional()) { if ($this->server->isFunctional()) {
$this->cleanup(notify: false); $this->cleanup();
$this->remove_unnecessary_coolify_yaml(); $this->remove_unnecessary_coolify_yaml();
if ($this->server->isSentinelEnabled()) { if ($this->server->isSentinelEnabled()) {
$this->server->checkSentinel(); $this->server->checkSentinel();
@ -109,11 +109,16 @@ private function remove_unnecessary_coolify_yaml()
} }
} }
public function cleanup(bool $notify = false): void public function cleanup(): void
{ {
if ($this->server->settings->is_force_cleanup_enabled) {
DockerCleanupJob::dispatch($this->server);
return;
}
$this->disk_usage = $this->server->getDiskUsage(); $this->disk_usage = $this->server->getDiskUsage();
if ($this->disk_usage >= $this->server->settings->cleanup_after_percentage) { if ($this->disk_usage >= $this->server->settings->cleanup_after_percentage) {
if ($notify) { DockerCleanupJob::dispatch($this->server);
if ($this->server->high_disk_usage_notification_sent) { if ($this->server->high_disk_usage_notification_sent) {
ray('high disk usage notification already sent'); ray('high disk usage notification already sent');
@ -123,10 +128,6 @@ public function cleanup(bool $notify = false): void
$this->server->save(); $this->server->save();
$this->server->team?->notify(new HighDiskUsage($this->server, $this->disk_usage, $this->server->settings->cleanup_after_percentage)); $this->server->team?->notify(new HighDiskUsage($this->server, $this->disk_usage, $this->server->settings->cleanup_after_percentage));
} }
} else {
DockerCleanupJob::dispatchSync($this->server);
$this->cleanup(notify: true);
}
} else { } else {
$this->server->high_disk_usage_notification_sent = false; $this->server->high_disk_usage_notification_sent = false;
$this->server->save(); $this->server->save();

View File

@ -37,6 +37,7 @@ class Form extends Component
'server.settings.is_swarm_manager' => 'required|boolean', 'server.settings.is_swarm_manager' => 'required|boolean',
'server.settings.is_swarm_worker' => 'required|boolean', 'server.settings.is_swarm_worker' => 'required|boolean',
'server.settings.is_build_server' => 'required|boolean', 'server.settings.is_build_server' => 'required|boolean',
'server.settings.is_force_cleanup_enabled' => 'required|boolean',
'server.settings.concurrent_builds' => 'required|integer|min:1', 'server.settings.concurrent_builds' => 'required|integer|min:1',
'server.settings.dynamic_timeout' => 'required|integer|min:1', 'server.settings.dynamic_timeout' => 'required|integer|min:1',
'server.settings.is_metrics_enabled' => 'required|boolean', 'server.settings.is_metrics_enabled' => 'required|boolean',

View File

@ -0,0 +1,28 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('server_settings', function (Blueprint $table) {
$table->boolean('is_force_cleanup_enabled')->default(false)->after('is_sentinel_enabled');
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('server_settings', function (Blueprint $table) {
$table->dropColumn('is_force_cleanup_enabled');
});
}
};

View File

@ -136,14 +136,33 @@ class="w-full mt-8 mb-4 font-bold box-without-bg bg-coollabs hover:bg-coollabs-1
@if ($server->isFunctional()) @if ($server->isFunctional())
<h3 class="pt-4">Settings</h3> <h3 class="pt-4">Settings</h3>
<div class="flex flex-wrap gap-2 sm:flex-nowrap"> <div class="flex flex-col gap-2">
<div class="flex flex-col flex-wrap gap-2 sm:flex-nowrap">
@if ($server->settings->is_force_cleanup_enabled)
<div class="w-64">
<x-forms.checkbox
helper="This will cleanup build caches / unused images / etc every 10 minutes."
instantSave id="server.settings.is_force_cleanup_enabled"
label="Force Cleanup Docker Engine" />
</div>
@else
<x-forms.input id="cleanup_after_percentage" label="Disk cleanup threshold (%)" required <x-forms.input id="cleanup_after_percentage" label="Disk cleanup threshold (%)" required
helper="The disk cleanup task will run when the disk usage exceeds this threshold." /> helper="The disk cleanup task will run when the disk usage exceeds this threshold." />
<div class="w-64">
<x-forms.checkbox
helper="This will cleanup build caches / unused images / etc every 10 minutes."
instantSave id="server.settings.is_force_cleanup_enabled"
label="Force Cleanup Docker Engine" />
</div>
@endif
</div>
<div class="flex flex-wrap gap-2 sm:flex-nowrap">
<x-forms.input id="server.settings.concurrent_builds" label="Number of concurrent builds" required <x-forms.input id="server.settings.concurrent_builds" label="Number of concurrent builds" required
helper="You can specify the number of simultaneous build processes/deployments that should run concurrently." /> helper="You can specify the number of simultaneous build processes/deployments that should run concurrently." />
<x-forms.input id="server.settings.dynamic_timeout" label="Deployment timeout (seconds)" required <x-forms.input id="server.settings.dynamic_timeout" label="Deployment timeout (seconds)" required
helper="You can define the maximum duration for a deployment to run before timing it out." /> helper="You can define the maximum duration for a deployment to run before timing it out." />
</div> </div>
</div>
<div class="flex items-center gap-2 pt-4 pb-2"> <div class="flex items-center gap-2 pt-4 pb-2">
<h3>Sentinel</h3> <h3>Sentinel</h3>
{{-- @if ($server->isSentinelEnabled()) --}} {{-- @if ($server->isSentinelEnabled()) --}}