From 2b805f869a90ce95103dc71324c8b031995006ba Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 11 Jul 2024 12:38:54 +0200 Subject: [PATCH] fix/feat: better volume cleanups --- app/Actions/Application/StopApplication.php | 3 --- .../Api/ApplicationsController.php | 14 ++++++++-- .../Controllers/Api/DatabasesController.php | 15 +++++++++-- app/Jobs/DeleteResourceJob.php | 11 ++++++-- app/Livewire/Project/Shared/Danger.php | 4 ++- app/Models/Application.php | 26 +++++++++++++------ app/Models/StandaloneClickhouse.php | 23 +++++++++------- app/Models/StandaloneDragonfly.php | 23 +++++++++------- app/Models/StandaloneKeydb.php | 23 +++++++++------- app/Models/StandaloneMariadb.php | 23 +++++++++------- app/Models/StandaloneMongodb.php | 23 +++++++++------- app/Models/StandaloneMysql.php | 23 +++++++++------- app/Models/StandalonePostgresql.php | 24 ++++++++++------- app/Models/StandaloneRedis.php | 23 +++++++++------- openapi.yaml | 19 ++++++++++++++ .../livewire/project/shared/danger.blade.php | 1 + 16 files changed, 188 insertions(+), 90 deletions(-) diff --git a/app/Actions/Application/StopApplication.php b/app/Actions/Application/StopApplication.php index 5d184091e..a812b0c6c 100644 --- a/app/Actions/Application/StopApplication.php +++ b/app/Actions/Application/StopApplication.php @@ -43,9 +43,6 @@ public function handle(Application $application) $uuid = $application->uuid; instant_remote_process(["docker network disconnect {$uuid} coolify-proxy"], $server, false); instant_remote_process(["docker network rm {$uuid}"], $server, false); - - // remove volumes - instant_remote_process(["cd {$application->dirOnServer()} && docker compose down -v"], $server, false); } } } diff --git a/app/Http/Controllers/Api/ApplicationsController.php b/app/Http/Controllers/Api/ApplicationsController.php index 71ceb97ac..6c2f8e24b 100644 --- a/app/Http/Controllers/Api/ApplicationsController.php +++ b/app/Http/Controllers/Api/ApplicationsController.php @@ -1240,6 +1240,16 @@ public function application_by_uuid(Request $request) format: 'uuid', ) ), + new OA\Parameter( + name: 'cleanup', + in: 'query', + description: 'Delete configurations and volumes.', + required: false, + schema: new OA\Schema( + type: 'boolean', + default: true, + ) + ), ], responses: [ new OA\Response( @@ -1273,7 +1283,7 @@ public function application_by_uuid(Request $request) public function delete_by_uuid(Request $request) { $teamId = getTeamIdFromToken(); - $cleanup = $request->query->get('cleanup') ?? false; + $cleanup = $request->query->get('cleanup') ?? true; if (is_null($teamId)) { return invalidTokenResponse(); } @@ -1287,7 +1297,7 @@ public function delete_by_uuid(Request $request) 'message' => 'Application not found', ], 404); } - DeleteResourceJob::dispatch($application, $cleanup); + DeleteResourceJob::dispatch($application, deleteConfigurations: $cleanup, deleteVolumes: $cleanup); return response()->json([ 'message' => 'Application deletion request queued.', diff --git a/app/Http/Controllers/Api/DatabasesController.php b/app/Http/Controllers/Api/DatabasesController.php index ef531568c..7cf32e058 100644 --- a/app/Http/Controllers/Api/DatabasesController.php +++ b/app/Http/Controllers/Api/DatabasesController.php @@ -9,6 +9,7 @@ use App\Actions\Database\StopDatabaseProxy; use App\Enums\NewDatabaseTypes; use App\Http\Controllers\Controller; +use App\Jobs\DeleteResourceJob; use App\Models\Project; use App\Models\Server; use Illuminate\Http\Request; @@ -1528,6 +1529,16 @@ public function create_database(Request $request, NewDatabaseTypes $type) format: 'uuid', ) ), + new OA\Parameter( + name: 'cleanup', + in: 'query', + description: 'Delete configurations and volumes.', + required: false, + schema: new OA\Schema( + type: 'boolean', + default: true, + ) + ), ], responses: [ new OA\Response( @@ -1561,6 +1572,7 @@ public function create_database(Request $request, NewDatabaseTypes $type) public function delete_by_uuid(Request $request) { $teamId = getTeamIdFromToken(); + $cleanup = $request->query->get('cleanup') ?? true; if (is_null($teamId)) { return invalidTokenResponse(); } @@ -1571,8 +1583,7 @@ public function delete_by_uuid(Request $request) if (! $database) { return response()->json(['message' => 'Database not found.'], 404); } - StopDatabase::dispatch($database); - $database->forceDelete(); + DeleteResourceJob::dispatch($database, deleteConfigurations: $cleanup, deleteVolumes: $cleanup); return response()->json([ 'message' => 'Database deletion request queued.', diff --git a/app/Jobs/DeleteResourceJob.php b/app/Jobs/DeleteResourceJob.php index 8710fda88..9ecbfc09e 100644 --- a/app/Jobs/DeleteResourceJob.php +++ b/app/Jobs/DeleteResourceJob.php @@ -28,14 +28,15 @@ class DeleteResourceJob implements ShouldBeEncrypted, ShouldQueue { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; - public function __construct(public Application|Service|StandalonePostgresql|StandaloneRedis|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $resource, public bool $deleteConfigurations = false) {} + public function __construct(public Application|Service|StandalonePostgresql|StandaloneRedis|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneKeydb|StandaloneDragonfly|StandaloneClickhouse $resource, public bool $deleteConfigurations = false, public bool $deleteVolumes = false) {} public function handle() { try { - $this->resource->forceDelete(); + $persistentStorages = collect(); switch ($this->resource->type()) { case 'application': + $persistentStorages = $this->resource?->persistentStorages()?->get(); StopApplication::run($this->resource); break; case 'standalone-postgresql': @@ -46,6 +47,7 @@ public function handle() case 'standalone-keydb': case 'standalone-dragonfly': case 'standalone-clickhouse': + $persistentStorages = $this->resource?->persistentStorages()?->get(); StopDatabase::run($this->resource); break; case 'service': @@ -53,6 +55,10 @@ public function handle() DeleteService::run($this->resource); break; } + + if ($this->deleteVolumes && $this->resource->type() !== 'service') { + $this->resource?->delete_volumes($persistentStorages); + } if ($this->deleteConfigurations) { $this->resource?->delete_configurations(); } @@ -61,6 +67,7 @@ public function handle() send_internal_notification('ContainerStoppingJob failed with: '.$e->getMessage()); throw $e; } finally { + $this->resource->forceDelete(); Artisan::queue('cleanup:stucked-resources'); } } diff --git a/app/Livewire/Project/Shared/Danger.php b/app/Livewire/Project/Shared/Danger.php index e754749a4..30c35410f 100644 --- a/app/Livewire/Project/Shared/Danger.php +++ b/app/Livewire/Project/Shared/Danger.php @@ -16,6 +16,8 @@ class Danger extends Component public bool $delete_configurations = true; + public bool $delete_volumes = true; + public ?string $modalId = null; public function mount() @@ -31,7 +33,7 @@ public function delete() try { // $this->authorize('delete', $this->resource); $this->resource->delete(); - DeleteResourceJob::dispatch($this->resource, $this->delete_configurations); + DeleteResourceJob::dispatch($this->resource, $this->delete_configurations, $this->delete_volumes); return redirect()->route('project.resource.index', [ 'project_uuid' => $this->projectUuid, diff --git a/app/Models/Application.php b/app/Models/Application.php index 2bfc88bcd..b8bffa66a 100644 --- a/app/Models/Application.php +++ b/app/Models/Application.php @@ -126,16 +126,9 @@ protected static function booted() $application->compose_parsing_version = '2'; $application->save(); }); - static::deleting(function ($application) { + static::forceDeleting(function ($application) { $application->update(['fqdn' => null]); $application->settings()->delete(); - $storages = $application->persistentStorages()->get(); - $server = data_get($application, 'destination.server'); - if ($server) { - foreach ($storages as $storage) { - instant_remote_process(["docker volume rm -f $storage->name"], $server, false); - } - } $application->persistentStorages()->delete(); $application->environment_variables()->delete(); $application->environment_variables_preview()->delete(); @@ -161,6 +154,23 @@ public function delete_configurations() } } + public function delete_volumes(?Collection $persistentStorages) + { + if ($this->build_pack === 'dockercompose') { + $server = data_get($this, 'destination.server'); + ray('Deleting volumes'); + instant_remote_process(["cd {$this->dirOnServer()} && docker compose down -v"], $server, false); + } else { + if ($persistentStorages->count() === 0) { + return; + } + $server = data_get($this, 'destination.server'); + foreach ($persistentStorages as $storage) { + instant_remote_process(["docker volume rm -f $storage->name"], $server, false); + } + } + } + public function additional_servers() { return $this->belongsToMany(Server::class, 'additional_destinations') diff --git a/app/Models/StandaloneClickhouse.php b/app/Models/StandaloneClickhouse.php index 718fc9927..f930226da 100644 --- a/app/Models/StandaloneClickhouse.php +++ b/app/Models/StandaloneClickhouse.php @@ -3,6 +3,7 @@ namespace App\Models; use Illuminate\Database\Eloquent\Casts\Attribute; +use Illuminate\Database\Eloquent\Collection; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Relations\HasMany; use Illuminate\Database\Eloquent\SoftDeletes; @@ -31,16 +32,9 @@ protected static function booted() 'is_readonly' => true, ]); }); - static::deleting(function ($database) { - $storages = $database->persistentStorages()->get(); - $server = data_get($database, 'destination.server'); - if ($server) { - foreach ($storages as $storage) { - instant_remote_process(["docker volume rm -f $storage->name"], $server, false); - } - } - $database->scheduledBackups()->delete(); + static::forceDeleting(function ($database) { $database->persistentStorages()->delete(); + $database->scheduledBackups()->delete(); $database->environment_variables()->delete(); $database->tags()->detach(); }); @@ -91,6 +85,17 @@ public function delete_configurations() } } + public function delete_volumes(Collection $persistentStorages) + { + if ($persistentStorages->count() === 0) { + return; + } + $server = data_get($this, 'destination.server'); + foreach ($persistentStorages as $storage) { + instant_remote_process(["docker volume rm -f $storage->name"], $server, false); + } + } + public function realStatus() { return $this->getRawOriginal('status'); diff --git a/app/Models/StandaloneDragonfly.php b/app/Models/StandaloneDragonfly.php index b8d16d512..be04e2d83 100644 --- a/app/Models/StandaloneDragonfly.php +++ b/app/Models/StandaloneDragonfly.php @@ -3,6 +3,7 @@ namespace App\Models; use Illuminate\Database\Eloquent\Casts\Attribute; +use Illuminate\Database\Eloquent\Collection; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Relations\HasMany; use Illuminate\Database\Eloquent\SoftDeletes; @@ -31,16 +32,9 @@ protected static function booted() 'is_readonly' => true, ]); }); - static::deleting(function ($database) { - $database->scheduledBackups()->delete(); - $storages = $database->persistentStorages()->get(); - $server = data_get($database, 'destination.server'); - if ($server) { - foreach ($storages as $storage) { - instant_remote_process(["docker volume rm -f $storage->name"], $server, false); - } - } + static::forceDeleting(function ($database) { $database->persistentStorages()->delete(); + $database->scheduledBackups()->delete(); $database->environment_variables()->delete(); $database->tags()->detach(); }); @@ -91,6 +85,17 @@ public function delete_configurations() } } + public function delete_volumes(Collection $persistentStorages) + { + if ($persistentStorages->count() === 0) { + return; + } + $server = data_get($this, 'destination.server'); + foreach ($persistentStorages as $storage) { + instant_remote_process(["docker volume rm -f $storage->name"], $server, false); + } + } + public function realStatus() { return $this->getRawOriginal('status'); diff --git a/app/Models/StandaloneKeydb.php b/app/Models/StandaloneKeydb.php index d2963cf02..1b7d5a958 100644 --- a/app/Models/StandaloneKeydb.php +++ b/app/Models/StandaloneKeydb.php @@ -3,6 +3,7 @@ namespace App\Models; use Illuminate\Database\Eloquent\Casts\Attribute; +use Illuminate\Database\Eloquent\Collection; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Relations\HasMany; use Illuminate\Database\Eloquent\SoftDeletes; @@ -31,16 +32,9 @@ protected static function booted() 'is_readonly' => true, ]); }); - static::deleting(function ($database) { - $database->scheduledBackups()->delete(); - $storages = $database->persistentStorages()->get(); - $server = data_get($database, 'destination.server'); - if ($server) { - foreach ($storages as $storage) { - instant_remote_process(["docker volume rm -f $storage->name"], $server, false); - } - } + static::forceDeleting(function ($database) { $database->persistentStorages()->delete(); + $database->scheduledBackups()->delete(); $database->environment_variables()->delete(); $database->tags()->detach(); }); @@ -91,6 +85,17 @@ public function delete_configurations() } } + public function delete_volumes(Collection $persistentStorages) + { + if ($persistentStorages->count() === 0) { + return; + } + $server = data_get($this, 'destination.server'); + foreach ($persistentStorages as $storage) { + instant_remote_process(["docker volume rm -f $storage->name"], $server, false); + } + } + public function realStatus() { return $this->getRawOriginal('status'); diff --git a/app/Models/StandaloneMariadb.php b/app/Models/StandaloneMariadb.php index b7907f251..2081d9c89 100644 --- a/app/Models/StandaloneMariadb.php +++ b/app/Models/StandaloneMariadb.php @@ -3,6 +3,7 @@ namespace App\Models; use Illuminate\Database\Eloquent\Casts\Attribute; +use Illuminate\Database\Eloquent\Collection; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Relations\HasMany; use Illuminate\Database\Eloquent\SoftDeletes; @@ -31,16 +32,9 @@ protected static function booted() 'is_readonly' => true, ]); }); - static::deleting(function ($database) { - $storages = $database->persistentStorages()->get(); - $server = data_get($database, 'destination.server'); - if ($server) { - foreach ($storages as $storage) { - instant_remote_process(["docker volume rm -f $storage->name"], $server, false); - } - } - $database->scheduledBackups()->delete(); + static::forceDeleting(function ($database) { $database->persistentStorages()->delete(); + $database->scheduledBackups()->delete(); $database->environment_variables()->delete(); $database->tags()->detach(); }); @@ -91,6 +85,17 @@ public function delete_configurations() } } + public function delete_volumes(Collection $persistentStorages) + { + if ($persistentStorages->count() === 0) { + return; + } + $server = data_get($this, 'destination.server'); + foreach ($persistentStorages as $storage) { + instant_remote_process(["docker volume rm -f $storage->name"], $server, false); + } + } + public function realStatus() { return $this->getRawOriginal('status'); diff --git a/app/Models/StandaloneMongodb.php b/app/Models/StandaloneMongodb.php index 0f9f9a426..066b34ab7 100644 --- a/app/Models/StandaloneMongodb.php +++ b/app/Models/StandaloneMongodb.php @@ -3,6 +3,7 @@ namespace App\Models; use Illuminate\Database\Eloquent\Casts\Attribute; +use Illuminate\Database\Eloquent\Collection; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Relations\HasMany; use Illuminate\Database\Eloquent\SoftDeletes; @@ -35,16 +36,9 @@ protected static function booted() 'is_readonly' => true, ]); }); - static::deleting(function ($database) { - $storages = $database->persistentStorages()->get(); - $server = data_get($database, 'destination.server'); - if ($server) { - foreach ($storages as $storage) { - instant_remote_process(["docker volume rm -f $storage->name"], $server, false); - } - } - $database->scheduledBackups()->delete(); + static::forceDeleting(function ($database) { $database->persistentStorages()->delete(); + $database->scheduledBackups()->delete(); $database->environment_variables()->delete(); $database->tags()->detach(); }); @@ -95,6 +89,17 @@ public function delete_configurations() } } + public function delete_volumes(Collection $persistentStorages) + { + if ($persistentStorages->count() === 0) { + return; + } + $server = data_get($this, 'destination.server'); + foreach ($persistentStorages as $storage) { + instant_remote_process(["docker volume rm -f $storage->name"], $server, false); + } + } + public function realStatus() { return $this->getRawOriginal('status'); diff --git a/app/Models/StandaloneMysql.php b/app/Models/StandaloneMysql.php index bc4de88ee..375b56133 100644 --- a/app/Models/StandaloneMysql.php +++ b/app/Models/StandaloneMysql.php @@ -3,6 +3,7 @@ namespace App\Models; use Illuminate\Database\Eloquent\Casts\Attribute; +use Illuminate\Database\Eloquent\Collection; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Relations\HasMany; use Illuminate\Database\Eloquent\SoftDeletes; @@ -32,16 +33,9 @@ protected static function booted() 'is_readonly' => true, ]); }); - static::deleting(function ($database) { - $storages = $database->persistentStorages()->get(); - $server = data_get($database, 'destination.server'); - if ($server) { - foreach ($storages as $storage) { - instant_remote_process(["docker volume rm -f $storage->name"], $server, false); - } - } - $database->scheduledBackups()->delete(); + static::forceDeleting(function ($database) { $database->persistentStorages()->delete(); + $database->scheduledBackups()->delete(); $database->environment_variables()->delete(); $database->tags()->detach(); }); @@ -92,6 +86,17 @@ public function delete_configurations() } } + public function delete_volumes(Collection $persistentStorages) + { + if ($persistentStorages->count() === 0) { + return; + } + $server = data_get($this, 'destination.server'); + foreach ($persistentStorages as $storage) { + instant_remote_process(["docker volume rm -f $storage->name"], $server, false); + } + } + public function realStatus() { return $this->getRawOriginal('status'); diff --git a/app/Models/StandalonePostgresql.php b/app/Models/StandalonePostgresql.php index 372d79fd8..ab6bcc626 100644 --- a/app/Models/StandalonePostgresql.php +++ b/app/Models/StandalonePostgresql.php @@ -3,6 +3,7 @@ namespace App\Models; use Illuminate\Database\Eloquent\Casts\Attribute; +use Illuminate\Database\Eloquent\Collection; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Relations\HasMany; use Illuminate\Database\Eloquent\SoftDeletes; @@ -32,16 +33,9 @@ protected static function booted() 'is_readonly' => true, ]); }); - static::deleting(function ($database) { - $storages = $database->persistentStorages()->get(); - $server = data_get($database, 'destination.server'); - if ($server) { - foreach ($storages as $storage) { - instant_remote_process(["docker volume rm -f $storage->name"], $server, false); - } - } - $database->scheduledBackups()->delete(); + static::forceDeleting(function ($database) { $database->persistentStorages()->delete(); + $database->scheduledBackups()->delete(); $database->environment_variables()->delete(); $database->tags()->detach(); }); @@ -61,6 +55,18 @@ public function delete_configurations() } } + public function delete_volumes(Collection $persistentStorages) + { + if ($persistentStorages->count() === 0) { + return; + } + $server = data_get($this, 'destination.server'); + foreach ($persistentStorages as $storage) { + ray('Deleting volume: '.$storage->name); + instant_remote_process(["docker volume rm -f $storage->name"], $server, false); + } + } + public function isConfigurationChanged(bool $save = false) { $newConfigHash = $this->image.$this->ports_mappings.$this->postgres_initdb_args.$this->postgres_host_auth_method; diff --git a/app/Models/StandaloneRedis.php b/app/Models/StandaloneRedis.php index 64731a28b..df6cc8aeb 100644 --- a/app/Models/StandaloneRedis.php +++ b/app/Models/StandaloneRedis.php @@ -3,6 +3,7 @@ namespace App\Models; use Illuminate\Database\Eloquent\Casts\Attribute; +use Illuminate\Database\Eloquent\Collection; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Relations\HasMany; use Illuminate\Database\Eloquent\SoftDeletes; @@ -27,16 +28,9 @@ protected static function booted() 'is_readonly' => true, ]); }); - static::deleting(function ($database) { - $database->scheduledBackups()->delete(); - $storages = $database->persistentStorages()->get(); - $server = data_get($database, 'destination.server'); - if ($server) { - foreach ($storages as $storage) { - instant_remote_process(["docker volume rm -f $storage->name"], $server, false); - } - } + static::forceDeleting(function ($database) { $database->persistentStorages()->delete(); + $database->scheduledBackups()->delete(); $database->environment_variables()->delete(); $database->tags()->detach(); }); @@ -87,6 +81,17 @@ public function delete_configurations() } } + public function delete_volumes(Collection $persistentStorages) + { + if ($persistentStorages->count() === 0) { + return; + } + $server = data_get($this, 'destination.server'); + foreach ($persistentStorages as $storage) { + instant_remote_process(["docker volume rm -f $storage->name"], $server, false); + } + } + public function realStatus() { return $this->getRawOriginal('status'); diff --git a/openapi.yaml b/openapi.yaml index 94e26ce38..8b3c170d0 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1343,6 +1343,14 @@ paths: schema: type: string format: uuid + - + name: cleanup + in: query + description: 'Delete configurations and volumes.' + required: false + schema: + type: boolean + default: true responses: '200': description: 'Application deleted.' @@ -1799,6 +1807,14 @@ paths: schema: type: string format: uuid + - + name: cleanup + in: query + description: 'Delete configurations and volumes.' + required: false + schema: + type: boolean + default: true responses: '200': description: 'Database deleted.' @@ -4026,6 +4042,9 @@ components: format: date-time nullable: true description: 'The date and time when the application was deleted.' + compose_parsing_version: + type: string + description: 'How Coolify parse the compose file.' type: object ApplicationDeploymentQueue: description: 'Project model' diff --git a/resources/views/livewire/project/shared/danger.blade.php b/resources/views/livewire/project/shared/danger.blade.php index f660dad18..276061a8e 100644 --- a/resources/views/livewire/project/shared/danger.blade.php +++ b/resources/views/livewire/project/shared/danger.blade.php @@ -11,5 +11,6 @@

Actions

+