Shiloh/lasthourcloud
Shiloh/lasthourcloud
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 4 Wiki Activity

fix: make sure resources are deleted in async mode

Browse Source
This commit is contained in:
Andras Bacsai 2024-02-08 13:10:29 +01:00
parent 548304765c
commit 23d121d67a
5 changed files with 88 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
app/Actions/Service/StopService.php
View File

@ -10,24 +10,32 @@ class StopService
use AsAction; use AsAction;
public function handle(Service $service) public function handle(Service $service)
{ {
$server = $service->destination->server; try {
if (!$server->isFunctional()) { throw new \Exception('Not implemented');
return 'Server is not functional'; $server = $service->destination->server;
if (!$server->isFunctional()) {
return 'Server is not functional';
}
ray('Stopping service: ' . $service->name);
$applications = $service->applications()->get();
foreach ($applications as $application) {
instant_remote_process(["docker rm -f {$application->name}-{$service->uuid}"], $service->server);
$application->update(['status' => 'exited']);
}
$dbs = $service->databases()->get();
foreach ($dbs as $db) {
instant_remote_process(["docker rm -f {$db->name}-{$service->uuid}"], $service->server);
$db->update(['status' => 'exited']);
}
instant_remote_process(["docker network disconnect {$service->uuid} coolify-proxy 2>/dev/null"], $service->server, false);
instant_remote_process(["docker network rm {$service->uuid} 2>/dev/null"], $service->server, false);
// TODO: make notification for databases
// $service->environment->project->team->notify(new StatusChanged($service));
} catch (\Exception $e) {
echo $e->getMessage();
ray($e->getMessage());
return $e->getMessage();
} }
ray('Stopping service: ' . $service->name);
$applications = $service->applications()->get();
foreach ($applications as $application) {
instant_remote_process(["docker rm -f {$application->name}-{$service->uuid}"], $service->server);
$application->update(['status' => 'exited']);
}
$dbs = $service->databases()->get();
foreach ($dbs as $db) {
instant_remote_process(["docker rm -f {$db->name}-{$service->uuid}"], $service->server);
$db->update(['status' => 'exited']);
}
instant_remote_process(["docker network disconnect {$service->uuid} coolify-proxy 2>/dev/null"], $service->server, false);
instant_remote_process(["docker network rm {$service->uuid} 2>/dev/null"], $service->server, false);
// TODO: make notification for databases
// $service->environment->project->team->notify(new StatusChanged($service));
} }
} }

95
app/Console/Commands/CleanupStuckedResources.php
View File

@ -20,7 +20,8 @@ class CleanupStuckedResources extends Command
public function handle() public function handle()
{ {
echo "Running cleanup stucked...\n"; ray('Running cleanup stucked resources.');
echo "Running cleanup stucked resources.\n";
$this->cleanup_stucked_resources(); $this->cleanup_stucked_resources();
} }
private function cleanup_stucked_resources() private function cleanup_stucked_resources()
@ -113,18 +114,18 @@ class CleanupStuckedResources extends Command
$applications = Application::all(); $applications = Application::all();
foreach ($applications as $application) { foreach ($applications as $application) {
if (!data_get($application, 'environment')) { if (!data_get($application, 'environment')) {
echo 'Application without environment: ' . $application->name . ' soft deleting\n'; echo 'Application without environment: ' . $application->name . '\n';
$application->delete(); $application->forceDelete();
continue; continue;
} }
if (!$application->destination()) { if (!$application->destination()) {
echo 'Application without destination: ' . $application->name . ' soft deleting\n'; echo 'Application without destination: ' . $application->name . '\n';
$application->delete(); $application->forceDelete();
continue; continue;
} }
if (!data_get($application, 'destination.server')) { if (!data_get($application, 'destination.server')) {
echo 'Application without server: ' . $application->name . ' soft deleting\n'; echo 'Application without server: ' . $application->name . '\n';
$application->delete(); $application->forceDelete();
continue; continue;
} }
} }
@ -135,18 +136,18 @@ class CleanupStuckedResources extends Command
$postgresqls = StandalonePostgresql::all()->where('id', '!=', 0); $postgresqls = StandalonePostgresql::all()->where('id', '!=', 0);
foreach ($postgresqls as $postgresql) { foreach ($postgresqls as $postgresql) {
if (!data_get($postgresql, 'environment')) { if (!data_get($postgresql, 'environment')) {
echo 'Postgresql without environment: ' . $postgresql->name . ' soft deleting\n'; echo 'Postgresql without environment: ' . $postgresql->name . '\n';
$postgresql->delete(); $postgresql->forceDelete();
continue; continue;
} }
if (!$postgresql->destination()) { if (!$postgresql->destination()) {
echo 'Postgresql without destination: ' . $postgresql->name . ' soft deleting\n'; echo 'Postgresql without destination: ' . $postgresql->name . '\n';
$postgresql->delete(); $postgresql->forceDelete();
continue; continue;
} }
if (!data_get($postgresql, 'destination.server')) { if (!data_get($postgresql, 'destination.server')) {
echo 'Postgresql without server: ' . $postgresql->name . ' soft deleting\n'; echo 'Postgresql without server: ' . $postgresql->name . '\n';
$postgresql->delete(); $postgresql->forceDelete();
continue; continue;
} }
} }
@ -157,18 +158,18 @@ class CleanupStuckedResources extends Command
$redis = StandaloneRedis::all(); $redis = StandaloneRedis::all();
foreach ($redis as $redis) { foreach ($redis as $redis) {
if (!data_get($redis, 'environment')) { if (!data_get($redis, 'environment')) {
echo 'Redis without environment: ' . $redis->name . ' soft deleting\n'; echo 'Redis without environment: ' . $redis->name . '\n';
$redis->delete(); $redis->forceDelete();
continue; continue;
} }
if (!$redis->destination()) { if (!$redis->destination()) {
echo 'Redis without destination: ' . $redis->name . ' soft deleting\n'; echo 'Redis without destination: ' . $redis->name . '\n';
$redis->delete(); $redis->forceDelete();
continue; continue;
} }
if (!data_get($redis, 'destination.server')) { if (!data_get($redis, 'destination.server')) {
echo 'Redis without server: ' . $redis->name . ' soft deleting\n'; echo 'Redis without server: ' . $redis->name . '\n';
$redis->delete(); $redis->forceDelete();
continue; continue;
} }
} }
@ -180,18 +181,18 @@ class CleanupStuckedResources extends Command
$mongodbs = StandaloneMongodb::all(); $mongodbs = StandaloneMongodb::all();
foreach ($mongodbs as $mongodb) { foreach ($mongodbs as $mongodb) {
if (!data_get($mongodb, 'environment')) { if (!data_get($mongodb, 'environment')) {
echo 'Mongodb without environment: ' . $mongodb->name . ' soft deleting\n'; echo 'Mongodb without environment: ' . $mongodb->name . '\n';
$mongodb->delete(); $mongodb->forceDelete();
continue; continue;
} }
if (!$mongodb->destination()) { if (!$mongodb->destination()) {
echo 'Mongodb without destination: ' . $mongodb->name . ' soft deleting\n'; echo 'Mongodb without destination: ' . $mongodb->name . '\n';
$mongodb->delete(); $mongodb->forceDelete();
continue; continue;
} }
if (!data_get($mongodb, 'destination.server')) { if (!data_get($mongodb, 'destination.server')) {
echo 'Mongodb without server: ' . $mongodb->name . ' soft deleting\n'; echo 'Mongodb without server: ' . $mongodb->name . '\n';
$mongodb->delete(); $mongodb->forceDelete();
continue; continue;
} }
} }
@ -203,18 +204,18 @@ class CleanupStuckedResources extends Command
$mysqls = StandaloneMysql::all(); $mysqls = StandaloneMysql::all();
foreach ($mysqls as $mysql) { foreach ($mysqls as $mysql) {
if (!data_get($mysql, 'environment')) { if (!data_get($mysql, 'environment')) {
echo 'Mysql without environment: ' . $mysql->name . ' soft deleting\n'; echo 'Mysql without environment: ' . $mysql->name . '\n';
$mysql->delete(); $mysql->forceDelete();
continue; continue;
} }
if (!$mysql->destination()) { if (!$mysql->destination()) {
echo 'Mysql without destination: ' . $mysql->name . ' soft deleting\n'; echo 'Mysql without destination: ' . $mysql->name . '\n';
$mysql->delete(); $mysql->forceDelete();
continue; continue;
} }
if (!data_get($mysql, 'destination.server')) { if (!data_get($mysql, 'destination.server')) {
echo 'Mysql without server: ' . $mysql->name . ' soft deleting\n'; echo 'Mysql without server: ' . $mysql->name . '\n';
$mysql->delete(); $mysql->forceDelete();
continue; continue;
} }
} }
@ -226,18 +227,18 @@ class CleanupStuckedResources extends Command
$mariadbs = StandaloneMariadb::all(); $mariadbs = StandaloneMariadb::all();
foreach ($mariadbs as $mariadb) { foreach ($mariadbs as $mariadb) {
if (!data_get($mariadb, 'environment')) { if (!data_get($mariadb, 'environment')) {
echo 'Mariadb without environment: ' . $mariadb->name . ' soft deleting\n'; echo 'Mariadb without environment: ' . $mariadb->name . '\n';
$mariadb->delete(); $mariadb->forceDelete();
continue; continue;
} }
if (!$mariadb->destination()) { if (!$mariadb->destination()) {
echo 'Mariadb without destination: ' . $mariadb->name . ' soft deleting\n'; echo 'Mariadb without destination: ' . $mariadb->name . '\n';
$mariadb->delete(); $mariadb->forceDelete();
continue; continue;
} }
if (!data_get($mariadb, 'destination.server')) { if (!data_get($mariadb, 'destination.server')) {
echo 'Mariadb without server: ' . $mariadb->name . ' soft deleting\n'; echo 'Mariadb without server: ' . $mariadb->name . '\n';
$mariadb->delete(); $mariadb->forceDelete();
continue; continue;
} }
} }
@ -249,18 +250,18 @@ class CleanupStuckedResources extends Command
$services = Service::all(); $services = Service::all();
foreach ($services as $service) { foreach ($services as $service) {
if (!data_get($service, 'environment')) { if (!data_get($service, 'environment')) {
echo 'Service without environment: ' . $service->name . ' soft deleting\n'; echo 'Service without environment: ' . $service->name . '\n';
$service->delete(); $service->forceDelete();
continue; continue;
} }
if (!$service->destination()) { if (!$service->destination()) {
echo 'Service without destination: ' . $service->name . ' soft deleting\n'; echo 'Service without destination: ' . $service->name . '\n';
$service->delete(); $service->forceDelete();
continue; continue;
} }
if (!data_get($service, 'server')) { if (!data_get($service, 'server')) {
echo 'Service without server: ' . $service->name . ' soft deleting\n'; echo 'Service without server: ' . $service->name . '\n';
$service->delete(); $service->forceDelete();
continue; continue;
} }
} }
@ -271,8 +272,8 @@ class CleanupStuckedResources extends Command
$serviceApplications = ServiceApplication::all(); $serviceApplications = ServiceApplication::all();
foreach ($serviceApplications as $service) { foreach ($serviceApplications as $service) {
if (!data_get($service, 'service')) { if (!data_get($service, 'service')) {
echo 'ServiceApplication without service: ' . $service->name . ' soft deleting\n'; echo 'ServiceApplication without service: ' . $service->name . '\n';
$service->delete(); $service->forceDelete();
continue; continue;
} }
} }
@ -283,8 +284,8 @@ class CleanupStuckedResources extends Command
$serviceDatabases = ServiceDatabase::all(); $serviceDatabases = ServiceDatabase::all();
foreach ($serviceDatabases as $service) { foreach ($serviceDatabases as $service) {
if (!data_get($service, 'service')) { if (!data_get($service, 'service')) {
echo 'ServiceDatabase without service: ' . $service->name . ' soft deleting\n'; echo 'ServiceDatabase without service: ' . $service->name . '\n';
$service->delete(); $service->forceDelete();
continue; continue;
} }
} }

3
app/Console/Commands/Init.php
View File

@ -28,8 +28,6 @@ class Init extends Command
return; return;
} }
if ($full_cleanup) { if ($full_cleanup) {
echo "Running init cleanupsg.\n";
// Required for falsely deleted coolify db // Required for falsely deleted coolify db
$this->restore_coolify_db_backup(); $this->restore_coolify_db_backup();
$this->cleanup_in_progress_application_deployments(); $this->cleanup_in_progress_application_deployments();
@ -52,7 +50,6 @@ class Init extends Command
} }
return; return;
} }
echo "Running cleanups.\n";
$this->cleanup_stucked_helper_containers(); $this->cleanup_stucked_helper_containers();
$this->call('cleanup:stucked-resources'); $this->call('cleanup:stucked-resources');
} }

4
app/Jobs/DeleteResourceJob.php
View File

@ -19,6 +19,7 @@ use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels; use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Artisan;
class DeleteResourceJob implements ShouldQueue, ShouldBeEncrypted class DeleteResourceJob implements ShouldQueue, ShouldBeEncrypted
{ {
@ -49,8 +50,11 @@ class DeleteResourceJob implements ShouldQueue, ShouldBeEncrypted
break; break;
} }
} catch (\Throwable $e) { } catch (\Throwable $e) {
ray($e->getMessage());
send_internal_notification('ContainerStoppingJob failed with: ' . $e->getMessage()); send_internal_notification('ContainerStoppingJob failed with: ' . $e->getMessage());
throw $e; throw $e;
} finally {
Artisan::queue('cleanup:stucked-resources');
} }
} }
} }

20
resources/views/livewire/project/new/select.blade.php
View File

@ -1,5 +1,5 @@
<div x-data x-init="$wire.loadServers"> <div x-data x-init="$wire.loadServers">
<div class="flex gap-2 "> <div class="flex gap-4 ">
<h1>New Resource</h1> <h1>New Resource</h1>
<div class="w-96"> <div class="w-96">
<x-forms.select wire:model="selectedEnvironment"> <x-forms.select wire:model="selectedEnvironment">
@ -10,7 +10,7 @@
</div> </div>
</div> </div>
<div class="pb-4 ">Deploy resources, like Applications, Databases, Services...</div> <div class="pb-4 ">Deploy resources, like Applications, Databases, Services...</div>
<div class="flex flex-col gap-2 pt-10"> <div class="flex flex-col gap-4 pt-10">
@if ($current_step === 'type') @if ($current_step === 'type')
<ul class="pb-10 steps"> <ul class="pb-10 steps">
<li class="step step-secondary">Select Resource Type</li> <li class="step step-secondary">Select Resource Type</li>
@ -18,7 +18,7 @@
<li class="step">Select a Destination</li> <li class="step">Select a Destination</li>
</ul> </ul>
<h2>Applications</h2> <h2>Applications</h2>
<div class="grid justify-start grid-cols-1 gap-2 text-left xl:grid-cols-3"> <div class="grid justify-start grid-cols-1 gap-4 text-left xl:grid-cols-3">
<div class="box group" wire:click="setType('public')"> <div class="box group" wire:click="setType('public')">
<div class="flex flex-col mx-6"> <div class="flex flex-col mx-6">
<div class="font-bold text-white group-hover:text-white"> <div class="font-bold text-white group-hover:text-white">
@ -50,7 +50,7 @@
</div> </div>
</div> </div>
</div> </div>
<div class="grid justify-start grid-cols-1 gap-2 text-left xl:grid-cols-3"> <div class="grid justify-start grid-cols-1 gap-4 text-left xl:grid-cols-3">
<div class="box group" wire:click="setType('dockerfile')"> <div class="box group" wire:click="setType('dockerfile')">
<div class="flex flex-col mx-6"> <div class="flex flex-col mx-6">
<div class="font-bold text-white group-hover:text-white"> <div class="font-bold text-white group-hover:text-white">
@ -83,7 +83,7 @@
</div> </div>
</div> </div>
<h2 class="py-4">Databases</h2> <h2 class="py-4">Databases</h2>
<div class="grid justify-start grid-cols-1 gap-2 text-left xl:grid-cols-3"> <div class="grid justify-start grid-cols-1 gap-4 text-left xl:grid-cols-5">
<div class="box group" wire:click="setType('postgresql')"> <div class="box group" wire:click="setType('postgresql')">
<div class="flex flex-col mx-6"> <div class="flex flex-col mx-6">
<div class="font-bold text-white group-hover:text-white"> <div class="font-bold text-white group-hover:text-white">
@ -151,14 +151,14 @@
</div> </div>
</div> --}} </div> --}}
</div> </div>
<div class="flex items-center gap-2" wire:init='loadServices'> <div class="flex items-center gap-4" wire:init='loadServices'>
<h2 class="py-4">Services</h2> <h2 class="py-4">Services</h2>
<x-forms.button wire:click='loadServices'>Reload Services List</x-forms.button> <x-forms.button wire:click='loadServices'>Reload Services List</x-forms.button>
<input <input
class="w-full text-white rounded input input-sm bg-coolgray-200 disabled:bg-coolgray-200/50 disabled:border-none placeholder:text-coolgray-500 read-only:text-neutral-500 read-only:bg-coolgray-200/50" class="w-full text-white rounded input input-sm bg-coolgray-200 disabled:bg-coolgray-200/50 disabled:border-none placeholder:text-coolgray-500 read-only:text-neutral-500 read-only:bg-coolgray-200/50"
wire:model.live.debounce.200ms="search" placeholder="Search..."> wire:model.live.debounce.200ms="search" placeholder="Search...">
</div> </div>
<div class="grid justify-start grid-cols-1 gap-2 text-left xl:grid-cols-3"> <div class="grid justify-start grid-cols-1 gap-4 text-left xl:grid-cols-5">
@if ($loadingServices) @if ($loadingServices)
<span class="loading loading-xs loading-spinner"></span> <span class="loading loading-xs loading-spinner"></span>
@else @else
@ -211,7 +211,7 @@
label="Include Swarm Clusters" /> label="Include Swarm Clusters" />
</div> </div>
@endif --}} @endif --}}
<div class="flex flex-col justify-center gap-2 text-left xl:flex-row xl:flex-wrap"> <div class="flex flex-col justify-center gap-4 text-left xl:flex-row xl:flex-wrap">
@forelse($servers as $server) @forelse($servers as $server)
<div class="box group" wire:click="setServer({{ $server }})"> <div class="box group" wire:click="setServer({{ $server }})">
<div class="flex flex-col mx-6"> <div class="flex flex-col mx-6">
@ -244,7 +244,7 @@
<li class="step step-secondary">Select a Destination</li> <li class="step step-secondary">Select a Destination</li>
</ul> </ul>
<div class="flex flex-col justify-center gap-2 text-left xl:flex-row xl:flex-wrap"> <div class="flex flex-col justify-center gap-4 text-left xl:flex-row xl:flex-wrap">
@if ($server->isSwarm()) @if ($server->isSwarm())
@foreach ($swarmDockers as $swarmDocker) @foreach ($swarmDockers as $swarmDocker)
<div class="box group" wire:click="setDestination('{{ $swarmDocker->uuid }}')"> <div class="box group" wire:click="setDestination('{{ $swarmDocker->uuid }}')">
@ -279,7 +279,7 @@
</div> </div>
@endif @endif
@if ($current_step === 'existing-postgresql') @if ($current_step === 'existing-postgresql')
<form wire:submit='addExistingPostgresql' class="flex items-end gap-2"> <form wire:submit='addExistingPostgresql' class="flex items-end gap-4">
<x-forms.input placeholder="postgres://username:password@database:5432" label="Database URL" <x-forms.input placeholder="postgres://username:password@database:5432" label="Database URL"
id="existingPostgresqlUrl" /> id="existingPostgresqlUrl" />
<x-forms.button type="submit">Add Database</x-forms.button> <x-forms.button type="submit">Add Database</x-forms.button>