Shiloh/lasthourcloud
Shiloh/lasthourcloud
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 4 Wiki Activity

fix: handle duplicate error instead of sql error

Browse Source 
fix: set fqdns to null if you delete an app or a serviceapp
fix: make stucked resources a separate command
This commit is contained in:
Andras Bacsai 2024-01-30 09:48:51 +01:00
parent 3ae9501814
commit 9667cd4a7a
9 changed files with 316 additions and 297 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
app/Actions/Application/StopApplication.php
View File

@ -11,6 +11,9 @@ class StopApplication
public function handle(Application $application)
{
$server = $application->destination->server;
if (!$server->isFunctional()) {
return 'Server is not functional';
}
if ($server->isSwarm()) {
instant_remote_process(["docker stack rm {$application->uuid}" ], $server);
} else {

3
app/Actions/Database/StopDatabase.php
View File

@ -17,6 +17,9 @@ class StopDatabase
public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb $database)
{
$server = $database->destination->server;
if (!$server->isFunctional()) {
return 'Server is not functional';
}
instant_remote_process(
["docker rm -f {$database->uuid}"],
$server

295
app/Console/Commands/CleanupStuckedResources.php Normal file
View File

@ -0,0 +1,295 @@
<?php
namespace App\Console\Commands;
use App\Models\Application;
use App\Models\Service;
use App\Models\ServiceApplication;
use App\Models\ServiceDatabase;
use App\Models\StandaloneMariadb;
use App\Models\StandaloneMongodb;
use App\Models\StandaloneMysql;
use App\Models\StandalonePostgresql;
use App\Models\StandaloneRedis;
use Illuminate\Console\Command;
class CleanupStuckedResources extends Command
{
protected $signature = 'cleanup:stucked-resources';
protected $description = 'Cleanup Stucked Resources';
public function handle()
{
echo "Running cleanup stucked...\n";
$this->cleanup_stucked_resources();
}
private function cleanup_stucked_resources()
{
try {
$applications = Application::withTrashed()->whereNotNull('deleted_at')->get();
foreach ($applications as $application) {
echo "Deleting stuck application: {$application->name}\n";
$application->forceDelete();
}
} catch (\Throwable $e) {
echo "Error in cleaning stuck application: {$e->getMessage()}\n";
}
try {
$postgresqls = StandalonePostgresql::withTrashed()->whereNotNull('deleted_at')->get();
foreach ($postgresqls as $postgresql) {
echo "Deleting stuck postgresql: {$postgresql->name}\n";
$postgresql->forceDelete();
}
} catch (\Throwable $e) {
echo "Error in cleaning stuck postgresql: {$e->getMessage()}\n";
}
try {
$redis = StandaloneRedis::withTrashed()->whereNotNull('deleted_at')->get();
foreach ($redis as $redis) {
echo "Deleting stuck redis: {$redis->name}\n";
$redis->forceDelete();
}
} catch (\Throwable $e) {
echo "Error in cleaning stuck redis: {$e->getMessage()}\n";
}
try {
$mongodbs = StandaloneMongodb::withTrashed()->whereNotNull('deleted_at')->get();
foreach ($mongodbs as $mongodb) {
echo "Deleting stuck mongodb: {$mongodb->name}\n";
$mongodb->forceDelete();
}
} catch (\Throwable $e) {
echo "Error in cleaning stuck mongodb: {$e->getMessage()}\n";
}
try {
$mysqls = StandaloneMysql::withTrashed()->whereNotNull('deleted_at')->get();
foreach ($mysqls as $mysql) {
echo "Deleting stuck mysql: {$mysql->name}\n";
$mysql->forceDelete();
}
} catch (\Throwable $e) {
echo "Error in cleaning stuck mysql: {$e->getMessage()}\n";
}
try {
$mariadbs = StandaloneMariadb::withTrashed()->whereNotNull('deleted_at')->get();
foreach ($mariadbs as $mariadb) {
echo "Deleting stuck mariadb: {$mariadb->name}\n";
$mariadb->forceDelete();
}
} catch (\Throwable $e) {
echo "Error in cleaning stuck mariadb: {$e->getMessage()}\n";
}
try {
$services = Service::withTrashed()->whereNotNull('deleted_at')->get();
foreach ($services as $service) {
echo "Deleting stuck service: {$service->name}\n";
$service->forceDelete();
}
} catch (\Throwable $e) {
echo "Error in cleaning stuck service: {$e->getMessage()}\n";
}
try {
$serviceApps = ServiceApplication::withTrashed()->whereNotNull('deleted_at')->get();
foreach ($serviceApps as $serviceApp) {
echo "Deleting stuck serviceapp: {$serviceApp->name}\n";
$serviceApp->forceDelete();
}
} catch (\Throwable $e) {
echo "Error in cleaning stuck serviceapp: {$e->getMessage()}\n";
}
try {
$serviceDbs = ServiceDatabase::withTrashed()->whereNotNull('deleted_at')->get();
foreach ($serviceDbs as $serviceDb) {
echo "Deleting stuck serviceapp: {$serviceDb->name}\n";
$serviceDb->forceDelete();
}
} catch (\Throwable $e) {
echo "Error in cleaning stuck serviceapp: {$e->getMessage()}\n";
}
// Cleanup any resources that are not attached to any environment or destination or server
try {
$applications = Application::all();
foreach ($applications as $application) {
if (!data_get($application, 'environment')) {
echo 'Application without environment: ' . $application->name . ' soft deleting\n';
$application->delete();
continue;
}
if (!$application->destination()) {
echo 'Application without destination: ' . $application->name . ' soft deleting\n';
$application->delete();
continue;
}
if (!data_get($application, 'destination.server')) {
echo 'Application without server: ' . $application->name . ' soft deleting\n';
$application->delete();
continue;
}
}
} catch (\Throwable $e) {
echo "Error in application: {$e->getMessage()}\n";
}
try {
$postgresqls = StandalonePostgresql::all()->where('id', '!=', 0);
foreach ($postgresqls as $postgresql) {
if (!data_get($postgresql, 'environment')) {
echo 'Postgresql without environment: ' . $postgresql->name . ' soft deleting\n';
$postgresql->delete();
continue;
}
if (!$postgresql->destination()) {
echo 'Postgresql without destination: ' . $postgresql->name . ' soft deleting\n';
$postgresql->delete();
continue;
}
if (!data_get($postgresql, 'destination.server')) {
echo 'Postgresql without server: ' . $postgresql->name . ' soft deleting\n';
$postgresql->delete();
continue;
}
}
} catch (\Throwable $e) {
echo "Error in postgresql: {$e->getMessage()}\n";
}
try {
$redis = StandaloneRedis::all();
foreach ($redis as $redis) {
if (!data_get($redis, 'environment')) {
echo 'Redis without environment: ' . $redis->name . ' soft deleting\n';
$redis->delete();
continue;
}
if (!$redis->destination()) {
echo 'Redis without destination: ' . $redis->name . ' soft deleting\n';
$redis->delete();
continue;
}
if (!data_get($redis, 'destination.server')) {
echo 'Redis without server: ' . $redis->name . ' soft deleting\n';
$redis->delete();
continue;
}
}
} catch (\Throwable $e) {
echo "Error in redis: {$e->getMessage()}\n";
}
try {
$mongodbs = StandaloneMongodb::all();
foreach ($mongodbs as $mongodb) {
if (!data_get($mongodb, 'environment')) {
echo 'Mongodb without environment: ' . $mongodb->name . ' soft deleting\n';
$mongodb->delete();
continue;
}
if (!$mongodb->destination()) {
echo 'Mongodb without destination: ' . $mongodb->name . ' soft deleting\n';
$mongodb->delete();
continue;
}
if (!data_get($mongodb, 'destination.server')) {
echo 'Mongodb without server: ' . $mongodb->name . ' soft deleting\n';
$mongodb->delete();
continue;
}
}
} catch (\Throwable $e) {
echo "Error in mongodb: {$e->getMessage()}\n";
}
try {
$mysqls = StandaloneMysql::all();
foreach ($mysqls as $mysql) {
if (!data_get($mysql, 'environment')) {
echo 'Mysql without environment: ' . $mysql->name . ' soft deleting\n';
$mysql->delete();
continue;
}
if (!$mysql->destination()) {
echo 'Mysql without destination: ' . $mysql->name . ' soft deleting\n';
$mysql->delete();
continue;
}
if (!data_get($mysql, 'destination.server')) {
echo 'Mysql without server: ' . $mysql->name . ' soft deleting\n';
$mysql->delete();
continue;
}
}
} catch (\Throwable $e) {
echo "Error in mysql: {$e->getMessage()}\n";
}
try {
$mariadbs = StandaloneMariadb::all();
foreach ($mariadbs as $mariadb) {
if (!data_get($mariadb, 'environment')) {
echo 'Mariadb without environment: ' . $mariadb->name . ' soft deleting\n';
$mariadb->delete();
continue;
}
if (!$mariadb->destination()) {
echo 'Mariadb without destination: ' . $mariadb->name . ' soft deleting\n';
$mariadb->delete();
continue;
}
if (!data_get($mariadb, 'destination.server')) {
echo 'Mariadb without server: ' . $mariadb->name . ' soft deleting\n';
$mariadb->delete();
continue;
}
}
} catch (\Throwable $e) {
echo "Error in mariadb: {$e->getMessage()}\n";
}
try {
$services = Service::all();
foreach ($services as $service) {
if (!data_get($service, 'environment')) {
echo 'Service without environment: ' . $service->name . ' soft deleting\n';
$service->delete();
continue;
}
if (!$service->destination()) {
echo 'Service without destination: ' . $service->name . ' soft deleting\n';
$service->delete();
continue;
}
if (!data_get($service, 'server')) {
echo 'Service without server: ' . $service->name . ' soft deleting\n';
$service->delete();
continue;
}
}
} catch (\Throwable $e) {
echo "Error in service: {$e->getMessage()}\n";
}
try {
$serviceApplications = ServiceApplication::all();
foreach ($serviceApplications as $service) {
if (!data_get($service, 'service')) {
echo 'ServiceApplication without service: ' . $service->name . ' soft deleting\n';
$service->delete();
continue;
}
}
} catch (\Throwable $e) {
echo "Error in serviceApplications: {$e->getMessage()}\n";
}
try {
$serviceDatabases = ServiceDatabase::all();
foreach ($serviceDatabases as $service) {
if (!data_get($service, 'service')) {
echo 'ServiceDatabase without service: ' . $service->name . ' soft deleting\n';
$service->delete();
continue;
}
}
} catch (\Throwable $e) {
echo "Error in ServiceDatabases: {$e->getMessage()}\n";
}
}
}

278
app/Console/Commands/Init.php
View File

@ -4,19 +4,11 @@
use App\Enums\ApplicationDeploymentStatus;
use App\Jobs\CleanupHelperContainersJob;
use App\Models\Application;
use App\Models\ApplicationDeploymentQueue;
use App\Models\InstanceSettings;
use App\Models\ScheduledDatabaseBackup;
use App\Models\Server;
use App\Models\Service;
use App\Models\ServiceApplication;
use App\Models\ServiceDatabase;
use App\Models\StandaloneMariadb;
use App\Models\StandaloneMongodb;
use App\Models\StandaloneMysql;
use App\Models\StandalonePostgresql;
use App\Models\StandaloneRedis;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Http;
@ -31,7 +23,7 @@ public function handle()
$cleanup = $this->option('cleanup');
if ($cleanup) {
echo "Running cleanups...\n";
$this->cleanup_stucked_resources();
$this->call('cleanup:stucked-resources');
// Required for falsely deleted coolify db
$this->restore_coolify_db_backup();
@ -137,273 +129,5 @@ private function cleanup_in_progress_application_deployments()
echo "Error: {$e->getMessage()}\n";
}
}
private function cleanup_stucked_resources()
{
try {
$applications = Application::withTrashed()->whereNotNull('deleted_at')->get();
foreach ($applications as $application) {
echo "Deleting stuck application: {$application->name}\n";
$application->forceDelete();
}
} catch (\Throwable $e) {
echo "Error in cleaning stuck application: {$e->getMessage()}\n";
}
try {
$postgresqls = StandalonePostgresql::withTrashed()->whereNotNull('deleted_at')->get();
foreach ($postgresqls as $postgresql) {
echo "Deleting stuck postgresql: {$postgresql->name}\n";
$postgresql->forceDelete();
}
} catch (\Throwable $e) {
echo "Error in cleaning stuck postgresql: {$e->getMessage()}\n";
}
try {
$redis = StandaloneRedis::withTrashed()->whereNotNull('deleted_at')->get();
foreach ($redis as $redis) {
echo "Deleting stuck redis: {$redis->name}\n";
$redis->forceDelete();
}
} catch (\Throwable $e) {
echo "Error in cleaning stuck redis: {$e->getMessage()}\n";
}
try {
$mongodbs = StandaloneMongodb::withTrashed()->whereNotNull('deleted_at')->get();
foreach ($mongodbs as $mongodb) {
echo "Deleting stuck mongodb: {$mongodb->name}\n";
$mongodb->forceDelete();
}
} catch (\Throwable $e) {
echo "Error in cleaning stuck mongodb: {$e->getMessage()}\n";
}
try {
$mysqls = StandaloneMysql::withTrashed()->whereNotNull('deleted_at')->get();
foreach ($mysqls as $mysql) {
echo "Deleting stuck mysql: {$mysql->name}\n";
$mysql->forceDelete();
}
} catch (\Throwable $e) {
echo "Error in cleaning stuck mysql: {$e->getMessage()}\n";
}
try {
$mariadbs = StandaloneMariadb::withTrashed()->whereNotNull('deleted_at')->get();
foreach ($mariadbs as $mariadb) {
echo "Deleting stuck mariadb: {$mariadb->name}\n";
$mariadb->forceDelete();
}
} catch (\Throwable $e) {
echo "Error in cleaning stuck mariadb: {$e->getMessage()}\n";
}
try {
$services = Service::withTrashed()->whereNotNull('deleted_at')->get();
foreach ($services as $service) {
echo "Deleting stuck service: {$service->name}\n";
$service->forceDelete();
}
} catch (\Throwable $e) {
echo "Error in cleaning stuck service: {$e->getMessage()}\n";
}
try {
$serviceApps = ServiceApplication::withTrashed()->whereNotNull('deleted_at')->get();
foreach ($serviceApps as $serviceApp) {
echo "Deleting stuck serviceapp: {$serviceApp->name}\n";
$serviceApp->forceDelete();
}
} catch (\Throwable $e) {
echo "Error in cleaning stuck serviceapp: {$e->getMessage()}\n";
}
try {
$serviceDbs = ServiceDatabase::withTrashed()->whereNotNull('deleted_at')->get();
foreach ($serviceDbs as $serviceDb) {
echo "Deleting stuck serviceapp: {$serviceDb->name}\n";
$serviceDb->forceDelete();
}
} catch (\Throwable $e) {
echo "Error in cleaning stuck serviceapp: {$e->getMessage()}\n";
}
// Cleanup any resources that are not attached to any environment or destination or server
try {
$applications = Application::all();
foreach ($applications as $application) {
if (!data_get($application, 'environment')) {
echo 'Application without environment: ' . $application->name . ' soft deleting\n';
$application->delete();
continue;
}
if (!$application->destination()) {
echo 'Application without destination: ' . $application->name . ' soft deleting\n';
$application->delete();
continue;
}
if (!data_get($application, 'destination.server')) {
echo 'Application without server: ' . $application->name . ' soft deleting\n';
$application->delete();
continue;
}
}
} catch (\Throwable $e) {
echo "Error in application: {$e->getMessage()}\n";
}
try {
$postgresqls = StandalonePostgresql::all()->where('id', '!=', 0);
foreach ($postgresqls as $postgresql) {
if (!data_get($postgresql, 'environment')) {
echo 'Postgresql without environment: ' . $postgresql->name . ' soft deleting\n';
$postgresql->delete();
continue;
}
if (!$postgresql->destination()) {
echo 'Postgresql without destination: ' . $postgresql->name . ' soft deleting\n';
$postgresql->delete();
continue;
}
if (!data_get($postgresql, 'destination.server')) {
echo 'Postgresql without server: ' . $postgresql->name . ' soft deleting\n';
$postgresql->delete();
continue;
}
}
} catch (\Throwable $e) {
echo "Error in postgresql: {$e->getMessage()}\n";
}
try {
$redis = StandaloneRedis::all();
foreach ($redis as $redis) {
if (!data_get($redis, 'environment')) {
echo 'Redis without environment: ' . $redis->name . ' soft deleting\n';
$redis->delete();
continue;
}
if (!$redis->destination()) {
echo 'Redis without destination: ' . $redis->name . ' soft deleting\n';
$redis->delete();
continue;
}
if (!data_get($redis, 'destination.server')) {
echo 'Redis without server: ' . $redis->name . ' soft deleting\n';
$redis->delete();
continue;
}
}
} catch (\Throwable $e) {
echo "Error in redis: {$e->getMessage()}\n";
}
try {
$mongodbs = StandaloneMongodb::all();
foreach ($mongodbs as $mongodb) {
if (!data_get($mongodb, 'environment')) {
echo 'Mongodb without environment: ' . $mongodb->name . ' soft deleting\n';
$mongodb->delete();
continue;
}
if (!$mongodb->destination()) {
echo 'Mongodb without destination: ' . $mongodb->name . ' soft deleting\n';
$mongodb->delete();
continue;
}
if (!data_get($mongodb, 'destination.server')) {
echo 'Mongodb without server: ' . $mongodb->name . ' soft deleting\n';
$mongodb->delete();
continue;
}
}
} catch (\Throwable $e) {
echo "Error in mongodb: {$e->getMessage()}\n";
}
try {
$mysqls = StandaloneMysql::all();
foreach ($mysqls as $mysql) {
if (!data_get($mysql, 'environment')) {
echo 'Mysql without environment: ' . $mysql->name . ' soft deleting\n';
$mysql->delete();
continue;
}
if (!$mysql->destination()) {
echo 'Mysql without destination: ' . $mysql->name . ' soft deleting\n';
$mysql->delete();
continue;
}
if (!data_get($mysql, 'destination.server')) {
echo 'Mysql without server: ' . $mysql->name . ' soft deleting\n';
$mysql->delete();
continue;
}
}
} catch (\Throwable $e) {
echo "Error in mysql: {$e->getMessage()}\n";
}
try {
$mariadbs = StandaloneMariadb::all();
foreach ($mariadbs as $mariadb) {
if (!data_get($mariadb, 'environment')) {
echo 'Mariadb without environment: ' . $mariadb->name . ' soft deleting\n';
$mariadb->delete();
continue;
}
if (!$mariadb->destination()) {
echo 'Mariadb without destination: ' . $mariadb->name . ' soft deleting\n';
$mariadb->delete();
continue;
}
if (!data_get($mariadb, 'destination.server')) {
echo 'Mariadb without server: ' . $mariadb->name . ' soft deleting\n';
$mariadb->delete();
continue;
}
}
} catch (\Throwable $e) {
echo "Error in mariadb: {$e->getMessage()}\n";
}
try {
$services = Service::all();
foreach ($services as $service) {
if (!data_get($service, 'environment')) {
echo 'Service without environment: ' . $service->name . ' soft deleting\n';
$service->delete();
continue;
}
if (!$service->destination()) {
echo 'Service without destination: ' . $service->name . ' soft deleting\n';
$service->delete();
continue;
}
if (!data_get($service, 'server')) {
echo 'Service without server: ' . $service->name . ' soft deleting\n';
$service->delete();
continue;
}
}
} catch (\Throwable $e) {
echo "Error in service: {$e->getMessage()}\n";
}
try {
$serviceApplications = ServiceApplication::all();
foreach ($serviceApplications as $service) {
if (!data_get($service, 'service')) {
echo 'ServiceApplication without service: ' . $service->name . ' soft deleting\n';
$service->delete();
continue;
}
}
} catch (\Throwable $e) {
echo "Error in serviceApplications: {$e->getMessage()}\n";
}
try {
$serviceDatabases = ServiceDatabase::all();
foreach ($serviceDatabases as $service) {
if (!data_get($service, 'service')) {
echo 'ServiceDatabase without service: ' . $service->name . ' soft deleting\n';
$service->delete();
continue;
}
}
} catch (\Throwable $e) {
echo "Error in ServiceDatabases: {$e->getMessage()}\n";
}
}
}

20
app/Jobs/DeleteResourceJob.php
View File

@ -30,17 +30,7 @@ public function __construct(public Application|Service|StandalonePostgresql|Stan
public function handle()
{
try {
$server = $this->resource->destination->server;
$this->resource->delete();
if (!$server->isFunctional()) {
if ($this->resource->type() === 'service') {
ray('dispatching delete service');
DeleteService::dispatch($this->resource);
} else {
$this->resource->forceDelete();
}
return 'Server is not functional';
}
switch ($this->resource->type()) {
case 'application':
StopApplication::run($this->resource);
@ -60,15 +50,15 @@ public function handle()
case 'standalone-mariadb':
StopDatabase::run($this->resource);
break;
}
if ($this->resource->type() === 'service') {
DeleteService::dispatch($this->resource);
} else {
$this->resource->forceDelete();
case 'service':
DeleteService::dispatch($this->resource);
break;
}
} catch (\Throwable $e) {
send_internal_notification('ContainerStoppingJob failed with: ' . $e->getMessage());
throw $e;
} finally {
$this->resource->forceDelete();
}
}
}

3
app/Models/Application.php
View File

@ -39,6 +39,7 @@ protected static function booted()
]);
});
static::deleting(function ($application) {
$application->update(['fqdn' => null]);
$application->settings()->delete();
$storages = $application->persistentStorages()->get();
$server = data_get($application, 'destination.server');
@ -52,8 +53,6 @@ protected static function booted()
$application->environment_variables_preview()->delete();
});
}
// Build packs / deployment types
public function servers(): Collection
{

2
app/Models/Service.php
View File

@ -2,13 +2,11 @@
namespace App\Models;
use App\Actions\Service\DeleteService;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Relations\HasMany;
use Illuminate\Database\Eloquent\SoftDeletes;
use Illuminate\Support\Collection;
use Illuminate\Support\Str;
use Symfony\Component\Yaml\Yaml;
class Service extends BaseModel
{

2
app/Models/ServiceApplication.php
View File

@ -5,7 +5,6 @@
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\SoftDeletes;
use Illuminate\Support\Facades\Cache;
class ServiceApplication extends BaseModel
{
@ -15,6 +14,7 @@ class ServiceApplication extends BaseModel
protected static function booted()
{
static::deleting(function ($service) {
$service->update(['fqdn' => null]);
$service->persistentStorages()->delete();
$service->fileStorages()->delete();
});

7
bootstrap/helpers/shared.php
View File

@ -22,6 +22,7 @@
use App\Notifications\Channels\TelegramChannel;
use App\Notifications\Internal\GeneralNotification;
use DanHarrin\LivewireRateLimiting\Exceptions\TooManyRequestsException;
use Illuminate\Database\UniqueConstraintViolationException;
use Illuminate\Mail\Message;
use Illuminate\Notifications\Messages\MailMessage;
use Illuminate\Support\Facades\Cache;
@ -107,6 +108,12 @@ function handleError(?Throwable $error = null, ?Livewire\Component $livewire = n
}
return "Too many requests. Please try again in {$error->secondsUntilAvailable} seconds.";
}
if ($error instanceof UniqueConstraintViolationException) {
if (isset($livewire)) {
return $livewire->dispatch('error', "A resource with the same name already exists.");
}
return "A resource with the same name already exists.";
}
if ($error instanceof Throwable) {
$message = $error->getMessage();