From d18de24cf9c5738fd0b8ba47e1b9d4b70353d921 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 9 Aug 2023 14:44:36 +0200 Subject: [PATCH] wip: scheduled backups fix: file locations vendor unlocking --- app/Actions/Database/StartPostgresql.php | 18 ++++--- .../Controllers/ApplicationController.php | 1 - .../Project/Database/Postgresql/General.php | 3 ++ app/Jobs/ApplicationDeploymentJob.php | 30 ++++++++---- app/Jobs/BackupDatabaseJob.php | 49 +++++++++++++++++-- app/Models/S3Storage.php | 1 - app/Models/StandalonePostgresql.php | 2 +- app/Models/Team.php | 5 ++ bootstrap/helpers/shared.php | 13 +++++ config/filesystems.php | 6 +++ ...reate_scheduled_database_backups_table.php | 2 + database/seeders/DatabaseSeeder.php | 1 + .../seeders/ScheduledDatabaseBackupSeeder.php | 13 +---- .../seeders/StandalonePostgresqlSeeder.php | 7 --- docker-compose.dev.yml | 4 +- docker-compose.prod.yml | 4 +- .../database/postgresql/general.blade.php | 2 +- scripts/install.sh | 3 +- 18 files changed, 116 insertions(+), 48 deletions(-) diff --git a/app/Actions/Database/StartPostgresql.php b/app/Actions/Database/StartPostgresql.php index 444af33b3..27e0620f1 100644 --- a/app/Actions/Database/StartPostgresql.php +++ b/app/Actions/Database/StartPostgresql.php @@ -12,17 +12,17 @@ class StartPostgresql public StandalonePostgresql $database; public array $commands = []; public array $init_scripts = []; - public string $base_dir; + public string $configuration_dir; public function __invoke(Server $server, StandalonePostgresql $database) { $this->database = $database; $container_name = generate_container_name($this->database->uuid); - $this->base_dir = '/data/coolify/databases/' . $container_name; + $this->configuration_dir = database_configuration_dir() . '/' . $container_name; $this->commands = [ - "mkdir -p $this->base_dir", - "mkdir -p $this->base_dir/docker-entrypoint-initdb.d/" + "mkdir -p $this->configuration_dir", + "mkdir -p $this->configuration_dir/docker-entrypoint-initdb.d/" ]; $persistent_storages = $this->generate_local_persistent_volumes(); @@ -92,8 +92,10 @@ class StartPostgresql } $docker_compose = Yaml::dump($docker_compose, 10); $docker_compose_base64 = base64_encode($docker_compose); - $this->commands[] = "echo '{$docker_compose_base64}' | base64 -d > $this->base_dir/docker-compose.yml"; - $this->commands[] = "docker compose -f $this->base_dir/docker-compose.yml up -d"; + $this->commands[] = "echo '{$docker_compose_base64}' | base64 -d > $this->configuration_dir/docker-compose.yml"; + $readme = generate_readme_file($this->database->name, now()); + $this->commands[] = "echo '{$readme}' > $this->configuration_dir/README.md"; + $this->commands[] = "docker compose -f $this->configuration_dir/docker-compose.yml up -d"; return remote_process($this->commands, $server); } @@ -155,8 +157,8 @@ class StartPostgresql $filename = data_get($init_script, 'filename'); $content = data_get($init_script, 'content'); $content_base64 = base64_encode($content); - $this->commands[] = "echo '{$content_base64}' | base64 -d > $this->base_dir/docker-entrypoint-initdb.d/{$filename}"; - $this->init_scripts[] = "$this->base_dir/docker-entrypoint-initdb.d/{$filename}"; + $this->commands[] = "echo '{$content_base64}' | base64 -d > $this->configuration_dir/docker-entrypoint-initdb.d/{$filename}"; + $this->init_scripts[] = "$this->configuration_dir/docker-entrypoint-initdb.d/{$filename}"; } } } diff --git a/app/Http/Controllers/ApplicationController.php b/app/Http/Controllers/ApplicationController.php index 3eb638426..41ae8f3ad 100644 --- a/app/Http/Controllers/ApplicationController.php +++ b/app/Http/Controllers/ApplicationController.php @@ -24,7 +24,6 @@ class ApplicationController extends Controller if (!$application) { return redirect()->route('dashboard'); } - ray($application->persistentStorages()->get()); return view('project.application.configuration', ['application' => $application]); } diff --git a/app/Http/Livewire/Project/Database/Postgresql/General.php b/app/Http/Livewire/Project/Database/Postgresql/General.php index 5977194cb..5f9da02ff 100644 --- a/app/Http/Livewire/Project/Database/Postgresql/General.php +++ b/app/Http/Livewire/Project/Database/Postgresql/General.php @@ -78,6 +78,9 @@ class General extends Component $this->emit('error', 'Filename already exists.'); return; } + if (!isset($this->database->init_scripts)) { + $this->database->init_scripts = []; + } $this->database->init_scripts = array_merge($this->database->init_scripts, [ [ 'index' => count($this->database->init_scripts), diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index 2151219af..c5695804b 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -15,13 +15,13 @@ use App\Models\SwarmDocker; use App\Notifications\Application\DeploymentFailed; use App\Notifications\Application\DeploymentSuccess; use App\Traits\ExecuteRemoteCommand; +use Exception; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; use Illuminate\Support\Collection; -use Illuminate\Support\Facades\Storage; use Illuminate\Support\Str; use Spatie\Url\Url; use Symfony\Component\Yaml\Yaml; @@ -51,6 +51,7 @@ class ApplicationDeploymentJob implements ShouldQueue private string $container_name; private string $workdir; + private string $configuration_dir; private string $build_workdir; private string $build_image_name; private string $production_image_name; @@ -58,6 +59,7 @@ class ApplicationDeploymentJob implements ShouldQueue private $build_args; private $env_args; private $docker_compose; + private $docker_compose_base64; private $log_model; private Collection $saved_outputs; @@ -78,9 +80,9 @@ class ApplicationDeploymentJob implements ShouldQueue $this->source = $this->application->source->getMorphClass()::where('id', $this->application->source->id)->first(); $this->destination = $this->application->destination->getMorphClass()::where('id', $this->application->destination->id)->first(); $this->server = $this->destination->server; - $this->private_key_location = save_private_key_for_server($this->server); $this->workdir = "/artifacts/{$this->deployment_uuid}"; + $this->configuration_dir = application_configuration_dir() . "/{$this->application->uuid}"; $this->build_workdir = "{$this->workdir}" . rtrim($this->application->base_directory, '/'); $this->is_debug_enabled = $this->application->settings->is_debug_enabled; @@ -122,12 +124,23 @@ class ApplicationDeploymentJob implements ShouldQueue } if ($this->application->fqdn) dispatch(new ProxyStartJob($this->server)); $this->next(ApplicationDeploymentStatus::FINISHED->value); - } catch (\Exception $e) { + } catch (Exception $e) { ray($e); $this->fail($e); } finally { - if (isset($this->docker_compose)) { - Storage::disk('deployments')->put(Str::kebab($this->application->name) . '/docker-compose.yml', $this->docker_compose); + if (isset($this->docker_compose_base64)) { + $readme = generate_readme_file($this->application->name, $this->application_deployment_queue->updated_at); + $this->execute_remote_command( + [ + "mkdir -p $this->configuration_dir" + ], + [ + "echo '{$this->docker_compose_base64}' | base64 -d > $this->configuration_dir/docker-compose.yml", + ], + [ + "echo '{$readme}' > $this->configuration_dir/README.md", + ] + ); } $this->execute_remote_command( [ @@ -135,7 +148,6 @@ class ApplicationDeploymentJob implements ShouldQueue "hidden" => true, ] ); - // ray()->measure(); } } @@ -368,8 +380,8 @@ class ApplicationDeploymentJob implements ShouldQueue $docker_compose['volumes'] = $volume_names; } $this->docker_compose = Yaml::dump($docker_compose, 10); - $docker_compose_base64 = base64_encode($this->docker_compose); - $this->execute_remote_command([$this->execute_in_builder("echo '{$docker_compose_base64}' | base64 -d > {$this->workdir}/docker-compose.yml"), "hidden" => true]); + $this->docker_compose_base64 = base64_encode($this->docker_compose); + $this->execute_remote_command([$this->execute_in_builder("echo '{$this->docker_compose_base64}' | base64 -d > {$this->workdir}/docker-compose.yml"), "hidden" => true]); } private function generate_local_persistent_volumes() @@ -591,7 +603,7 @@ COPY ./nginx.conf /etc/nginx/conf.d/default.conf"); $tag = Str::of("{$this->commit}-{$this->application->id}-{$this->pull_request_id}"); if (strlen($tag) > 128) { $tag = $tag->substr(0, 128); - }; + } $this->build_image_name = "{$this->application->git_repository}:{$tag}-build"; $this->production_image_name = "{$this->application->uuid}:{$tag}"; diff --git a/app/Jobs/BackupDatabaseJob.php b/app/Jobs/BackupDatabaseJob.php index 276451ad0..1b353f63b 100644 --- a/app/Jobs/BackupDatabaseJob.php +++ b/app/Jobs/BackupDatabaseJob.php @@ -2,23 +2,66 @@ namespace App\Jobs; +use App\Models\ScheduledDatabaseBackup; +use App\Models\Server; +use App\Models\StandalonePostgresql; +use App\Models\Team; +use Carbon\Carbon; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; +use Throwable; class BackupDatabaseJob implements ShouldQueue { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; - public function __construct(public $backup) + public Team|null $team = null; + public Server $server; + public ScheduledDatabaseBackup|null $backup; + public string $database_type; + public StandalonePostgresql $database; + public string $status; + + public function __construct($backup) { + $this->backup = $backup; + $this->team = Team::find($backup->team_id); + $this->database = $this->backup->database->first(); + $this->database_type = $this->database->type(); + $this->server = $this->database->destination->server; + $this->status = $this->database->status; } public function handle() { - ray('BackupDatabaseJob'); - ray($this->backup); + if ($this->status !== 'running') { + ray('database not running'); + return; + } + if ($this->database_type === 'standalone-postgresql') { + $this->backup_standalone_postgresql(); + } + } + + private function backup_standalone_postgresql() + { + try { + $backup_filename = backup_dir() . "/{$this->database->uuid}/dumpall-" . Carbon::now()->timestamp . ".sql"; + $commands[] = "mkdir -p " . backup_dir(); + $commands[] = "mkdir -p " . backup_dir() . "/{$this->database->uuid}"; + $commands[] = "docker exec {$this->database->uuid} pg_dumpall -U {$this->database->postgres_user} > $backup_filename"; + instant_remote_process($commands, $this->server); + ray('Backup done for ' . $this->database->uuid . ' at ' . $this->server->name . ':' . $backup_filename); + if (!$this->backup->keep_locally) { + $commands[] = "rm -rf $backup_filename"; + instant_remote_process($commands, $this->server); + } + } catch (Throwable $th) { + ray($th); + //throw $th; + } } } diff --git a/app/Models/S3Storage.php b/app/Models/S3Storage.php index 754f53635..69be23f0a 100644 --- a/app/Models/S3Storage.php +++ b/app/Models/S3Storage.php @@ -30,5 +30,4 @@ class S3Storage extends BaseModel set_s3_target($this); return \Storage::disk('custom-s3')->files(); } - } diff --git a/app/Models/StandalonePostgresql.php b/app/Models/StandalonePostgresql.php index 6cbcffab2..55fe96e91 100644 --- a/app/Models/StandalonePostgresql.php +++ b/app/Models/StandalonePostgresql.php @@ -49,7 +49,7 @@ class StandalonePostgresql extends BaseModel ); } - public function type() + public function type(): string { return 'standalone-postgresql'; } diff --git a/app/Models/Team.php b/app/Models/Team.php index c377ca955..30b3980d5 100644 --- a/app/Models/Team.php +++ b/app/Models/Team.php @@ -83,4 +83,9 @@ class Team extends Model implements SendsDiscord, SendsEmail $sources = $sources->merge($github_apps)->merge($gitlab_apps); return $sources; } + + public function s3() + { + return $this->hasOne(S3Storage::class); + } } diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php index 2954c37eb..388d7f31c 100644 --- a/bootstrap/helpers/shared.php +++ b/bootstrap/helpers/shared.php @@ -7,6 +7,19 @@ use Illuminate\Support\Facades\Route; use Illuminate\Support\Str; use Visus\Cuid2\Cuid2; +function application_configuration_dir() :string { + return '/data/coolify/applications'; +} +function database_configuration_dir(): string { + return '/data/coolify/databases'; +} +function backup_dir(): string { + return '/data/coolify/backups'; +} +function generate_readme_file(string $name, string $updated_at): string { + return "Resource name: {$name}\nLatest Deployment Date: {$updated_at}"; +} + function general_error_handler(\Throwable|null $err = null, $that = null, $isJson = false, $customErrorMessage = null) { try { diff --git a/config/filesystems.php b/config/filesystems.php index a7a0ba465..5c9ffc39c 100644 --- a/config/filesystems.php +++ b/config/filesystems.php @@ -62,6 +62,12 @@ return [ 'visibility' => 'private', 'throw' => false, ], + 'backups' => [ + 'driver' => 'local', + 'root' => storage_path('app/backups'), + 'visibility' => 'private', + 'throw' => false, + ], 's3' => [ 'driver' => 's3', diff --git a/database/migrations/2023_08_08_150103_create_scheduled_database_backups_table.php b/database/migrations/2023_08_08_150103_create_scheduled_database_backups_table.php index 991e1d1b7..f8c4f62db 100644 --- a/database/migrations/2023_08_08_150103_create_scheduled_database_backups_table.php +++ b/database/migrations/2023_08_08_150103_create_scheduled_database_backups_table.php @@ -10,6 +10,8 @@ return new class extends Migration { Schema::create('scheduled_database_backups', function (Blueprint $table) { $table->id(); $table->boolean('enabled')->default(true); + $table->boolean('keep_locally')->default(true); + $table->string('save_s3')->default(true); $table->string('frequency'); $table->morphs('database'); $table->foreignId('team_id'); diff --git a/database/seeders/DatabaseSeeder.php b/database/seeders/DatabaseSeeder.php index 3ead195e4..c76beb63f 100644 --- a/database/seeders/DatabaseSeeder.php +++ b/database/seeders/DatabaseSeeder.php @@ -32,6 +32,7 @@ class DatabaseSeeder extends Seeder LocalPersistentVolumeSeeder::class, S3StorageSeeder::class, StandalonePostgresqlSeeder::class, + ScheduledDatabaseBackupSeeder::class ]); } } diff --git a/database/seeders/ScheduledDatabaseBackupSeeder.php b/database/seeders/ScheduledDatabaseBackupSeeder.php index 9d7032181..c471ce3d1 100644 --- a/database/seeders/ScheduledDatabaseBackupSeeder.php +++ b/database/seeders/ScheduledDatabaseBackupSeeder.php @@ -13,22 +13,11 @@ class ScheduledDatabaseBackupSeeder extends Seeder public function run(): void { ScheduledDatabaseBackup::create([ + 'enabled' => true, 'frequency' => '* * * * *', 'database_id' => 1, 'database_type' => 'App\Models\StandalonePostgresql', 'team_id' => 0, ]); - ScheduledDatabaseBackup::create([ - 'frequency' => '*/2 * * * *', - 'database_id' => 1, - 'database_type' => 'App\Models\StandalonePostgresql', - 'team_id' => 0, - ]); - ScheduledDatabaseBackup::create([ - 'frequency' => '*/3 * * * *', - 'database_id' => 1, - 'database_type' => 'App\Models\StandalonePostgresql', - 'team_id' => 0, - ]); } } diff --git a/database/seeders/StandalonePostgresqlSeeder.php b/database/seeders/StandalonePostgresqlSeeder.php index fe6010247..fefeee38e 100644 --- a/database/seeders/StandalonePostgresqlSeeder.php +++ b/database/seeders/StandalonePostgresqlSeeder.php @@ -17,13 +17,6 @@ class StandalonePostgresqlSeeder extends Seeder 'environment_id' => 1, 'destination_id' => 1, 'destination_type' => StandaloneDocker::class, - 'init_scripts' => [ - [ - 'index' => 0, - 'filename' => 'init_test_db.sql', - 'content' => 'CREATE DATABASE test;' - ] - ] ]); } } diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index bcd06e788..75320e374 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -54,13 +54,13 @@ services: container_name: coolify-testing-host volumes: - /var/run/docker.sock:/var/run/docker.sock - - "./_data/coolify/testing-local-docker-container/proxy:/data/coolify/proxy" + - "./_data/_servers/testing-local-docker-container:/data/coolify" testing-host-2: <<: *testing-host-base container_name: coolify-testing-host-2 volumes: - /var/run/docker.sock:/var/run/docker.sock - - "./_data/coolify/testing-local-docker-container-2/proxy:/data/coolify/proxy" + - "./_data/_servers/testing-local-docker-container-2:/data/coolify" mailpit: image: "axllent/mailpit:latest" container_name: coolify-mail diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index a2d894aa4..73b29c8ef 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -8,7 +8,9 @@ services: target: /var/www/html/.env read_only: true - /data/coolify/ssh:/var/www/html/storage/app/ssh - - /data/coolify/deployments:/var/www/html/storage/app/deployments + - /data/coolify/applications:/var/www/html/storage/app/applications + - /data/coolify/databases:/var/www/html/storage/app/databases + - /data/coolify/backups:/var/www/html/storage/app/backups environment: - APP_ID - APP_ENV=production diff --git a/resources/views/livewire/project/database/postgresql/general.blade.php b/resources/views/livewire/project/database/postgresql/general.blade.php index ab99311a7..af5d7db56 100644 --- a/resources/views/livewire/project/database/postgresql/general.blade.php +++ b/resources/views/livewire/project/database/postgresql/general.blade.php @@ -64,7 +64,7 @@ + Add
- @forelse(data_get($database,'init_scripts') as $script) + @forelse(data_get($database,'init_scripts', []) as $script) @empty
No initialization scripts found.
diff --git a/scripts/install.sh b/scripts/install.sh index 530bc4ea6..5c3014353 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -12,7 +12,7 @@ CDN="https://cdn.coollabs.io/coolify" OS_TYPE=$(cat /etc/os-release | grep -w "ID" | cut -d "=" -f 2 | tr -d '"') OS_VERSION=$(cat /etc/os-release | grep -w "VERSION_ID" | cut -d "=" -f 2 | tr -d '"') LATEST_VERSION=$(curl --silent $CDN/versions.json | grep -i version | sed -n '2p' | xargs | awk '{print $2}' | tr -d ',') -DATE=$(date +"%Y%m%d-%H%M%S") +DATE=$(date +"%Y%m%d-%H%M%S") if [ $EUID != 0 ]; then echo "Please run as root" @@ -84,7 +84,6 @@ fi echo -e "-------------" -mkdir -p /data/coolify/deployments mkdir -p /data/coolify/ssh/keys mkdir -p /data/coolify/ssh/mux mkdir -p /data/coolify/source