Shiloh/lasthourcloud
Shiloh/lasthourcloud
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 4 Wiki Activity

wip: scheduled backups

Browse Source 
fix: file locations vendor unlocking
This commit is contained in:
Andras Bacsai 2023-08-09 14:44:36 +02:00
parent 46909dca85
commit d18de24cf9
18 changed files with 116 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
app/Actions/Database/StartPostgresql.php
View File

@ -12,17 +12,17 @@ class StartPostgresql
public StandalonePostgresql $database;
public array $commands = [];
public array $init_scripts = [];
public string $base_dir;
public string $configuration_dir;
public function __invoke(Server $server, StandalonePostgresql $database)
{
$this->database = $database;
$container_name = generate_container_name($this->database->uuid);
$this->base_dir = '/data/coolify/databases/' . $container_name;
$this->configuration_dir = database_configuration_dir() . '/' . $container_name;
$this->commands = [
"mkdir -p $this->base_dir",
"mkdir -p $this->base_dir/docker-entrypoint-initdb.d/"
"mkdir -p $this->configuration_dir",
"mkdir -p $this->configuration_dir/docker-entrypoint-initdb.d/"
];
$persistent_storages = $this->generate_local_persistent_volumes();
@ -92,8 +92,10 @@ public function __invoke(Server $server, StandalonePostgresql $database)
}
$docker_compose = Yaml::dump($docker_compose, 10);
$docker_compose_base64 = base64_encode($docker_compose);
$this->commands[] = "echo '{$docker_compose_base64}' | base64 -d > $this->base_dir/docker-compose.yml";
$this->commands[] = "docker compose -f $this->base_dir/docker-compose.yml up -d";
$this->commands[] = "echo '{$docker_compose_base64}' | base64 -d > $this->configuration_dir/docker-compose.yml";
$readme = generate_readme_file($this->database->name, now());
$this->commands[] = "echo '{$readme}' > $this->configuration_dir/README.md";
$this->commands[] = "docker compose -f $this->configuration_dir/docker-compose.yml up -d";
return remote_process($this->commands, $server);
}
@ -155,8 +157,8 @@ private function generate_init_scripts()
$filename = data_get($init_script, 'filename');
$content = data_get($init_script, 'content');
$content_base64 = base64_encode($content);
$this->commands[] = "echo '{$content_base64}' | base64 -d > $this->base_dir/docker-entrypoint-initdb.d/{$filename}";
$this->init_scripts[] = "$this->base_dir/docker-entrypoint-initdb.d/{$filename}";
$this->commands[] = "echo '{$content_base64}' | base64 -d > $this->configuration_dir/docker-entrypoint-initdb.d/{$filename}";
$this->init_scripts[] = "$this->configuration_dir/docker-entrypoint-initdb.d/{$filename}";
}
}
}

1
app/Http/Controllers/ApplicationController.php
View File

@ -24,7 +24,6 @@ public function configuration()
if (!$application) {
return redirect()->route('dashboard');
}
ray($application->persistentStorages()->get());
return view('project.application.configuration', ['application' => $application]);
}

3
app/Http/Livewire/Project/Database/Postgresql/General.php
View File

@ -78,6 +78,9 @@ public function save_new_init_script()
$this->emit('error', 'Filename already exists.');
return;
}
if (!isset($this->database->init_scripts)) {
$this->database->init_scripts = [];
}
$this->database->init_scripts = array_merge($this->database->init_scripts, [
[
'index' => count($this->database->init_scripts),

30
app/Jobs/ApplicationDeploymentJob.php
View File

@ -15,13 +15,13 @@
use App\Notifications\Application\DeploymentFailed;
use App\Notifications\Application\DeploymentSuccess;
use App\Traits\ExecuteRemoteCommand;
use Exception;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Storage;
use Illuminate\Support\Str;
use Spatie\Url\Url;
use Symfony\Component\Yaml\Yaml;
@ -51,6 +51,7 @@ class ApplicationDeploymentJob implements ShouldQueue
private string $container_name;
private string $workdir;
private string $configuration_dir;
private string $build_workdir;
private string $build_image_name;
private string $production_image_name;
@ -58,6 +59,7 @@ class ApplicationDeploymentJob implements ShouldQueue
private $build_args;
private $env_args;
private $docker_compose;
private $docker_compose_base64;
private $log_model;
private Collection $saved_outputs;
@ -78,9 +80,9 @@ public function __construct(int $application_deployment_queue_id)
$this->source = $this->application->source->getMorphClass()::where('id', $this->application->source->id)->first();
$this->destination = $this->application->destination->getMorphClass()::where('id', $this->application->destination->id)->first();
$this->server = $this->destination->server;
$this->private_key_location = save_private_key_for_server($this->server);
$this->workdir = "/artifacts/{$this->deployment_uuid}";
$this->configuration_dir = application_configuration_dir() . "/{$this->application->uuid}";
$this->build_workdir = "{$this->workdir}" . rtrim($this->application->base_directory, '/');
$this->is_debug_enabled = $this->application->settings->is_debug_enabled;
@ -122,12 +124,23 @@ public function handle(): void
}
if ($this->application->fqdn) dispatch(new ProxyStartJob($this->server));
$this->next(ApplicationDeploymentStatus::FINISHED->value);
} catch (\Exception $e) {
} catch (Exception $e) {
ray($e);
$this->fail($e);
} finally {
if (isset($this->docker_compose)) {
Storage::disk('deployments')->put(Str::kebab($this->application->name) . '/docker-compose.yml', $this->docker_compose);
if (isset($this->docker_compose_base64)) {
$readme = generate_readme_file($this->application->name, $this->application_deployment_queue->updated_at);
$this->execute_remote_command(
[
"mkdir -p $this->configuration_dir"
],
[
"echo '{$this->docker_compose_base64}' | base64 -d > $this->configuration_dir/docker-compose.yml",
],
[
"echo '{$readme}' > $this->configuration_dir/README.md",
]
);
}
$this->execute_remote_command(
[
@ -135,7 +148,6 @@ public function handle(): void
"hidden" => true,
]
);
// ray()->measure();
}
}
@ -368,8 +380,8 @@ private function generate_compose_file()
$docker_compose['volumes'] = $volume_names;
}
$this->docker_compose = Yaml::dump($docker_compose, 10);
$docker_compose_base64 = base64_encode($this->docker_compose);
$this->execute_remote_command([$this->execute_in_builder("echo '{$docker_compose_base64}' | base64 -d > {$this->workdir}/docker-compose.yml"), "hidden" => true]);
$this->docker_compose_base64 = base64_encode($this->docker_compose);
$this->execute_remote_command([$this->execute_in_builder("echo '{$this->docker_compose_base64}' | base64 -d > {$this->workdir}/docker-compose.yml"), "hidden" => true]);
}
private function generate_local_persistent_volumes()
@ -591,7 +603,7 @@ private function deploy()
$tag = Str::of("{$this->commit}-{$this->application->id}-{$this->pull_request_id}");
if (strlen($tag) > 128) {
$tag = $tag->substr(0, 128);
};
}
$this->build_image_name = "{$this->application->git_repository}:{$tag}-build";
$this->production_image_name = "{$this->application->uuid}:{$tag}";

49
app/Jobs/BackupDatabaseJob.php
View File

@ -2,23 +2,66 @@
namespace App\Jobs;
use App\Models\ScheduledDatabaseBackup;
use App\Models\Server;
use App\Models\StandalonePostgresql;
use App\Models\Team;
use Carbon\Carbon;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Throwable;
class BackupDatabaseJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public function __construct(public $backup)
public Team|null $team = null;
public Server $server;
public ScheduledDatabaseBackup|null $backup;
public string $database_type;
public StandalonePostgresql $database;
public string $status;
public function __construct($backup)
{
$this->backup = $backup;
$this->team = Team::find($backup->team_id);
$this->database = $this->backup->database->first();
$this->database_type = $this->database->type();
$this->server = $this->database->destination->server;
$this->status = $this->database->status;
}
public function handle()
{
ray('BackupDatabaseJob');
ray($this->backup);
if ($this->status !== 'running') {
ray('database not running');
return;
}
if ($this->database_type === 'standalone-postgresql') {
$this->backup_standalone_postgresql();
}
}
private function backup_standalone_postgresql()
{
try {
$backup_filename = backup_dir() . "/{$this->database->uuid}/dumpall-" . Carbon::now()->timestamp . ".sql";
$commands[] = "mkdir -p " . backup_dir();
$commands[] = "mkdir -p " . backup_dir() . "/{$this->database->uuid}";
$commands[] = "docker exec {$this->database->uuid} pg_dumpall -U {$this->database->postgres_user} > $backup_filename";
instant_remote_process($commands, $this->server);
ray('Backup done for ' . $this->database->uuid . ' at ' . $this->server->name . ':' . $backup_filename);
if (!$this->backup->keep_locally) {
$commands[] = "rm -rf $backup_filename";
instant_remote_process($commands, $this->server);
}
} catch (Throwable $th) {
ray($th);
//throw $th;
}
}
}

1
app/Models/S3Storage.php
View File

@ -30,5 +30,4 @@ public function testConnection()
set_s3_target($this);
return \Storage::disk('custom-s3')->files();
}
}

2
app/Models/StandalonePostgresql.php
View File

@ -49,7 +49,7 @@ public function portsMappingsArray(): Attribute
);
}
public function type()
public function type(): string
{
return 'standalone-postgresql';
}

5
app/Models/Team.php
View File

@ -83,4 +83,9 @@ public function sources()
$sources = $sources->merge($github_apps)->merge($gitlab_apps);
return $sources;
}
public function s3()
{
return $this->hasOne(S3Storage::class);
}
}

13
bootstrap/helpers/shared.php
View File

@ -7,6 +7,19 @@
use Illuminate\Support\Str;
use Visus\Cuid2\Cuid2;
function application_configuration_dir() :string {
return '/data/coolify/applications';
}
function database_configuration_dir(): string {
return '/data/coolify/databases';
}
function backup_dir(): string {
return '/data/coolify/backups';
}
function generate_readme_file(string $name, string $updated_at): string {
return "Resource name: {$name}\nLatest Deployment Date: {$updated_at}";
}
function general_error_handler(\Throwable|null $err = null, $that = null, $isJson = false, $customErrorMessage = null)
{
try {

6
config/filesystems.php
View File

@ -62,6 +62,12 @@
'visibility' => 'private',
'throw' => false,
],
'backups' => [
'driver' => 'local',
'root' => storage_path('app/backups'),
'visibility' => 'private',
'throw' => false,
],
's3' => [
'driver' => 's3',

2
database/migrations/2023_08_08_150103_create_scheduled_database_backups_table.php
View File

@ -10,6 +10,8 @@ public function up(): void
Schema::create('scheduled_database_backups', function (Blueprint $table) {
$table->id();
$table->boolean('enabled')->default(true);
$table->boolean('keep_locally')->default(true);
$table->string('save_s3')->default(true);
$table->string('frequency');
$table->morphs('database');
$table->foreignId('team_id');

1
database/seeders/DatabaseSeeder.php
View File

@ -32,6 +32,7 @@ public function run(): void
LocalPersistentVolumeSeeder::class,
S3StorageSeeder::class,
StandalonePostgresqlSeeder::class,
ScheduledDatabaseBackupSeeder::class
]);
}
}

13
database/seeders/ScheduledDatabaseBackupSeeder.php
View File

@ -13,22 +13,11 @@ class ScheduledDatabaseBackupSeeder extends Seeder
public function run(): void
{
ScheduledDatabaseBackup::create([
'enabled' => true,
'frequency' => '* * * * *',
'database_id' => 1,
'database_type' => 'App\Models\StandalonePostgresql',
'team_id' => 0,
]);
ScheduledDatabaseBackup::create([
'frequency' => '*/2 * * * *',
'database_id' => 1,
'database_type' => 'App\Models\StandalonePostgresql',
'team_id' => 0,
]);
ScheduledDatabaseBackup::create([
'frequency' => '*/3 * * * *',
'database_id' => 1,
'database_type' => 'App\Models\StandalonePostgresql',
'team_id' => 0,
]);
}
}

7
database/seeders/StandalonePostgresqlSeeder.php
View File

@ -17,13 +17,6 @@ public function run(): void
'environment_id' => 1,
'destination_id' => 1,
'destination_type' => StandaloneDocker::class,
'init_scripts' => [
[
'index' => 0,
'filename' => 'init_test_db.sql',
'content' => 'CREATE DATABASE test;'
]
]
]);
}
}

4
docker-compose.dev.yml
View File

@ -54,13 +54,13 @@ services:
container_name: coolify-testing-host
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- "./_data/coolify/testing-local-docker-container/proxy:/data/coolify/proxy"
- "./_data/_servers/testing-local-docker-container:/data/coolify"
testing-host-2:
<<: *testing-host-base
container_name: coolify-testing-host-2
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- "./_data/coolify/testing-local-docker-container-2/proxy:/data/coolify/proxy"
- "./_data/_servers/testing-local-docker-container-2:/data/coolify"
mailpit:
image: "axllent/mailpit:latest"
container_name: coolify-mail

4
docker-compose.prod.yml
View File

@ -8,7 +8,9 @@ services:
target: /var/www/html/.env
read_only: true
- /data/coolify/ssh:/var/www/html/storage/app/ssh
- /data/coolify/deployments:/var/www/html/storage/app/deployments
- /data/coolify/applications:/var/www/html/storage/app/applications
- /data/coolify/databases:/var/www/html/storage/app/databases
- /data/coolify/backups:/var/www/html/storage/app/backups
environment:
- APP_ID
- APP_ENV=production

2
resources/views/livewire/project/database/postgresql/general.blade.php
View File

@ -64,7 +64,7 @@
<x-forms.button class="btn" onclick="newInitScript.showModal()">+ Add</x-forms.button>
</div>
<div class="flex flex-col gap-2">
@forelse(data_get($database,'init_scripts') as $script)
@forelse(data_get($database,'init_scripts', []) as $script)
<livewire:project.database.init-script :script="$script" :wire:key="$script['index']"/>
@empty
<div>No initialization scripts found.</div>

3
scripts/install.sh
View File

@ -12,7 +12,7 @@ CDN="https://cdn.coollabs.io/coolify"
OS_TYPE=$(cat /etc/os-release | grep -w "ID" | cut -d "=" -f 2 | tr -d '"')
OS_VERSION=$(cat /etc/os-release | grep -w "VERSION_ID" | cut -d "=" -f 2 | tr -d '"')
LATEST_VERSION=$(curl --silent $CDN/versions.json | grep -i version | sed -n '2p' | xargs | awk '{print $2}' | tr -d ',')
DATE=$(date +"%Y%m%d-%H%M%S")
DATE=$(date +"%Y%m%d-%H%M%S")
if [ $EUID != 0 ]; then
echo "Please run as root"
@ -84,7 +84,6 @@ fi
echo -e "-------------"
mkdir -p /data/coolify/deployments
mkdir -p /data/coolify/ssh/keys
mkdir -p /data/coolify/ssh/mux
mkdir -p /data/coolify/source