fix: database backups

This commit is contained in:
Andras Bacsai 2023-10-10 13:10:43 +02:00
parent 84d8e35411
commit 24fa56762e
25 changed files with 149 additions and 65 deletions

View File

@ -8,6 +8,7 @@ class BackupEdit extends Component
{
public $backup;
public $s3s;
public ?string $status = null;
public array $parameters;
protected $rules = [

View File

@ -64,7 +64,7 @@ public function submit()
}
$this->storage->team_id = currentTeam()->id;
$this->storage->testConnection();
$this->emit('success', 'Connection is working. Tested with "ListObjectsV2" action.');
$this->storage->is_usable = true;
$this->storage->save();
return redirect()->route('team.storages.show', $this->storage->uuid);
} catch (\Throwable $e) {

View File

@ -9,6 +9,7 @@ class Form extends Component
{
public S3Storage $storage;
protected $rules = [
'storage.is_usable' => 'nullable|boolean',
'storage.name' => 'nullable|min:3|max:255',
'storage.description' => 'nullable|min:3|max:255',
'storage.region' => 'required|max:255',
@ -18,6 +19,7 @@ class Form extends Component
'storage.endpoint' => 'required|url|max:255',
];
protected $validationAttributes = [
'storage.is_usable' => 'Is Usable',
'storage.name' => 'Name',
'storage.description' => 'Description',
'storage.region' => 'Region',

View File

@ -31,7 +31,7 @@ class DatabaseBackupJob implements ShouldQueue, ShouldBeEncrypted
public ?string $container_name = null;
public ?ScheduledDatabaseBackupExecution $backup_log = null;
public string $backup_status;
public string $backup_status = 'failed';
public ?string $backup_location = null;
public string $backup_dir;
public string $backup_file;
@ -74,7 +74,7 @@ public function handle(): void
$ip = Str::slug($this->server->ip);
$this->backup_dir = backup_dir() . "/coolify" . "/coolify-db-$ip";
}
$this->backup_file = "/pg_dump-" . Carbon::now()->timestamp . ".dump";
$this->backup_file = "/pg-backup-customformat-" . Carbon::now()->timestamp . ".backup";
$this->backup_location = $this->backup_dir . $this->backup_file;
$this->backup_log = ScheduledDatabaseBackupExecution::create([
@ -90,10 +90,17 @@ public function handle(): void
$this->upload_to_s3();
}
$this->save_backup_logs();
$this->team->notify(new BackupSuccess($this->backup, $this->database));
$this->backup_status = 'success';
} catch (\Throwable $e) {
ray($e->getMessage());
$this->backup_status = 'failed';
send_internal_notification('DatabaseBackupJob failed with: ' . $e->getMessage());
$this->team->notify(new BackupFailed($this->backup, $this->database, $this->backup_output));
throw $e;
} finally {
$this->backup_log->update([
'status' => $this->backup_status,
]);
}
}
@ -103,28 +110,15 @@ private function backup_standalone_postgresql(): void
ray($this->backup_dir);
$commands[] = "mkdir -p " . $this->backup_dir;
$commands[] = "docker exec $this->container_name pg_dump -Fc -U {$this->database->postgres_user} > $this->backup_location";
$this->backup_output = instant_remote_process($commands, $this->server);
$this->backup_output = trim($this->backup_output);
if ($this->backup_output === '') {
$this->backup_output = null;
}
ray('Backup done for ' . $this->container_name . ' at ' . $this->server->name . ':' . $this->backup_location);
$this->backup_status = 'success';
$this->team->notify(new BackupSuccess($this->backup, $this->database));
} catch (\Throwable $e) {
$this->backup_status = 'failed';
$this->add_to_backup_output($e->getMessage());
ray('Backup failed for ' . $this->container_name . ' at ' . $this->server->name . ':' . $this->backup_location . '\n\nError:' . $e->getMessage());
$this->team->notify(new BackupFailed($this->backup, $this->database, $this->backup_output));
} finally {
$this->backup_log->update([
'status' => $this->backup_status,
]);
}
}
@ -163,11 +157,16 @@ private function upload_to_s3(): void
}
$key = $this->s3->key;
$secret = $this->s3->secret;
// $region = $this->s3->region;
// $region = $this->s3->region;
$bucket = $this->s3->bucket;
$endpoint = $this->s3->endpoint;
$this->s3->testConnection();
if (isDev()) {
$commands[] = "docker run --pull=always -d --network {$this->database->destination->network} --name backup-of-{$this->backup->uuid} --rm -v coolify_coolify-data-dev:/data/coolify:ro ghcr.io/coollabsio/coolify-helper >/dev/null 2>&1";
} else {
$commands[] = "docker run --pull=always -d --network {$this->database->destination->network} --name backup-of-{$this->backup->uuid} --rm -v $this->backup_location:$this->backup_location:ro ghcr.io/coollabsio/coolify-helper >/dev/null 2>&1";
}
$commands[] = "docker run --pull=always -d --network {$this->database->destination->network} --name backup-of-{$this->backup->uuid} --rm -v $this->backup_location:$this->backup_location:ro ghcr.io/coollabsio/coolify-helper";
$commands[] = "docker exec backup-of-{$this->backup->uuid} mc config host add temporary {$endpoint} $key $secret";
$commands[] = "docker exec backup-of-{$this->backup->uuid} mc cp $this->backup_location temporary/$bucket{$this->backup_dir}/";
instant_remote_process($commands, $this->server);
@ -175,7 +174,7 @@ private function upload_to_s3(): void
ray('Uploaded to S3. ' . $this->backup_location . ' to s3://' . $bucket . $this->backup_dir);
} catch (\Throwable $e) {
$this->add_to_backup_output($e->getMessage());
ray($e->getMessage());
throw $e;
} finally {
$command = "docker rm -f backup-of-{$this->backup->uuid}";
instant_remote_process([$command], $this->server);

View File

@ -3,6 +3,8 @@
namespace App\Models;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Notifications\Messages\MailMessage;
use Illuminate\Support\Facades\Storage;
class S3Storage extends BaseModel
{
@ -10,6 +12,7 @@ class S3Storage extends BaseModel
protected $guarded = [];
protected $casts = [
'is_usable' => 'boolean',
'key' => 'encrypted',
'secret' => 'encrypted',
];
@ -19,7 +22,15 @@ static public function ownedByCurrentTeam(array $select = ['*'])
$selectArray = collect($select)->concat(['id']);
return S3Storage::whereTeamId(currentTeam()->id)->select($selectArray->all())->orderBy('name');
}
public function isUsable()
{
return $this->is_usable;
}
public function team()
{
return $this->belongsTo(Team::class);
}
public function awsUrl()
{
return "{$this->endpoint}/{$this->bucket}";
@ -27,7 +38,34 @@ public function awsUrl()
public function testConnection()
{
set_s3_target($this);
return \Storage::disk('custom-s3')->files();
try {
set_s3_target($this);
Storage::disk('custom-s3')->files();
$this->unusable_email_sent = false;
$this->is_usable = true;
return;
} catch (\Throwable $e) {
$this->is_usable = false;
if ($this->unusable_email_sent === false) {
$mail = new MailMessage();
$mail->subject('Coolify: S3 Storage Connection Error');
$mail->view('emails.s3-connection-error', ['name' => $this->name, 'reason' => $e->getMessage(), 'url' => route('team.storages.show', ['storage_uuid' => $this->uuid])]);
$users = collect([]);
$members = $this->team->members()->get();
foreach ($members as $user) {
if ($user->isAdmin()) {
$users->push($user);
}
}
foreach ($users as $user) {
send_user_an_email($mail, $user->email);
}
$this->unusable_email_sent = true;
}
throw $e;
} finally {
$this->save();
}
}
}

View File

@ -48,6 +48,7 @@ public function getRecepients($notification)
}
return explode(',', $recipients);
}
public function limits(): Attribute
{
return Attribute::make(
@ -125,7 +126,7 @@ public function sources()
public function s3s()
{
return $this->hasMany(S3Storage::class);
return $this->hasMany(S3Storage::class)->where('is_usable', true);
}
public function trialEnded() {
foreach ($this->servers as $server) {

View File

@ -52,10 +52,10 @@ public function toMail(): MailMessage
$pull_request_id = data_get($this->preview, 'pull_request_id', 0);
$fqdn = $this->fqdn;
if ($pull_request_id === 0) {
$mail->subject(' Deployment failed of ' . $this->application_name . '.');
$mail->subject('Coolify: Deployment failed of ' . $this->application_name . '.');
} else {
$fqdn = $this->preview->fqdn;
$mail->subject(' Deployment failed of pull request #' . $this->preview->pull_request_id . ' of ' . $this->application_name . '.');
$mail->subject('Coolify: Deployment failed of pull request #' . $this->preview->pull_request_id . ' of ' . $this->application_name . '.');
}
$mail->view('emails.application-deployment-failed', [
'name' => $this->application_name,
@ -69,10 +69,10 @@ public function toMail(): MailMessage
public function toDiscord(): string
{
if ($this->preview) {
$message = ' Pull request #' . $this->preview->pull_request_id . ' of **' . $this->application_name . '** (' . $this->preview->fqdn . ') deployment failed: ';
$message = 'Coolify: Pull request #' . $this->preview->pull_request_id . ' of **' . $this->application_name . '** (' . $this->preview->fqdn . ') deployment failed: ';
$message .= '[View Deployment Logs](' . $this->deployment_url . ')';
} else {
$message = ' Deployment failed of **' . $this->application_name . '** (' . $this->fqdn . '): ';
$message = 'Coolify: Deployment failed of **' . $this->application_name . '** (' . $this->fqdn . '): ';
$message .= '[View Deployment Logs](' . $this->deployment_url . ')';
}
return $message;
@ -80,9 +80,9 @@ public function toDiscord(): string
public function toTelegram(): array
{
if ($this->preview) {
$message = ' Pull request #' . $this->preview->pull_request_id . ' of **' . $this->application_name . '** (' . $this->preview->fqdn . ') deployment failed: ';
$message = 'Coolify: Pull request #' . $this->preview->pull_request_id . ' of **' . $this->application_name . '** (' . $this->preview->fqdn . ') deployment failed: ';
} else {
$message = ' Deployment failed of **' . $this->application_name . '** (' . $this->fqdn . '): ';
$message = 'Coolify: Deployment failed of **' . $this->application_name . '** (' . $this->fqdn . '): ';
}
return [
"message" => $message,

View File

@ -52,10 +52,10 @@ public function toMail(): MailMessage
$pull_request_id = data_get($this->preview, 'pull_request_id', 0);
$fqdn = $this->fqdn;
if ($pull_request_id === 0) {
$mail->subject(" New version is deployed of {$this->application_name}");
$mail->subject("Coolify: New version is deployed of {$this->application_name}");
} else {
$fqdn = $this->preview->fqdn;
$mail->subject(" Pull request #{$pull_request_id} of {$this->application_name} deployed successfully");
$mail->subject("Coolify: Pull request #{$pull_request_id} of {$this->application_name} deployed successfully");
}
$mail->view('emails.application-deployment-success', [
'name' => $this->application_name,
@ -69,7 +69,7 @@ public function toMail(): MailMessage
public function toDiscord(): string
{
if ($this->preview) {
$message = ' New PR' . $this->preview->pull_request_id . ' version successfully deployed of ' . $this->application_name . '
$message = 'Coolify: New PR' . $this->preview->pull_request_id . ' version successfully deployed of ' . $this->application_name . '
';
if ($this->preview->fqdn) {
@ -77,7 +77,7 @@ public function toDiscord(): string
}
$message .= '[Deployment logs](' . $this->deployment_url . ')';
} else {
$message = ' New version successfully deployed of ' . $this->application_name . '
$message = 'Coolify: New version successfully deployed of ' . $this->application_name . '
';
if ($this->fqdn) {
@ -90,7 +90,7 @@ public function toDiscord(): string
public function toTelegram(): array
{
if ($this->preview) {
$message = ' New PR' . $this->preview->pull_request_id . ' version successfully deployed of ' . $this->application_name . '';
$message = 'Coolify: New PR' . $this->preview->pull_request_id . ' version successfully deployed of ' . $this->application_name . '';
if ($this->preview->fqdn) {
$buttons[] = [
"text" => "Open Application",

View File

@ -45,7 +45,7 @@ public function toMail(): MailMessage
{
$mail = new MailMessage();
$fqdn = $this->fqdn;
$mail->subject(" {$this->application_name} has been stopped");
$mail->subject("Coolify: {$this->application_name} has been stopped");
$mail->view('emails.application-status-changes', [
'name' => $this->application_name,
'fqdn' => $fqdn,
@ -56,7 +56,7 @@ public function toMail(): MailMessage
public function toDiscord(): string
{
$message = ' ' . $this->application_name . ' has been stopped.
$message = 'Coolify: ' . $this->application_name . ' has been stopped.
';
$message .= '[Open Application in Coolify](' . $this->application_url . ')';
@ -64,7 +64,7 @@ public function toDiscord(): string
}
public function toTelegram(): array
{
$message = ' ' . $this->application_name . ' has been stopped.';
$message = 'Coolify: ' . $this->application_name . ' has been stopped.';
return [
"message" => $message,
"buttons" => [

View File

@ -27,7 +27,7 @@ public function via(object $notifiable): array
public function toMail(): MailMessage
{
$mail = new MailMessage();
$mail->subject(" Container ({$this->name}) has been restarted automatically on {$this->server->name}");
$mail->subject("Coolify: Container ({$this->name}) has been restarted automatically on {$this->server->name}");
$mail->view('emails.container-restarted', [
'containerName' => $this->name,
'serverName' => $this->server->name,
@ -38,12 +38,12 @@ public function toMail(): MailMessage
public function toDiscord(): string
{
$message = " Container ({$this->name}) has been restarted automatically on {$this->server->name}";
$message = "Coolify: Container ({$this->name}) has been restarted automatically on {$this->server->name}";
return $message;
}
public function toTelegram(): array
{
$message = " Container ({$this->name}) has been restarted automatically on {$this->server->name}";
$message = "Coolify: Container ({$this->name}) has been restarted automatically on {$this->server->name}";
$payload = [
"message" => $message,
];

View File

@ -26,7 +26,7 @@ public function via(object $notifiable): array
public function toMail(): MailMessage
{
$mail = new MailMessage();
$mail->subject("⛔ Container {$this->name} has been stopped on {$this->server->name}");
$mail->subject("Coolify: Container ({$this->name}) has been stopped on {$this->server->name}");
$mail->view('emails.container-stopped', [
'containerName' => $this->name,
'serverName' => $this->server->name,
@ -37,12 +37,12 @@ public function toMail(): MailMessage
public function toDiscord(): string
{
$message = "⛔ Container {$this->name} has been stopped on {$this->server->name}";
$message = "Coolify: Container ({$this->name}) has been stopped on {$this->server->name}";
return $message;
}
public function toTelegram(): array
{
$message = " Container ($this->name} has been stopped on {$this->server->name}";
$message = "Coolify: Container ($this->name} has been stopped on {$this->server->name}";
$payload = [
"message" => $message,
];

View File

@ -30,7 +30,7 @@ public function via(object $notifiable): array
public function toMail(): MailMessage
{
$mail = new MailMessage();
$mail->subject(" [ACTION REQUIRED] Backup FAILED for {$this->database->name}");
$mail->subject("Coolify: [ACTION REQUIRED] Backup FAILED for {$this->database->name}");
$mail->view('emails.backup-failed', [
'name' => $this->name,
'frequency' => $this->frequency,
@ -41,11 +41,11 @@ public function toMail(): MailMessage
public function toDiscord(): string
{
return " Database backup for {$this->name} with frequency of {$this->frequency} was FAILED.\n\nReason: {$this->output}";
return "Coolify: Database backup for {$this->name} with frequency of {$this->frequency} was FAILED.\n\nReason: {$this->output}";
}
public function toTelegram(): array
{
$message = " Database backup for {$this->name} with frequency of {$this->frequency} was FAILED.\n\nReason: {$this->output}";
$message = "Coolify: Database backup for {$this->name} with frequency of {$this->frequency} was FAILED.\n\nReason: {$this->output}";
return [
"message" => $message,
];

View File

@ -30,7 +30,7 @@ public function via(object $notifiable): array
public function toMail(): MailMessage
{
$mail = new MailMessage();
$mail->subject(" Backup successfully done for {$this->database->name}");
$mail->subject("Coolify: Backup successfully done for {$this->database->name}");
$mail->view('emails.backup-success', [
'name' => $this->name,
'frequency' => $this->frequency,
@ -40,11 +40,11 @@ public function toMail(): MailMessage
public function toDiscord(): string
{
return " Database backup for {$this->name} with frequency of {$this->frequency} was successful.";
return "Coolify: Database backup for {$this->name} with frequency of {$this->frequency} was successful.";
}
public function toTelegram(): array
{
$message = " Database backup for {$this->name} with frequency of {$this->frequency} was successful.";
$message = "Coolify: Database backup for {$this->name} with frequency of {$this->frequency} was successful.";
return [
"message" => $message,
];

View File

@ -45,7 +45,7 @@ public function via(object $notifiable): array
public function toMail(): MailMessage
{
$mail = new MailMessage();
$mail->subject(" Server ({$this->server->name}) revived.");
$mail->subject("Coolify: Server ({$this->server->name}) revived.");
$mail->view('emails.server-revived', [
'name' => $this->server->name,
]);
@ -54,13 +54,13 @@ public function toMail(): MailMessage
public function toDiscord(): string
{
$message = " Server '{$this->server->name}' revived. All automations & integrations are turned on again!";
$message = "Coolify: Server '{$this->server->name}' revived. All automations & integrations are turned on again!";
return $message;
}
public function toTelegram(): array
{
return [
"message" => " Server '{$this->server->name}' revived. All automations & integrations are turned on again!"
"message" => "Coolify: Server '{$this->server->name}' revived. All automations & integrations are turned on again!"
];
}
}

View File

@ -43,7 +43,7 @@ public function via(object $notifiable): array
public function toMail(): MailMessage
{
$mail = new MailMessage();
$mail->subject(" Server ({$this->server->name}) is unreachable after trying to connect to it 5 times");
$mail->subject("Coolify: Server ({$this->server->name}) is unreachable after trying to connect to it 5 times");
$mail->view('emails.server-lost-connection', [
'name' => $this->server->name,
]);
@ -52,13 +52,13 @@ public function toMail(): MailMessage
public function toDiscord(): string
{
$message = " Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: We automatically try to revive your server. If your server is back online, we will automatically turn on all automations & integrations.";
$message = "Coolify: Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: We automatically try to revive your server. If your server is back online, we will automatically turn on all automations & integrations.";
return $message;
}
public function toTelegram(): array
{
return [
"message" => " Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: We automatically try to revive your server. If your server is back online, we will automatically turn on all automations & integrations."
"message" => "Coolify: Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: We automatically try to revive your server. If your server is back online, we will automatically turn on all automations & integrations."
];
}
}

View File

@ -24,14 +24,14 @@ public function via(object $notifiable): array
public function toMail(): MailMessage
{
$mail = new MailMessage();
$mail->subject("Test Email");
$mail->subject("Coolify: Test Email");
$mail->view('emails.test');
return $mail;
}
public function toDiscord(): string
{
$message = 'This is a test Discord notification from Coolify.';
$message = 'Coolify: This is a test Discord notification from Coolify.';
$message .= "\n\n";
$message .= '[Go to your dashboard](' . base_url() . ')';
return $message;
@ -39,7 +39,7 @@ public function toDiscord(): string
public function toTelegram(): array
{
return [
"message" => 'This is a test Telegram notification from Coolify.',
"message" => 'Coolify: This is a test Telegram notification from Coolify.',
"buttons" => [
[
"text" => "Go to your dashboard",

View File

@ -30,7 +30,7 @@ public function toMail(): MailMessage
$invitation_team = Team::find($invitation->team->id);
$mail = new MailMessage();
$mail->subject('Invitation for ' . $invitation_team->name);
$mail->subject('Coolify: Invitation for ' . $invitation_team->name);
$mail->view('emails.invitation-link', [
'team' => $invitation_team->name,
'email' => $this->user->email,

View File

@ -50,7 +50,7 @@ public function toMail($notifiable)
protected function buildMailMessage($url)
{
$mail = new MailMessage();
$mail->subject('Reset Password');
$mail->subject('Coolify: Reset Password');
$mail->view('emails.reset-password', ['url' => $url, 'count' => config('auth.passwords.' . config('auth.defaults.passwords') . '.expire')]);
return $mail;
}

View File

@ -25,7 +25,7 @@ public function via(): array
public function toMail(): MailMessage
{
$mail = new MailMessage();
$mail->subject('Test Email');
$mail->subject('Coolify: Test Email');
$mail->view('emails.test');
return $mail;
}

View File

@ -0,0 +1,30 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('s3_storages', function (Blueprint $table) {
$table->boolean('is_usable')->default(false);
$table->boolean('unusable_email_sent')->default(false);
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('s3_storages', function (Blueprint $table) {
$table->dropColumn('is_usable');
$table->dropColumn('unusable_email_sent');
});
}
};

View File

@ -0,0 +1,6 @@
<x-emails.layout>
Connection could not be establised with one of your S3 Storage ({{ $name }}). Please fix it
[here]({{ $url }}).
{{ $reason }}
</x-emails.layout>

View File

@ -4,7 +4,9 @@
<x-forms.button type="submit">
Save
</x-forms.button>
<livewire:project.database.backup-now :backup="$backup" />
@if (Str::of($status)->startsWith('running'))
<livewire:project.database.backup-now :backup="$backup" />
@endif
@if ($backup->database_id !== 0)
<x-forms.button isError wire:click="delete">Delete</x-forms.button>
@endif

View File

@ -22,7 +22,7 @@
<x-forms.input type="password" label="Password" readonly id="database.postgres_password" />
</div>
</div>
<livewire:project.database.backup-edit :backup="$backup" :s3s="$s3s" />
<livewire:project.database.backup-edit :backup="$backup" :s3s="$s3s" :status="data_get($database,'status')" />
@else
To configure automatic backup for your Coolify instance, you first need to add as a database resource
into Coolify.

View File

@ -10,12 +10,17 @@
<div class="pb-4">
<h2>Storage Details</h2>
<div>{{ $storage->name }}</div>
@if ($storage->is_usable)
<div> Usable </div>
@else
<div class="text-red-500"> Not Usable </div>
@endif
</div>
<x-forms.button type="submit">
Save
</x-forms.button>
<x-forms.button wire:click="test_s3_connection">
Test Connection
Validate Connection
</x-forms.button>
<x-forms.button isError isModal modalId="deleteS3Storage">
Delete

View File

@ -12,7 +12,7 @@
</x-slot:modalSubmit>
</x-modal>
<div class="pt-6">
<livewire:project.database.backup-edit :backup="$backup" :s3s="$s3s" />
<livewire:project.database.backup-edit :backup="$backup" :s3s="$s3s" :status="data_get($database,'status')" />
<h3 class="py-4">Executions</h3>
<livewire:project.database.backup-executions :backup="$backup" :executions="$executions" />
</div>