Shiloh/lasthourcloud
Shiloh/lasthourcloud
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 4 Wiki Activity

wip: mongodb backup

Browse Source
This commit is contained in:
Andras Bacsai 2023-10-19 17:17:38 +02:00
parent 53f5674771
commit 11bd46b200
8 changed files with 121 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
app/Http/Livewire/Project/Database/Mongodb/General.php
View File

@ -67,7 +67,7 @@ public function instantSave()
StopDatabaseProxy::run($this->database);
$this->emit('success', 'Database is no longer publicly accessible.');
}
$this->getDbUrl();
$this->db_url = $this->database->getDbUrl();
$this->database->save();
} catch(\Throwable $e) {
$this->database->is_public = !$this->database->is_public;
@ -81,16 +81,9 @@ public function refresh(): void
public function mount()
{
$this->getDbUrl();
$this->db_url = $this->database->getDbUrl();
}
public function getDbUrl() {
if ($this->database->is_public) {
$this->db_url = "mongodb://{$this->database->mongo_initdb_root_username}:{$this->database->mongo_initdb_root_password}@{$this->database->destination->server->getIp}:{$this->database->public_port}/?directConnection=true";
} else {
$this->db_url = "mongodb://{$this->database->mongo_initdb_root_username}:{$this->database->mongo_initdb_root_password}@{$this->database->uuid}:27017/?directConnection=true";
}
}
public function render()
{
return view('livewire.project.database.mongodb.general');

12
app/Http/Livewire/Project/Database/Postgresql/General.php
View File

@ -49,15 +49,7 @@ class General extends Component
];
public function mount()
{
$this->getDbUrl();
}
public function getDbUrl() {
if ($this->database->is_public) {
$this->db_url = "postgres://{$this->database->postgres_user}:{$this->database->postgres_password}@{$this->database->destination->server->getIp}:{$this->database->public_port}/{$this->database->postgres_db}";
} else {
$this->db_url = "postgres://{$this->database->postgres_user}:{$this->database->postgres_password}@{$this->database->uuid}:5432/{$this->database->postgres_db}";
}
$this->db_url = $this->database->getDbUrl();
}
public function instantSave()
{
@ -75,7 +67,7 @@ public function instantSave()
StopDatabaseProxy::run($this->database);
$this->emit('success', 'Database is no longer publicly accessible.');
}
$this->getDbUrl();
$this->db_url = $this->database->getDbUrl();
$this->database->save();
} catch(\Throwable $e) {
$this->database->is_public = !$this->database->is_public;

12
app/Http/Livewire/Project/Database/Redis/General.php
View File

@ -63,7 +63,7 @@ public function instantSave()
StopDatabaseProxy::run($this->database);
$this->emit('success', 'Database is no longer publicly accessible.');
}
$this->getDbUrl();
$this->db_url = $this->database->getDbUrl();
$this->database->save();
} catch(\Throwable $e) {
$this->database->is_public = !$this->database->is_public;
@ -77,15 +77,7 @@ public function refresh(): void
public function mount()
{
$this->getDbUrl();
}
public function getDbUrl() {
if ($this->database->is_public) {
$this->db_url = "redis://:{$this->database->redis_password}@{$this->database->destination->server->getIp}:{$this->database->public_port}/0";
} else {
$this->db_url = "redis://:{$this->database->redis_password}@{$this->database->uuid}:6379/0";
}
$this->db_url = $this->database->getDbUrl();
}
public function render()
{

93
app/Jobs/DatabaseBackupJob.php
View File

@ -73,12 +73,24 @@ public function handle(): void
if (is_null($databasesToBackup)) {
if ($databaseType === 'standalone-postgresql') {
$databasesToBackup = [$this->database->postgres_db];
} else if ($databaseType === 'standalone-mongodb') {
$databasesToBackup = ['*'];
} else {
return;
}
} else {
$databasesToBackup = explode(',', $databasesToBackup);
$databasesToBackup = array_map('trim', $databasesToBackup);
if ($databaseType === 'standalone-postgresql') {
// Format: db1,db2,db3
$databasesToBackup = explode(',', $databasesToBackup);
$databasesToBackup = array_map('trim', $databasesToBackup);
} else if ($databaseType === 'standalone-mongodb') {
// Format: db1:collection1,collection2|db2:collection3,collection4
$databasesToBackup = explode('|', $databasesToBackup);
$databasesToBackup = array_map('trim', $databasesToBackup);
ray($databasesToBackup);
} else {
return;
}
}
$this->container_name = $this->database->uuid;
$this->backup_dir = backup_dir() . "/databases/" . Str::of($this->team->name)->slug() . '-' . $this->team->id . '/' . $this->container_name;
@ -93,15 +105,37 @@ public function handle(): void
$size = 0;
ray('Backing up ' . $database);
try {
$this->backup_file = "/pg-dump-$database-" . Carbon::now()->timestamp . ".dmp";
$this->backup_location = $this->backup_dir . $this->backup_file;
$this->backup_log = ScheduledDatabaseBackupExecution::create([
'database_name' => $database,
'filename' => $this->backup_location,
'scheduled_database_backup_id' => $this->backup->id,
]);
if ($databaseType === 'standalone-postgresql') {
$this->backup_file = "/pg-dump-$database-" . Carbon::now()->timestamp . ".dmp";
$this->backup_location = $this->backup_dir . $this->backup_file;
$this->backup_log = ScheduledDatabaseBackupExecution::create([
'database_name' => $database,
'filename' => $this->backup_location,
'scheduled_database_backup_id' => $this->backup->id,
]);
$this->backup_standalone_postgresql($database);
} else if ($databaseType === 'standalone-mongodb') {
if ($database === '*') {
$database = 'all';
$databaseName = 'all';
} else {
if (str($database)->contains(':')) {
$databaseName = str($database)->before(':');
} else {
$databaseName = $database;
}
ray($databaseName);
}
$this->backup_file = "/mongo-dump-$databaseName-" . Carbon::now()->timestamp . ".tar.gz";
$this->backup_location = $this->backup_dir . $this->backup_file;
$this->backup_log = ScheduledDatabaseBackupExecution::create([
'database_name' => $databaseName,
'filename' => $this->backup_location,
'scheduled_database_backup_id' => $this->backup->id,
]);
$this->backup_standalone_mongodb($database);
} else {
throw new \Exception('Unsupported database type');
}
$size = $this->calculate_size();
$this->remove_old_backups();
@ -115,12 +149,14 @@ public function handle(): void
'size' => $size,
]);
} catch (\Throwable $e) {
$this->backup_log->update([
'status' => 'failed',
'message' => $this->backup_output,
'size' => $size,
'filename' => null
]);
if ($this->backup_log) {
$this->backup_log->update([
'status' => 'failed',
'message' => $this->backup_output,
'size' => $size,
'filename' => null
]);
}
send_internal_notification('DatabaseBackupJob failed with: ' . $e->getMessage());
$this->team->notify(new BackupFailed($this->backup, $this->database, $this->backup_output));
throw $e;
@ -131,11 +167,36 @@ public function handle(): void
throw $e;
}
}
private function backup_standalone_mongodb(string $databaseWithCollections): void
{
try {
$url = $this->database->getDbUrl();
if ($databaseWithCollections === 'all') {
$commands[] = "mkdir -p " . $this->backup_dir;
$commands[] = "docker exec $this->container_name mongodump --authenticationDatabase=admin --uri=$url --gzip --archive > $this->backup_location";
} else {
$collectionsToExclude = str($databaseWithCollections)->after(':')->explode(',');
$databaseName = str($databaseWithCollections)->before(':');
$commands[] = "mkdir -p " . $this->backup_dir;
$commands[] = "docker exec $this->container_name mongodump --authenticationDatabase=admin --uri=$url --db $databaseName --gzip --excludeCollection " . $collectionsToExclude->implode(' --excludeCollection ') . " --archive > $this->backup_location";
}
ray($commands);
$this->backup_output = instant_remote_process($commands, $this->server);
$this->backup_output = trim($this->backup_output);
if ($this->backup_output === '') {
$this->backup_output = null;
}
ray('Backup done for ' . $this->container_name . ' at ' . $this->server->name . ':' . $this->backup_location);
} catch (\Throwable $e) {
$this->add_to_backup_output($e->getMessage());
ray('Backup failed for ' . $this->container_name . ' at ' . $this->server->name . ':' . $this->backup_location . '\n\nError:' . $e->getMessage());
throw $e;
}
}
private function backup_standalone_postgresql(string $database): void
{
try {
ray($this->backup_dir);
$commands[] = "mkdir -p " . $this->backup_dir;
$commands[] = "docker exec $this->container_name pg_dump --format=custom --no-acl --no-owner --username {$this->database->postgres_user} $database > $this->backup_location";
$this->backup_output = instant_remote_process($commands, $this->server);

8
app/Models/StandaloneMongodb.php
View File

@ -55,7 +55,13 @@ public function type(): string
{
return 'standalone-mongodb';
}
public function getDbUrl() {
if ($this->is_public) {
return "mongodb://{$this->mongo_initdb_root_username}:{$this->mongo_initdb_root_password}@{$this->destination->server->getIp}:{$this->public_port}/?directConnection=true";
} else {
return "mongodb://{$this->mongo_initdb_root_username}:{$this->mongo_initdb_root_password}@{$this->uuid}:27017/?directConnection=true";
}
}
public function environment()
{
return $this->belongsTo(Environment::class);

8
app/Models/StandalonePostgresql.php
View File

@ -62,6 +62,14 @@ public function type(): string
{
return 'standalone-postgresql';
}
public function getDbUrl(): string
{
if ($this->is_public) {
return "postgres://{$this->postgres_user}:{$this->postgres_password}@{$this->destination->server->getIp}:{$this->public_port}/{$this->postgres_db}";
} else {
return "postgres://{$this->postgres_user}:{$this->postgres_password}@{$this->uuid}:5432/{$this->postgres_db}";
}
}
public function environment()
{

7
app/Models/StandaloneRedis.php
View File

@ -55,6 +55,13 @@ public function type(): string
{
return 'standalone-redis';
}
public function getDbUrl(): string {
if ($this->is_public) {
return "redis://:{$this->redis_password}@{$this->destination->server->getIp}:{$this->public_port}/0";
} else {
return "redis://:{$this->redis_password}@{$this->uuid}:6379/0";
}
}
public function environment()
{

20
resources/views/livewire/project/database/backup-edit.blade.php
View File

@ -25,9 +25,21 @@
</x-forms.select>
</div>
@endif
<div class="flex gap-2">
<x-forms.input label="Databases To Backup" helper="Comma separated list of databases to backup. Empty will include the default one." id="backup.databases_to_backup" />
<x-forms.input label="Frequency" id="backup.frequency" />
<x-forms.input label="Number of backups to keep (locally)" id="backup.number_of_backups_locally" />
<div class="flex flex-col gap-2">
<div class="flex gap-2">
@if ($backup->database_type === 'App\Models\StandalonePostgres')
<x-forms.input label="Databases To Backup"
helper="Comma separated list of databases to backup. Empty will include the default one."
id="backup.databases_to_backup" />
@elseif($backup->database_type === 'App\Models\StandaloneMongodb')
<x-forms.input label="Databases To Include"
helper="A list of databases to backup. You can specify which collection(s) per database to exclude from the backup. Empty will include all databases and collections.<br><br>Example:<br><br>database1:collection1,collection2|database2:collection3,collection4<br><br> database1 will include all collections except collection1 and collection2. <br>database2 will include all collections except collection3 and collection4.<br><br>Another Example:<br><br>database1:collection1|database2<br><br> database1 will include all collections except collection1.<br>database2 will include ALL collections."
id="backup.databases_to_backup" />
@endif
</div>
<div class="flex gap-2">
<x-forms.input label="Frequency" id="backup.frequency" />
<x-forms.input label="Number of backups to keep (locally)" id="backup.number_of_backups_locally" />
</div>
</div>
</form>