From 9c7f40e4fe83891c9d789d541933e36b10427c84 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 11 Apr 2024 12:13:11 +0200 Subject: [PATCH] feat: upload large backups --- app/Http/Controllers/UploadController.php | 83 ++++++++++++++ app/Livewire/Project/Database/Import.php | 46 ++++---- composer.json | 1 + composer.lock | 68 +++++++++++- config/chunk-upload.php | 44 ++++++++ resources/css/app.css | 4 + .../database/backup-executions.blade.php | 11 +- .../project/database/import.blade.php | 103 ++++++++++-------- routes/web.php | 2 + 9 files changed, 287 insertions(+), 75 deletions(-) create mode 100644 app/Http/Controllers/UploadController.php create mode 100644 config/chunk-upload.php diff --git a/app/Http/Controllers/UploadController.php b/app/Http/Controllers/UploadController.php new file mode 100644 index 000000000..e0a7d1b23 --- /dev/null +++ b/app/Http/Controllers/UploadController.php @@ -0,0 +1,83 @@ +route('databaseUuid'), data_get(auth()->user()->currentTeam(), 'id')); + if (is_null($resource)) { + return response()->json(['error' => 'You do not have permission for this database'], 500); + } + $receiver = new FileReceiver("file", $request, HandlerFactory::classFromRequest($request)); + + if ($receiver->isUploaded() === false) { + throw new UploadMissingFileException(); + } + + $save = $receiver->receive(); + + if ($save->isFinished()) { + return $this->saveFile($save->getFile(), $resource); + } + + $handler = $save->handler(); + return response()->json([ + "done" => $handler->getPercentageDone(), + 'status' => true + ]); + } + // protected function saveFileToS3($file) + // { + // $fileName = $this->createFilename($file); + + // $disk = Storage::disk('s3'); + // // It's better to use streaming Streaming (laravel 5.4+) + // $disk->putFileAs('photos', $file, $fileName); + + // // for older laravel + // // $disk->put($fileName, file_get_contents($file), 'public'); + // $mime = str_replace('/', '-', $file->getMimeType()); + + // // We need to delete the file when uploaded to s3 + // unlink($file->getPathname()); + + // return response()->json([ + // 'path' => $disk->url($fileName), + // 'name' => $fileName, + // 'mime_type' => $mime + // ]); + // } + protected function saveFile(UploadedFile $file, $resource) + { + $mime = str_replace('/', '-', $file->getMimeType()); + $filePath = "upload/{$resource->uuid}"; + $finalPath = storage_path("app/" . $filePath); + $file->move($finalPath, 'restore'); + + return response()->json([ + 'mime_type' => $mime + ]); + } + protected function createFilename(UploadedFile $file) + { + $extension = $file->getClientOriginalExtension(); + $filename = str_replace("." . $extension, "", $file->getClientOriginalName()); // Filename without extension + + $filename .= "_" . md5(time()) . "." . $extension; + + return $filename; + } +} diff --git a/app/Livewire/Project/Database/Import.php b/app/Livewire/Project/Database/Import.php index b27a567b0..4a2a9fff2 100644 --- a/app/Livewire/Project/Database/Import.php +++ b/app/Livewire/Project/Database/Import.php @@ -3,22 +3,24 @@ namespace App\Livewire\Project\Database; use Livewire\Component; -use Livewire\WithFileUploads; use App\Models\Server; use Illuminate\Support\Facades\Storage; class Import extends Component { - use WithFileUploads; - public $file; public $resource; public $parameters; public $containers; - public bool $validated = true; public bool $scpInProgress = false; public bool $importRunning = false; - public string $validationMsg = ''; + + public ?string $filename = null; + public ?string $filesize = null; + public bool $isUploading = false; + public int $progress = 0; + public bool $error = false; + public Server $server; public string $container; public array $importCommands = []; @@ -45,7 +47,7 @@ public function getContainers() if (!data_get($this->parameters, 'database_uuid')) { abort(404); } - $resource = getResourceByUuid($this->parameters['database_uuid'], data_get(auth()->user()->currentTeam(),'id')); + $resource = getResourceByUuid($this->parameters['database_uuid'], data_get(auth()->user()->currentTeam(), 'id')); if (is_null($resource)) { abort(404); } @@ -56,11 +58,6 @@ public function getContainers() $this->containers->push($this->container); } - if ($this->containers->count() > 1) { - $this->validated = false; - $this->validationMsg = 'The database service has more than one container running. Cannot import.'; - } - if ( $this->resource->getMorphClass() == 'App\Models\StandaloneRedis' || $this->resource->getMorphClass() == 'App\Models\StandaloneKeydb' || @@ -68,29 +65,27 @@ public function getContainers() $this->resource->getMorphClass() == 'App\Models\StandaloneClickhouse' || $this->resource->getMorphClass() == 'App\Models\StandaloneMongodb' ) { - $this->validated = false; - $this->validationMsg = 'This database type is not currently supported.'; + $this->dispatch('error', 'Import is not supported for this resource.'); } } public function runImport() { - $this->validate([ - 'file' => 'required|file|max:102400' - ]); - - $this->importRunning = true; - $this->scpInProgress = true; + if ($this->filename == '') { + $this->dispatch('error', 'Please select a file to import.'); + return; + } try { - $uploadedFilename = $this->file->store('backup-import'); + $uploadedFilename = "upload/{$this->resource->uuid}/restore"; $path = Storage::path($uploadedFilename); + if (!Storage::exists($uploadedFilename)) { + $this->dispatch('error', 'The file does not exist or has been deleted.'); + return; + } $tmpPath = '/tmp/' . basename($uploadedFilename); - - // SCP the backup file to the server. instant_scp($path, $tmpPath, $this->server); - $this->scpInProgress = false; - + Storage::delete($uploadedFilename); $this->importCommands[] = "docker cp {$tmpPath} {$this->container}:{$tmpPath}"; switch ($this->resource->getMorphClass()) { @@ -116,8 +111,7 @@ public function runImport() $this->dispatch('activityMonitor', $activity->id); } } catch (\Throwable $e) { - $this->validated = false; - $this->validationMsg = $e->getMessage(); + return handleError($e, $this); } } } diff --git a/composer.json b/composer.json index b57263534..cb98eba57 100644 --- a/composer.json +++ b/composer.json @@ -27,6 +27,7 @@ "lorisleiva/laravel-actions": "^2.7", "nubs/random-name-generator": "^2.2", "phpseclib/phpseclib": "~3.0", + "pion/laravel-chunk-upload": "^1.5", "poliander/cron": "^3.0", "purplepixie/phpdns": "^2.1", "pusher/pusher-php-server": "^7.2", diff --git a/composer.lock b/composer.lock index 91613ec5f..55cd48f95 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "e095b8a9eb22df2943cbc3e9649ff9e8", + "content-hash": "e6fd1d5c5183226a78df717b52343393", "packages": [ { "name": "amphp/amp", @@ -6370,6 +6370,72 @@ }, "time": "2021-10-28T11:13:42+00:00" }, + { + "name": "pion/laravel-chunk-upload", + "version": "v1.5.4", + "source": { + "type": "git", + "url": "https://github.com/pionl/laravel-chunk-upload.git", + "reference": "cfbc4292ddcace51308a4f2f446d310aa04e6133" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/pionl/laravel-chunk-upload/zipball/cfbc4292ddcace51308a4f2f446d310aa04e6133", + "reference": "cfbc4292ddcace51308a4f2f446d310aa04e6133", + "shasum": "" + }, + "require": { + "illuminate/console": "5.2 - 5.8 | ^6.0 | ^7.0 | ^8.0 | ^9.0 | ^10.0 | ^11.0", + "illuminate/filesystem": "5.2 - 5.8 | ^6.0 | ^7.0 | ^8.0 | ^9.0 | ^10.0 | ^11.0", + "illuminate/http": "5.2 - 5.8 | ^6.0 | ^7.0 | ^8.0 | ^9.0 | ^10.0 | ^11.0", + "illuminate/support": "5.2 - 5.8 | ^6.0 | ^7.0 | ^8.0 | ^9.0 | ^10.0 | ^11.0" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "^2.16.0 | ^3.52.0", + "mockery/mockery": "^1.1.0 | ^1.3.0 | ^1.6.0", + "overtrue/phplint": "^1.1 | ^2.0 | ^9.1", + "phpunit/phpunit": "5.7 | 6.0 | 7.0 | 7.5 | 8.4 | ^8.5 | ^9.3 | ^10.0 | ^11.0" + }, + "type": "library", + "extra": { + "laravel": { + "providers": [ + "Pion\\Laravel\\ChunkUpload\\Providers\\ChunkUploadServiceProvider" + ] + } + }, + "autoload": { + "psr-4": { + "Pion\\Laravel\\ChunkUpload\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Martin Kluska", + "email": "martin@kluska.cz" + } + ], + "description": "Service for chunked upload with several js providers", + "support": { + "issues": "https://github.com/pionl/laravel-chunk-upload/issues", + "source": "https://github.com/pionl/laravel-chunk-upload/tree/v1.5.4" + }, + "funding": [ + { + "url": "https://revolut.me/martinpv7n", + "type": "custom" + }, + { + "url": "https://github.com/pionl", + "type": "github" + } + ], + "time": "2024-03-25T15:50:07+00:00" + }, { "name": "poliander/cron", "version": "3.1.0", diff --git a/config/chunk-upload.php b/config/chunk-upload.php new file mode 100644 index 000000000..0294f1b86 --- /dev/null +++ b/config/chunk-upload.php @@ -0,0 +1,44 @@ + [ + /* + * Returns the folder name of the chunks. The location is in storage/app/{folder_name} + */ + 'chunks' => 'chunks', + 'disk' => 'local', + ], + 'clear' => [ + /* + * How old chunks we should delete + */ + 'timestamp' => '-1 HOURS', + 'schedule' => [ + 'enabled' => true, + 'cron' => '25 * * * *', // run every hour on the 25th minute + ], + ], + 'chunk' => [ + // setup for the chunk naming setup to ensure same name upload at same time + 'name' => [ + 'use' => [ + 'session' => true, // should the chunk name use the session id? The uploader must send cookie!, + 'browser' => false, // instead of session we can use the ip and browser? + ], + ], + ], + 'handlers' => [ + // A list of handlers/providers that will be appended to existing list of handlers + 'custom' => [], + // Overrides the list of handlers - use only what you really want + 'override' => [ + // \Pion\Laravel\ChunkUpload\Handler\DropZoneUploadHandler::class + ], + ], +]; diff --git a/resources/css/app.css b/resources/css/app.css index 371efd8ed..719836599 100644 --- a/resources/css/app.css +++ b/resources/css/app.css @@ -289,3 +289,7 @@ .fullscreen { .toast { z-index: 1; } + +.dz-button { + @apply w-full p-4 py-10 my-4 font-bold bg-white border dark:border-coolgray-400 dark:text-white dark:bg-transparent hover:dark:bg-coolgray-400; +} diff --git a/resources/views/livewire/project/database/backup-executions.blade.php b/resources/views/livewire/project/database/backup-executions.blade.php index a84b88e5e..ae9428923 100644 --- a/resources/views/livewire/project/database/backup-executions.blade.php +++ b/resources/views/livewire/project/database/backup-executions.blade.php @@ -34,12 +34,13 @@ class="relative flex flex-col p-4 bg-white box-without-bg dark:bg-coolgray-100" - + @empty
No executions found.
@endforelse + diff --git a/resources/views/livewire/project/database/import.blade.php b/resources/views/livewire/project/database/import.blade.php index 159a49ea7..ad0199563 100644 --- a/resources/views/livewire/project/database/import.blade.php +++ b/resources/views/livewire/project/database/import.blade.php @@ -1,4 +1,41 @@ -
+
+ + @script + + @endscript

Import Backup

@@ -9,52 +46,32 @@
@if (str(data_get($resource, 'status'))->startsWith('running')) - @if (!$validated) -
{{ $validationMsg }}
- @else -
- @if ($resource->type() === 'standalone-postgresql') - - @elseif ($resource->type() === 'standalone-mysql') - - @elseif ($resource->type() === 'standalone-mariadb') - - @endif -
- - -

Max file size: 256MB -

- - @error('file') - {{ $message }} - @enderror -
- -
-
- Import Backup -
+ @if ($resource->type() === 'standalone-postgresql') + + @elseif ($resource->type() === 'standalone-mysql') + + @elseif ($resource->type() === 'standalone-mariadb') + @endif - @if ($scpInProgress) -
Database backup is being copied to server...
- @endif +
+ +
+
+
File: /
+ Restore Backup +
+
+ @csrf +
-
- +
+
@else -
Database must be running to import a backup.
+
Database must be running to restore a backup.
@endif
diff --git a/routes/web.php b/routes/web.php index 8d0251fda..ce2c4f418 100644 --- a/routes/web.php +++ b/routes/web.php @@ -12,6 +12,7 @@ use App\Http\Controllers\Controller; use App\Http\Controllers\MagicController; use App\Http\Controllers\OauthController; +use App\Http\Controllers\UploadController; use App\Livewire\Admin\Index as AdminIndex; use App\Livewire\Dev\Compose as Compose; @@ -225,6 +226,7 @@ }); Route::middleware(['auth'])->group(function () { + Route::post('/upload/backup/{databaseUuid}', [UploadController::class, 'upload'])->name('upload.backup'); Route::get('/download/backup/{executionId}', function () { try { $team = auth()->user()->currentTeam();