feat: upload large backups

app/Http/Controllers/UploadController.php

@@ -0,0 +1,83 @@
+<?php
+
+namespace App\Http\Controllers;
+
+use Illuminate\Routing\Controller as BaseController;
+use Illuminate\Http\JsonResponse;
+use Pion\Laravel\ChunkUpload\Exceptions\UploadFailedException;
+use Illuminate\Http\Request;
+use Illuminate\Http\UploadedFile;
+use Illuminate\Support\Facades\Storage;
+use Pion\Laravel\ChunkUpload\Exceptions\UploadMissingFileException;
+use Pion\Laravel\ChunkUpload\Handler\AbstractHandler;
+use Pion\Laravel\ChunkUpload\Handler\HandlerFactory;
+use Pion\Laravel\ChunkUpload\Receiver\FileReceiver;
+
+class UploadController extends BaseController
+{
+    public function upload(Request $request)
+    {
+        $resource = getResourceByUuid(request()->route('databaseUuid'), data_get(auth()->user()->currentTeam(), 'id'));
+        if (is_null($resource)) {
+            return response()->json(['error' => 'You do not have permission for this database'], 500);
+        }
+        $receiver = new FileReceiver("file", $request, HandlerFactory::classFromRequest($request));
+
+        if ($receiver->isUploaded() === false) {
+            throw new UploadMissingFileException();
+        }
+
+        $save = $receiver->receive();
+
+        if ($save->isFinished()) {
+            return $this->saveFile($save->getFile(), $resource);
+        }
+
+        $handler = $save->handler();
+        return response()->json([
+            "done" => $handler->getPercentageDone(),
+            'status' => true
+        ]);
+    }
+    // protected function saveFileToS3($file)
+    // {
+    //     $fileName = $this->createFilename($file);
+
+    //     $disk = Storage::disk('s3');
+    //     // It's better to use streaming Streaming (laravel 5.4+)
+    //     $disk->putFileAs('photos', $file, $fileName);
+
+    //     // for older laravel
+    //     // $disk->put($fileName, file_get_contents($file), 'public');
+    //     $mime = str_replace('/', '-', $file->getMimeType());
+
+    //     // We need to delete the file when uploaded to s3
+    //     unlink($file->getPathname());
+
+    //     return response()->json([
+    //         'path' => $disk->url($fileName),
+    //         'name' => $fileName,
+    //         'mime_type' => $mime
+    //     ]);
+    // }
+    protected function saveFile(UploadedFile $file, $resource)
+    {
+        $mime = str_replace('/', '-', $file->getMimeType());
+        $filePath = "upload/{$resource->uuid}";
+        $finalPath = storage_path("app/" . $filePath);
+        $file->move($finalPath, 'restore');
+
+        return response()->json([
+            'mime_type' => $mime
+        ]);
+    }
+    protected function createFilename(UploadedFile $file)
+    {
+        $extension = $file->getClientOriginalExtension();
+        $filename = str_replace("." . $extension, "", $file->getClientOriginalName()); // Filename without extension
+
+        $filename .= "_" . md5(time()) . "." . $extension;
+
+        return $filename;
+    }
+}

app/Livewire/Project/Database/Import.php

@@ -3,22 +3,24 @@
 namespace App\Livewire\Project\Database;
 
 use Livewire\Component;
-use Livewire\WithFileUploads;
 use App\Models\Server;
 use Illuminate\Support\Facades\Storage;
 
 class Import extends Component
 {
-    use WithFileUploads;
 
-    public $file;
     public $resource;
     public $parameters;
     public $containers;
-    public bool $validated = true;
     public bool $scpInProgress = false;
     public bool $importRunning = false;
-    public string $validationMsg = '';
+
+    public ?string $filename = null;
+    public ?string $filesize = null;
+    public bool $isUploading = false;
+    public int $progress = 0;
+    public bool $error = false;
+
     public Server $server;
     public string $container;
     public array $importCommands = [];
@@ -56,11 +58,6 @@ public function getContainers()
             $this->containers->push($this->container);
         }
 
-        if ($this->containers->count() > 1) {
-            $this->validated = false;
-            $this->validationMsg = 'The database service has more than one container running. Cannot import.';
-        }
-
         if (
             $this->resource->getMorphClass() == 'App\Models\StandaloneRedis' ||
             $this->resource->getMorphClass() == 'App\Models\StandaloneKeydb' ||
@@ -68,29 +65,27 @@ public function getContainers()
             $this->resource->getMorphClass() == 'App\Models\StandaloneClickhouse' ||
             $this->resource->getMorphClass() == 'App\Models\StandaloneMongodb'
         ) {
-            $this->validated = false;
+            $this->dispatch('error', 'Import is not supported for this resource.');
-            $this->validationMsg = 'This database type is not currently supported.';
         }
     }
 
     public function runImport()
     {
-        $this->validate([
-            'file' => 'required|file|max:102400'
-        ]);
-
-        $this->importRunning = true;
-        $this->scpInProgress = true;
 
+        if ($this->filename == '') {
+            $this->dispatch('error', 'Please select a file to import.');
+            return;
+        }
         try {
-            $uploadedFilename = $this->file->store('backup-import');
+            $uploadedFilename = "upload/{$this->resource->uuid}/restore";
             $path = Storage::path($uploadedFilename);
+            if (!Storage::exists($uploadedFilename)) {
+                $this->dispatch('error', 'The file does not exist or has been deleted.');
+                return;
+            }
             $tmpPath = '/tmp/' . basename($uploadedFilename);
-
-            // SCP the backup file to the server.
             instant_scp($path, $tmpPath, $this->server);
-            $this->scpInProgress = false;
+            Storage::delete($uploadedFilename);
-
             $this->importCommands[] = "docker cp {$tmpPath} {$this->container}:{$tmpPath}";
 
             switch ($this->resource->getMorphClass()) {
@@ -116,8 +111,7 @@ public function runImport()
                 $this->dispatch('activityMonitor', $activity->id);
             }
         } catch (\Throwable $e) {
-            $this->validated = false;
+            return handleError($e, $this);
-            $this->validationMsg = $e->getMessage();
         }
     }
 }

composer.json

@@ -27,6 +27,7 @@
         "lorisleiva/laravel-actions": "^2.7",
         "nubs/random-name-generator": "^2.2",
         "phpseclib/phpseclib": "~3.0",
+        "pion/laravel-chunk-upload": "^1.5",
         "poliander/cron": "^3.0",
         "purplepixie/phpdns": "^2.1",
         "pusher/pusher-php-server": "^7.2",

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "e095b8a9eb22df2943cbc3e9649ff9e8",
+    "content-hash": "e6fd1d5c5183226a78df717b52343393",
     "packages": [
         {
             "name": "amphp/amp",
@@ -6370,6 +6370,72 @@
             },
             "time": "2021-10-28T11:13:42+00:00"
         },
+        {
+            "name": "pion/laravel-chunk-upload",
+            "version": "v1.5.4",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/pionl/laravel-chunk-upload.git",
+                "reference": "cfbc4292ddcace51308a4f2f446d310aa04e6133"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/pionl/laravel-chunk-upload/zipball/cfbc4292ddcace51308a4f2f446d310aa04e6133",
+                "reference": "cfbc4292ddcace51308a4f2f446d310aa04e6133",
+                "shasum": ""
+            },
+            "require": {
+                "illuminate/console": "5.2 - 5.8 | ^6.0 | ^7.0 | ^8.0 | ^9.0 | ^10.0 | ^11.0",
+                "illuminate/filesystem": "5.2 - 5.8 | ^6.0 | ^7.0 | ^8.0 | ^9.0 | ^10.0 | ^11.0",
+                "illuminate/http": "5.2 - 5.8 | ^6.0 | ^7.0 | ^8.0 | ^9.0 | ^10.0 | ^11.0",
+                "illuminate/support": "5.2 - 5.8 | ^6.0 | ^7.0 | ^8.0 | ^9.0 | ^10.0 | ^11.0"
+            },
+            "require-dev": {
+                "friendsofphp/php-cs-fixer": "^2.16.0 | ^3.52.0",
+                "mockery/mockery": "^1.1.0 | ^1.3.0 | ^1.6.0",
+                "overtrue/phplint": "^1.1 | ^2.0 | ^9.1",
+                "phpunit/phpunit": "5.7 | 6.0 | 7.0 | 7.5 | 8.4 | ^8.5 | ^9.3 | ^10.0 | ^11.0"
+            },
+            "type": "library",
+            "extra": {
+                "laravel": {
+                    "providers": [
+                        "Pion\\Laravel\\ChunkUpload\\Providers\\ChunkUploadServiceProvider"
+                    ]
+                }
+            },
+            "autoload": {
+                "psr-4": {
+                    "Pion\\Laravel\\ChunkUpload\\": "src/"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Martin Kluska",
+                    "email": "martin@kluska.cz"
+                }
+            ],
+            "description": "Service for chunked upload with several js providers",
+            "support": {
+                "issues": "https://github.com/pionl/laravel-chunk-upload/issues",
+                "source": "https://github.com/pionl/laravel-chunk-upload/tree/v1.5.4"
+            },
+            "funding": [
+                {
+                    "url": "https://revolut.me/martinpv7n",
+                    "type": "custom"
+                },
+                {
+                    "url": "https://github.com/pionl",
+                    "type": "github"
+                }
+            ],
+            "time": "2024-03-25T15:50:07+00:00"
+        },
         {
             "name": "poliander/cron",
             "version": "3.1.0",

config/chunk-upload.php

@@ -0,0 +1,44 @@
+<?php
+/**
+ * @see https://github.com/pionl/laravel-chunk-upload
+ */
+
+return [
+    /*
+     * The storage config
+     */
+    'storage' => [
+        /*
+         * Returns the folder name of the chunks. The location is in storage/app/{folder_name}
+         */
+        'chunks' => 'chunks',
+        'disk' => 'local',
+    ],
+    'clear' => [
+        /*
+         * How old chunks we should delete
+         */
+        'timestamp' => '-1 HOURS',
+        'schedule' => [
+            'enabled' => true,
+            'cron' => '25 * * * *', // run every hour on the 25th minute
+        ],
+    ],
+    'chunk' => [
+        // setup for the chunk naming setup to ensure same name upload at same time
+        'name' => [
+            'use' => [
+                'session' => true, // should the chunk name use the session id? The uploader must send cookie!,
+                'browser' => false, // instead of session we can use the ip and browser?
+            ],
+        ],
+    ],
+    'handlers' => [
+        // A list of handlers/providers that will be appended to existing list of handlers
+        'custom' => [],
+        // Overrides the list of handlers - use only what you really want
+        'override' => [
+            // \Pion\Laravel\ChunkUpload\Handler\DropZoneUploadHandler::class
+        ],
+    ],
+];

resources/css/app.css

@@ -289,3 +289,7 @@ .fullscreen {
 .toast {
     z-index: 1;
 }
+
+.dz-button {
+    @apply w-full p-4 py-10 my-4 font-bold bg-white border dark:border-coolgray-400 dark:text-white dark:bg-transparent hover:dark:bg-coolgray-400;
+}

resources/views/livewire/project/database/backup-executions.blade.php

@@ -34,12 +34,13 @@ class="relative flex flex-col p-4 bg-white box-without-bg dark:bg-coolgray-100"
                 </x-modal-confirmation>
             </div>
         </form>
+
+    @empty
+        <div>No executions found.</div>
+    @endforelse
     <script>
         function download_file(executionId) {
             window.open('/download/backup/' + executionId, '_blank');
         }
     </script>
-    @empty
-        <div>No executions found.</div>
-    @endforelse
 </div>

resources/views/livewire/project/database/import.blade.php

@@ -1,4 +1,41 @@
-<div>
+<div x-data="{ error: $wire.entangle('error'), filesize: $wire.entangle('filesize'), filename: $wire.entangle('filename'), isUploading: $wire.entangle('isUploading'), progress: $wire.entangle('progress') }">
+    <script src="https://unpkg.com/dropzone@5/dist/min/dropzone.min.js"></script>
+    @script
+        <script>
+            Dropzone.options.myDropzone = {
+                chunking: true,
+                method: "POST",
+                maxFilesize: 1000000000,
+                chunkSize: 10000000,
+                createImageThumbnails: false,
+                disablePreviews: true,
+                parallelChunkUploads: false,
+                init: function() {
+                    let button = this.element.querySelector('button');
+                    button.innerText = 'Select or drop files here...'
+                    this.on('sending', function(file, xhr, formData) {
+                        const token = document.querySelector('meta[name="csrf-token"]').getAttribute('content');
+                        formData.append("_token", token);
+                    });
+                    this.on("addedfile", file => {
+                        $wire.isUploading = true;
+                    });
+                    this.on('uploadprogress', function(file, progress, bytesSent) {
+                        $wire.progress = progress;
+                    });
+                    this.on('complete', function(file) {
+                        $wire.filename = file.name;
+                        $wire.filesize = Number(file.size / 1024 / 1024).toFixed(2) + ' MB';
+                        $wire.isUploading = false;
+                    });
+                    this.on('error', function(file, message) {
+                        $wire.error = true;
+                        $wire.$dispatch('error', message.error)
+                    });
+                }
+            };
+        </script>
+    @endscript
     <h2>Import Backup</h2>
     <div class="mt-2 mb-4 rounded alert-error">
         <svg xmlns="http://www.w3.org/2000/svg" class="w-6 h-6 stroke-current shrink-0" fill="none" viewBox="0 0 24 24">
@@ -9,10 +46,6 @@
     </div>
 
     @if (str(data_get($resource, 'status'))->startsWith('running'))
-        @if (!$validated)
-            <div>{{ $validationMsg }}</div>
-        @else
-            <form disabled wire:submit.prevent="runImport" x-data="{ isFinished: false, isUploading: false, progress: 0 }">
         @if ($resource->type() === 'standalone-postgresql')
             <x-forms.input class="mb-2" label="Custom Import Command"
                 wire:model='postgresqlRestoreCommand'></x-forms.input>
@@ -23,38 +56,22 @@
             <x-forms.input class="mb-2" label="Custom Import Command"
                 wire:model='mariadbRestoreCommand'></x-forms.input>
         @endif
-                <div x-on:livewire-upload-start="isUploading = true; isFinished = false"
-                    x-on:livewire-upload-finish="isUploading = false; isFinished = true"
-                    x-on:livewire-upload-error="isUploading = false"
-                    x-on:livewire-upload-progress="progress = $event.detail.progress">
-                    <label class="block mb-2 text-sm font-medium text-gray-900 dark:text-white" for="file_input">Upload
-                        file</label>
-                    <input wire:model="file"
-                        class="block w-full text-sm rounded cursor-pointer text-whiteborder bg-coolgray-100 border-coolgray-400 focus:outline-none"
-                        aria-describedby="file_input_help" id="file_input" type="file">
-                    <p class="mt-1 text-sm text-neutral-500" id="file_input_help">Max file size: 256MB
-                    </p>
 
-                    @error('file')
-                        <span class="error">{{ $message }}</span>
-                    @enderror
         <div x-show="isUploading">
-                        <progress max="100" x-bind:value="progress"
+            <progress max="100" x-bind:value="progress" class="progress progress-warning"></progress>
-                            class="progress progress-warning"></progress>
         </div>
+        <div x-show="filename && !error">
+            <div>File: <span x-text="filename ?? 'N/A'"></span> <span x-text="filesize">/ </span></div>
+            <x-forms.button class="w-full my-4" wire:click='runImport'>Restore Backup</x-forms.button>
         </div>
-                <x-forms.button type="submit" class="w-full mt-4" x-show="isFinished">Import Backup</x-forms.button>
+        <form action="/upload/backup/{{ $resource->uuid }}" class="dropzone" id="my-dropzone">
+            @csrf
         </form>
-        @endif
 
-        @if ($scpInProgress)
+        <div class="container w-full mx-auto">
-            <div>Database backup is being copied to server...</div>
+            <livewire:activity-monitor header="Database restore output" />
-        @endif
-
-        <div class="container w-full pt-4 mx-auto">
-            <livewire:activity-monitor header="Database import output" />
         </div>
     @else
-        <div>Database must be running to import a backup.</div>
+        <div>Database must be running to restore a backup.</div>
     @endif
 </div>

routes/web.php

@@ -12,6 +12,7 @@
 use App\Http\Controllers\Controller;
 use App\Http\Controllers\MagicController;
 use App\Http\Controllers\OauthController;
+use App\Http\Controllers\UploadController;
 use App\Livewire\Admin\Index as AdminIndex;
 use App\Livewire\Dev\Compose as Compose;
 
@@ -225,6 +226,7 @@
 });
 
 Route::middleware(['auth'])->group(function () {
+    Route::post('/upload/backup/{databaseUuid}', [UploadController::class, 'upload'])->name('upload.backup');
     Route::get('/download/backup/{executionId}', function () {
         try {
             $team = auth()->user()->currentTeam();