Shiloh/lasthourcloud
Shiloh/lasthourcloud
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 4 Wiki Activity

feat: upload large backups

Browse Source
This commit is contained in:
Andras Bacsai 2024-04-11 12:13:11 +02:00
parent f35b7ab6f4
commit 9c7f40e4fe
9 changed files with 287 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
app/Http/Controllers/UploadController.php Normal file
View File

@ -0,0 +1,83 @@
<?php
namespace App\Http\Controllers;
use Illuminate\Routing\Controller as BaseController;
use Illuminate\Http\JsonResponse;
use Pion\Laravel\ChunkUpload\Exceptions\UploadFailedException;
use Illuminate\Http\Request;
use Illuminate\Http\UploadedFile;
use Illuminate\Support\Facades\Storage;
use Pion\Laravel\ChunkUpload\Exceptions\UploadMissingFileException;
use Pion\Laravel\ChunkUpload\Handler\AbstractHandler;
use Pion\Laravel\ChunkUpload\Handler\HandlerFactory;
use Pion\Laravel\ChunkUpload\Receiver\FileReceiver;
class UploadController extends BaseController
{
public function upload(Request $request)
{
$resource = getResourceByUuid(request()->route('databaseUuid'), data_get(auth()->user()->currentTeam(), 'id'));
if (is_null($resource)) {
return response()->json(['error' => 'You do not have permission for this database'], 500);
}
$receiver = new FileReceiver("file", $request, HandlerFactory::classFromRequest($request));
if ($receiver->isUploaded() === false) {
throw new UploadMissingFileException();
}
$save = $receiver->receive();
if ($save->isFinished()) {
return $this->saveFile($save->getFile(), $resource);
}
$handler = $save->handler();
return response()->json([
"done" => $handler->getPercentageDone(),
'status' => true
]);
}
// protected function saveFileToS3($file)
// {
// $fileName = $this->createFilename($file);
// $disk = Storage::disk('s3');
// // It's better to use streaming Streaming (laravel 5.4+)
// $disk->putFileAs('photos', $file, $fileName);
// // for older laravel
// // $disk->put($fileName, file_get_contents($file), 'public');
// $mime = str_replace('/', '-', $file->getMimeType());
// // We need to delete the file when uploaded to s3
// unlink($file->getPathname());
// return response()->json([
// 'path' => $disk->url($fileName),
// 'name' => $fileName,
// 'mime_type' => $mime
// ]);
// }
protected function saveFile(UploadedFile $file, $resource)
{
$mime = str_replace('/', '-', $file->getMimeType());
$filePath = "upload/{$resource->uuid}";
$finalPath = storage_path("app/" . $filePath);
$file->move($finalPath, 'restore');
return response()->json([
'mime_type' => $mime
]);
}
protected function createFilename(UploadedFile $file)
{
$extension = $file->getClientOriginalExtension();
$filename = str_replace("." . $extension, "", $file->getClientOriginalName()); // Filename without extension
$filename .= "_" . md5(time()) . "." . $extension;
return $filename;
}
}

46
app/Livewire/Project/Database/Import.php
View File

@ -3,22 +3,24 @@
namespace App\Livewire\Project\Database;
use Livewire\Component;
use Livewire\WithFileUploads;
use App\Models\Server;
use Illuminate\Support\Facades\Storage;
class Import extends Component
{
use WithFileUploads;
public $file;
public $resource;
public $parameters;
public $containers;
public bool $validated = true;
public bool $scpInProgress = false;
public bool $importRunning = false;
public string $validationMsg = '';
public ?string $filename = null;
public ?string $filesize = null;
public bool $isUploading = false;
public int $progress = 0;
public bool $error = false;
public Server $server;
public string $container;
public array $importCommands = [];
@ -45,7 +47,7 @@ public function getContainers()
if (!data_get($this->parameters, 'database_uuid')) {
abort(404);
}
$resource = getResourceByUuid($this->parameters['database_uuid'], data_get(auth()->user()->currentTeam(),'id'));
$resource = getResourceByUuid($this->parameters['database_uuid'], data_get(auth()->user()->currentTeam(), 'id'));
if (is_null($resource)) {
abort(404);
}
@ -56,11 +58,6 @@ public function getContainers()
$this->containers->push($this->container);
}
if ($this->containers->count() > 1) {
$this->validated = false;
$this->validationMsg = 'The database service has more than one container running. Cannot import.';
}
if (
$this->resource->getMorphClass() == 'App\Models\StandaloneRedis' ||
$this->resource->getMorphClass() == 'App\Models\StandaloneKeydb' ||
@ -68,29 +65,27 @@ public function getContainers()
$this->resource->getMorphClass() == 'App\Models\StandaloneClickhouse' ||
$this->resource->getMorphClass() == 'App\Models\StandaloneMongodb'
) {
$this->validated = false;
$this->validationMsg = 'This database type is not currently supported.';
$this->dispatch('error', 'Import is not supported for this resource.');
}
}
public function runImport()
{
$this->validate([
'file' => 'required|file|max:102400'
]);
$this->importRunning = true;
$this->scpInProgress = true;
if ($this->filename == '') {
$this->dispatch('error', 'Please select a file to import.');
return;
}
try {
$uploadedFilename = $this->file->store('backup-import');
$uploadedFilename = "upload/{$this->resource->uuid}/restore";
$path = Storage::path($uploadedFilename);
if (!Storage::exists($uploadedFilename)) {
$this->dispatch('error', 'The file does not exist or has been deleted.');
return;
}
$tmpPath = '/tmp/' . basename($uploadedFilename);
// SCP the backup file to the server.
instant_scp($path, $tmpPath, $this->server);
$this->scpInProgress = false;
Storage::delete($uploadedFilename);
$this->importCommands[] = "docker cp {$tmpPath} {$this->container}:{$tmpPath}";
switch ($this->resource->getMorphClass()) {
@ -116,8 +111,7 @@ public function runImport()
$this->dispatch('activityMonitor', $activity->id);
}
} catch (\Throwable $e) {
$this->validated = false;
$this->validationMsg = $e->getMessage();
return handleError($e, $this);
}
}
}

1
composer.json
View File

@ -27,6 +27,7 @@
"lorisleiva/laravel-actions": "^2.7",
"nubs/random-name-generator": "^2.2",
"phpseclib/phpseclib": "~3.0",
"pion/laravel-chunk-upload": "^1.5",
"poliander/cron": "^3.0",
"purplepixie/phpdns": "^2.1",
"pusher/pusher-php-server": "^7.2",

68
composer.lock generated
View File

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "e095b8a9eb22df2943cbc3e9649ff9e8",
"content-hash": "e6fd1d5c5183226a78df717b52343393",
"packages": [
{
"name": "amphp/amp",
@ -6370,6 +6370,72 @@
},
"time": "2021-10-28T11:13:42+00:00"
},
{
"name": "pion/laravel-chunk-upload",
"version": "v1.5.4",
"source": {
"type": "git",
"url": "https://github.com/pionl/laravel-chunk-upload.git",
"reference": "cfbc4292ddcace51308a4f2f446d310aa04e6133"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/pionl/laravel-chunk-upload/zipball/cfbc4292ddcace51308a4f2f446d310aa04e6133",
"reference": "cfbc4292ddcace51308a4f2f446d310aa04e6133",
"shasum": ""
},
"require": {
"illuminate/console": "5.2 - 5.8 | ^6.0 | ^7.0 | ^8.0 | ^9.0 | ^10.0 | ^11.0",
"illuminate/filesystem": "5.2 - 5.8 | ^6.0 | ^7.0 | ^8.0 | ^9.0 | ^10.0 | ^11.0",
"illuminate/http": "5.2 - 5.8 | ^6.0 | ^7.0 | ^8.0 | ^9.0 | ^10.0 | ^11.0",
"illuminate/support": "5.2 - 5.8 | ^6.0 | ^7.0 | ^8.0 | ^9.0 | ^10.0 | ^11.0"
},
"require-dev": {
"friendsofphp/php-cs-fixer": "^2.16.0 | ^3.52.0",
"mockery/mockery": "^1.1.0 | ^1.3.0 | ^1.6.0",
"overtrue/phplint": "^1.1 | ^2.0 | ^9.1",
"phpunit/phpunit": "5.7 | 6.0 | 7.0 | 7.5 | 8.4 | ^8.5 | ^9.3 | ^10.0 | ^11.0"
},
"type": "library",
"extra": {
"laravel": {
"providers": [
"Pion\\Laravel\\ChunkUpload\\Providers\\ChunkUploadServiceProvider"
]
}
},
"autoload": {
"psr-4": {
"Pion\\Laravel\\ChunkUpload\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Martin Kluska",
"email": "martin@kluska.cz"
}
],
"description": "Service for chunked upload with several js providers",
"support": {
"issues": "https://github.com/pionl/laravel-chunk-upload/issues",
"source": "https://github.com/pionl/laravel-chunk-upload/tree/v1.5.4"
},
"funding": [
{
"url": "https://revolut.me/martinpv7n",
"type": "custom"
},
{
"url": "https://github.com/pionl",
"type": "github"
}
],
"time": "2024-03-25T15:50:07+00:00"
},
{
"name": "poliander/cron",
"version": "3.1.0",

44
config/chunk-upload.php Normal file
View File

@ -0,0 +1,44 @@
<?php
/**
* @see https://github.com/pionl/laravel-chunk-upload
*/
return [
/*
* The storage config
*/
'storage' => [
/*
* Returns the folder name of the chunks. The location is in storage/app/{folder_name}
*/
'chunks' => 'chunks',
'disk' => 'local',
],
'clear' => [
/*
* How old chunks we should delete
*/
'timestamp' => '-1 HOURS',
'schedule' => [
'enabled' => true,
'cron' => '25 * * * *', // run every hour on the 25th minute
],
],
'chunk' => [
// setup for the chunk naming setup to ensure same name upload at same time
'name' => [
'use' => [
'session' => true, // should the chunk name use the session id? The uploader must send cookie!,
'browser' => false, // instead of session we can use the ip and browser?
],
],
],
'handlers' => [
// A list of handlers/providers that will be appended to existing list of handlers
'custom' => [],
// Overrides the list of handlers - use only what you really want
'override' => [
// \Pion\Laravel\ChunkUpload\Handler\DropZoneUploadHandler::class
],
],
];

4
resources/css/app.css
View File

@ -289,3 +289,7 @@ .fullscreen {
.toast {
z-index: 1;
}
.dz-button {
@apply w-full p-4 py-10 my-4 font-bold bg-white border dark:border-coolgray-400 dark:text-white dark:bg-transparent hover:dark:bg-coolgray-400;
}

11
resources/views/livewire/project/database/backup-executions.blade.php
View File

@ -34,12 +34,13 @@ class="relative flex flex-col p-4 bg-white box-without-bg dark:bg-coolgray-100"
</x-modal-confirmation>
</div>
</form>
<script>
function download_file(executionId) {
window.open('/download/backup/' + executionId, '_blank');
}
</script>
@empty
<div>No executions found.</div>
@endforelse
<script>
function download_file(executionId) {
window.open('/download/backup/' + executionId, '_blank');
}
</script>
</div>

103
resources/views/livewire/project/database/import.blade.php
View File

@ -1,4 +1,41 @@
<div>
<div x-data="{ error: $wire.entangle('error'), filesize: $wire.entangle('filesize'), filename: $wire.entangle('filename'), isUploading: $wire.entangle('isUploading'), progress: $wire.entangle('progress') }">
<script src="https://unpkg.com/dropzone@5/dist/min/dropzone.min.js"></script>
@script
<script>
Dropzone.options.myDropzone = {
chunking: true,
method: "POST",
maxFilesize: 1000000000,
chunkSize: 10000000,
createImageThumbnails: false,
disablePreviews: true,
parallelChunkUploads: false,
init: function() {
let button = this.element.querySelector('button');
button.innerText = 'Select or drop files here...'
this.on('sending', function(file, xhr, formData) {
const token = document.querySelector('meta[name="csrf-token"]').getAttribute('content');
formData.append("_token", token);
});
this.on("addedfile", file => {
$wire.isUploading = true;
});
this.on('uploadprogress', function(file, progress, bytesSent) {
$wire.progress = progress;
});
this.on('complete', function(file) {
$wire.filename = file.name;
$wire.filesize = Number(file.size / 1024 / 1024).toFixed(2) + ' MB';
$wire.isUploading = false;
});
this.on('error', function(file, message) {
$wire.error = true;
$wire.$dispatch('error', message.error)
});
}
};
</script>
@endscript
<h2>Import Backup</h2>
<div class="mt-2 mb-4 rounded alert-error">
<svg xmlns="http://www.w3.org/2000/svg" class="w-6 h-6 stroke-current shrink-0" fill="none" viewBox="0 0 24 24">
@ -9,52 +46,32 @@
</div>
@if (str(data_get($resource, 'status'))->startsWith('running'))
@if (!$validated)
<div>{{ $validationMsg }}</div>
@else
<form disabled wire:submit.prevent="runImport" x-data="{ isFinished: false, isUploading: false, progress: 0 }">
@if ($resource->type() === 'standalone-postgresql')
<x-forms.input class="mb-2" label="Custom Import Command"
wire:model='postgresqlRestoreCommand'></x-forms.input>
@elseif ($resource->type() === 'standalone-mysql')
<x-forms.input class="mb-2" label="Custom Import Command"
wire:model='mysqlRestoreCommand'></x-forms.input>
@elseif ($resource->type() === 'standalone-mariadb')
<x-forms.input class="mb-2" label="Custom Import Command"
wire:model='mariadbRestoreCommand'></x-forms.input>
@endif
<div x-on:livewire-upload-start="isUploading = true; isFinished = false"
x-on:livewire-upload-finish="isUploading = false; isFinished = true"
x-on:livewire-upload-error="isUploading = false"
x-on:livewire-upload-progress="progress = $event.detail.progress">
<label class="block mb-2 text-sm font-medium text-gray-900 dark:text-white" for="file_input">Upload
file</label>
<input wire:model="file"
class="block w-full text-sm rounded cursor-pointer text-whiteborder bg-coolgray-100 border-coolgray-400 focus:outline-none"
aria-describedby="file_input_help" id="file_input" type="file">
<p class="mt-1 text-sm text-neutral-500" id="file_input_help">Max file size: 256MB
</p>
@error('file')
<span class="error">{{ $message }}</span>
@enderror
<div x-show="isUploading">
<progress max="100" x-bind:value="progress"
class="progress progress-warning"></progress>
</div>
</div>
<x-forms.button type="submit" class="w-full mt-4" x-show="isFinished">Import Backup</x-forms.button>
</form>
@if ($resource->type() === 'standalone-postgresql')
<x-forms.input class="mb-2" label="Custom Import Command"
wire:model='postgresqlRestoreCommand'></x-forms.input>
@elseif ($resource->type() === 'standalone-mysql')
<x-forms.input class="mb-2" label="Custom Import Command"
wire:model='mysqlRestoreCommand'></x-forms.input>
@elseif ($resource->type() === 'standalone-mariadb')
<x-forms.input class="mb-2" label="Custom Import Command"
wire:model='mariadbRestoreCommand'></x-forms.input>
@endif
@if ($scpInProgress)
<div>Database backup is being copied to server...</div>
@endif
<div x-show="isUploading">
<progress max="100" x-bind:value="progress" class="progress progress-warning"></progress>
</div>
<div x-show="filename && !error">
<div>File: <span x-text="filename ?? 'N/A'"></span> <span x-text="filesize">/ </span></div>
<x-forms.button class="w-full my-4" wire:click='runImport'>Restore Backup</x-forms.button>
</div>
<form action="/upload/backup/{{ $resource->uuid }}" class="dropzone" id="my-dropzone">
@csrf
</form>
<div class="container w-full pt-4 mx-auto">
<livewire:activity-monitor header="Database import output" />
<div class="container w-full mx-auto">
<livewire:activity-monitor header="Database restore output" />
</div>
@else
<div>Database must be running to import a backup.</div>
<div>Database must be running to restore a backup.</div>
@endif
</div>

2
routes/web.php
View File

@ -12,6 +12,7 @@
use App\Http\Controllers\Controller;
use App\Http\Controllers\MagicController;
use App\Http\Controllers\OauthController;
use App\Http\Controllers\UploadController;
use App\Livewire\Admin\Index as AdminIndex;
use App\Livewire\Dev\Compose as Compose;
@ -225,6 +226,7 @@
});
Route::middleware(['auth'])->group(function () {
Route::post('/upload/backup/{databaseUuid}', [UploadController::class, 'upload'])->name('upload.backup');
Route::get('/download/backup/{executionId}', function () {
try {
$team = auth()->user()->currentTeam();