Shiloh/lasthourcloud
Shiloh/lasthourcloud
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 4 Wiki Activity

feat: monitor server connection

Browse Source
This commit is contained in:
Andras Bacsai 2023-08-16 17:18:50 +02:00
parent fd74e07fc8
commit b34ab8a128
11 changed files with 184 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
app/Console/Kernel.php
View File

@ -3,13 +3,12 @@
namespace App\Console;
use App\Jobs\CheckResaleLicenseJob;
use App\Jobs\CheckResaleLicenseKeys;
use App\Jobs\CleanupInstanceStuffsJob;
use App\Jobs\DatabaseBackupJob;
use App\Jobs\DockerCleanupJob;
use App\Jobs\InstanceApplicationsStatusJob;
use App\Jobs\InstanceAutoUpdateJob;
use App\Jobs\ProxyCheckJob;
use App\Jobs\ResourceStatusJob;
use App\Models\ScheduledDatabaseBackup;
use Illuminate\Console\Scheduling\Schedule;
use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
@ -21,7 +20,7 @@ protected function schedule(Schedule $schedule): void
// $schedule->call(fn() => $this->check_scheduled_backups($schedule))->everyTenSeconds();
if (is_dev()) {
$schedule->command('horizon:snapshot')->everyMinute();
$schedule->job(new InstanceApplicationsStatusJob)->everyMinute();
$schedule->job(new ResourceStatusJob)->everyMinute();
$schedule->job(new ProxyCheckJob)->everyFiveMinutes();
$schedule->job(new CleanupInstanceStuffsJob)->everyMinute();
@ -31,7 +30,7 @@ protected function schedule(Schedule $schedule): void
} else {
$schedule->command('horizon:snapshot')->everyFiveMinutes();
$schedule->job(new CleanupInstanceStuffsJob)->everyMinute();
$schedule->job(new InstanceApplicationsStatusJob)->everyMinute();
$schedule->job(new ResourceStatusJob)->everyMinute();
$schedule->job(new CheckResaleLicenseJob)->hourly();
$schedule->job(new ProxyCheckJob)->everyFiveMinutes();
$schedule->job(new DockerCleanupJob)->everyTenMinutes();
@ -49,7 +48,10 @@ private function check_scheduled_backups($schedule)
return;
}
foreach ($scheduled_backups as $scheduled_backup) {
if (!$scheduled_backup->enabled) continue;
if (!$scheduled_backup->enabled) {
continue;
}
if (isset(VALID_CRON_STRINGS[$scheduled_backup->frequency])) {
$scheduled_backup->frequency = VALID_CRON_STRINGS[$scheduled_backup->frequency];
}

2
app/Http/Livewire/PrivateKey/Change.php
View File

@ -43,7 +43,7 @@ public function changePrivateKey()
$this->private_key->private_key .= "\n";
}
$this->private_key->save();
refreshPrivateKey($this->private_key);
refresh_server_connection($this->private_key);
} catch (\Exception $e) {
return general_error_handler(err: $e, that: $this);
}

2
app/Http/Livewire/Server/PrivateKey.php
View File

@ -17,7 +17,7 @@ public function setPrivateKey($private_key_id)
$this->server->update([
'private_key_id' => $private_key_id
]);
refreshPrivateKey($this->server->privateKey);
refresh_server_connection($this->server->privateKey);
$this->server->refresh();
$this->checkConnection();
}

2
app/Jobs/ContainerStatusJob.php
View File

@ -35,7 +35,7 @@ public function handle(): void
{
try {
$status = get_container_status(server: $this->resource->destination->server, container_id: $this->container_name, throwError: false);
if ($this->resource->status === 'running' && $status === 'stopped') {
if ($this->resource->status === 'running' && $status !== 'running') {
$this->resource->environment->project->team->notify(new StatusChanged($this->resource));
}

2
app/Jobs/InstanceApplicationsStatusJob.php → app/Jobs/ResourceStatusJob.php
View File

@ -10,7 +10,7 @@
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
class InstanceApplicationsStatusJob implements ShouldQueue, ShouldBeUnique
class ResourceStatusJob implements ShouldQueue, ShouldBeUnique
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;

58
app/Notifications/Server/NotReachable.php Normal file
View File

@ -0,0 +1,58 @@
<?php
namespace App\Notifications\Server;
use App\Models\Server;
use App\Notifications\Channels\DiscordChannel;
use App\Notifications\Channels\EmailChannel;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Notifications\Messages\MailMessage;
use Illuminate\Notifications\Notification;
use Illuminate\Support\Str;
class NotReachable extends Notification implements ShouldQueue
{
use Queueable;
public function __construct(public Server $server)
{
}
public function via(object $notifiable): array
{
$channels = [];
$isEmailEnabled = data_get($notifiable, 'smtp_enabled');
$isDiscordEnabled = data_get($notifiable, 'discord_enabled');
$isSubscribedToEmailEvent = data_get($notifiable, 'smtp_notifications_status_changes');
$isSubscribedToDiscordEvent = data_get($notifiable, 'discord_notifications_status_changes');
// if ($isEmailEnabled && $isSubscribedToEmailEvent) {
// $channels[] = EmailChannel::class;
// }
if ($isDiscordEnabled && $isSubscribedToDiscordEvent) {
$channels[] = DiscordChannel::class;
}
return $channels;
}
public function toMail(): MailMessage
{
$mail = new MailMessage();
// $fqdn = $this->fqdn;
$mail->subject("⛔ Server '{$this->server->name}' is unreachable");
// $mail->view('emails.application-status-changes', [
// 'name' => $this->application_name,
// 'fqdn' => $fqdn,
// 'application_url' => $this->application_url,
// ]);
return $mail;
}
public function toDiscord(): string
{
$message = '⛔ Server \'' . $this->server->name . '\' is unreachable (could be a temporary issue). If you receive this more than twice in a row, please check your server.';
return $message;
}
}

7
bootstrap/helpers/docker.php
View File

@ -8,8 +8,8 @@ function format_docker_command_output_to_json($rawOutput): Collection
$outputLines = explode(PHP_EOL, $rawOutput);
return collect($outputLines)
->reject(fn ($line) => empty($line))
->map(fn ($outputLine) => json_decode($outputLine, true, flags: JSON_THROW_ON_ERROR));
->reject(fn($line) => empty($line))
->map(fn($outputLine) => json_decode($outputLine, true, flags: JSON_THROW_ON_ERROR));
}
function format_docker_labels_to_json($rawOutput): Collection
@ -17,7 +17,7 @@ function format_docker_labels_to_json($rawOutput): Collection
$outputLines = explode(PHP_EOL, $rawOutput);
return collect($outputLines)
->reject(fn ($line) => empty($line))
->reject(fn($line) => empty($line))
->map(function ($outputLine) {
$outputArray = explode(',', $outputLine);
return collect($outputArray)
@ -45,6 +45,7 @@ function format_docker_envs_to_json($rawOutput)
function get_container_status(Server $server, string $container_id, bool $all_data = false, bool $throwError = false)
{
check_server_connection($server);
$container = instant_remote_process(["docker inspect --format '{{json .}}' {$container_id}"], $server, $throwError);
if (!$container) {
return 'exited';

36
bootstrap/helpers/remoteProcess.php
View File

@ -7,6 +7,7 @@
use App\Models\ApplicationDeploymentQueue;
use App\Models\PrivateKey;
use App\Models\Server;
use App\Notifications\Server\NotReachable;
use Carbon\Carbon;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Support\Collection;
@ -109,8 +110,8 @@ function instant_remote_process(array $command, Server $server, $throwError = tr
$exitCode = $process->exitCode();
if ($exitCode !== 0) {
if ($repeat > 1) {
ray("repeat: ", $repeat);
Sleep::for(200)->milliseconds();
ray('executing again');
return instant_remote_process($command, $server, $throwError, $repeat - 1);
}
// ray('ERROR OCCURED: ' . $process->errorOutput());
@ -152,21 +153,22 @@ function decode_remote_command_output(?ApplicationDeploymentQueue $application_d
return $formatted;
}
function refreshPrivateKey(PrivateKey $private_key)
function refresh_server_connection(PrivateKey $private_key)
{
foreach ($private_key->servers as $server) {
// Delete the old ssh mux file to force a new one to be created
Storage::disk('ssh-mux')->delete($server->muxFilename());
if (auth()->user()->currentTeam()->id) {
auth()->user()->currentTeam()->privateKeys = PrivateKey::where('team_id', auth()->user()->currentTeam()->id)->get();
}
// check if user is authenticated
if (auth()?->user()?->currentTeam()->id) {
auth()->user()->currentTeam()->privateKeys = PrivateKey::where('team_id', auth()->user()->currentTeam()->id)->get();
}
}
}
function validateServer(Server $server)
{
try {
refreshPrivateKey($server->privateKey);
refresh_server_connection($server->privateKey);
$uptime = instant_remote_process(['uptime'], $server);
if (!$uptime) {
$uptime = 'Server not reachable.';
@ -192,3 +194,25 @@ function validateServer(Server $server)
$server->settings->save();
}
}
function check_server_connection(Server $server) {
try {
refresh_server_connection($server->privateKey);
instant_remote_process(['uptime'], $server);
$server->unreachable_count = 0;
$server->settings->is_reachable = true;
} catch (\Exception $e) {
if ($server->unreachable_count == 2) {
$server->team->notify(new NotReachable($server));
$server->settings->is_reachable = false;
$server->settings->save();
} else {
$server->unreachable_count += 1;
}
throw $e;
} finally {
$server->settings->save();
$server->save();
}
}

28
database/migrations/2023_08_15_111126_update_servers_add_unreachable_count_table.php Normal file
View File

@ -0,0 +1,28 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('servers', function (Blueprint $table) {
$table->integer('unreachable_count')->default(0);
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('servers', function (Blueprint $table) {
$table->dropColumn('unreachable_count');
});
}
};

4
resources/views/components/server/navbar.blade.php
View File

@ -1,7 +1,9 @@
<div class="pb-6">
<div class="flex items-center gap-2">
<h1>Server</h1>
<livewire:server.proxy.status :server="$server" />
@if ($server->settings->is_reachable)
<livewire:server.proxy.status :server="$server" />
@endif
</div>
<div class="subtitle ">{{ data_get($server, 'name') }}</div>
<nav class="navbar-main">

97
resources/views/livewire/server/form.blade.php
View File

@ -10,42 +10,51 @@
running on.<br>Please think again.</p>
</x-slot:modalBody>
</x-modal>
@if ($server->settings->is_reachable)
<form wire:submit.prevent='submit' class="flex flex-col">
<div class="flex gap-2">
<h2>General</h2>
@if ($server->id === 0)
<x-forms.button isModal modalId="changeLocalhost">Save</x-forms.button>
@else
<x-forms.button type="submit">Save</x-forms.button>
@endif
</div>
<div class="flex flex-col gap-2 ">
<div class="flex flex-col w-full gap-2 lg:flex-row">
<x-forms.input id="server.name" label="Name" required />
<x-forms.input id="server.description" label="Description" />
<x-forms.input placeholder="https://example.com" id="wildcard_domain" label="Wildcard Domain"
helper="Wildcard domain for your applications. If you set this, you will get a random generated domain for your new applications.<br><span class='font-bold text-white'>Example</span>In case you set:<span class='text-helper'>https://example.com</span>your applications will get: <span class='text-helper'>https://randomId.example.com</span>" />
{{-- <x-forms.checkbox disabled type="checkbox" id="server.settings.is_part_of_swarm"
<form wire:submit.prevent='submit' class="flex flex-col">
<div class="flex gap-2">
<h2>General</h2>
@if ($server->id === 0)
<x-forms.button isModal modalId="changeLocalhost">Save</x-forms.button>
@else
<x-forms.button type="submit">Save</x-forms.button>
@endif
<x-forms.button wire:click.prevent='validateServer'>
Validate Server
</x-forms.button>
</div>
@if (!$server->settings->is_reachable)
You can't use this server until it is validated.
@else
Server validated.
@endif
<div class="flex flex-col gap-2 pt-4">
<div class="flex flex-col w-full gap-2 lg:flex-row">
<x-forms.input id="server.name" label="Name" required />
<x-forms.input id="server.description" label="Description" />
<x-forms.input placeholder="https://example.com" id="wildcard_domain" label="Wildcard Domain"
helper="Wildcard domain for your applications. If you set this, you will get a random generated domain for your new applications.<br><span class='font-bold text-white'>Example</span>In case you set:<span class='text-helper'>https://example.com</span>your applications will get: <span class='text-helper'>https://randomId.example.com</span>" />
{{-- <x-forms.checkbox disabled type="checkbox" id="server.settings.is_part_of_swarm"
label="Is it part of a Swarm cluster?" /> --}}
</div>
<div class="flex flex-col w-full gap-2 lg:flex-row">
@if ($server->id === 0)
<x-forms.input id="server.ip" label="IP Address" required />
@else
<x-forms.input id="server.ip" label="IP Address" readonly required />
@endif
<div class="flex gap-2">
<x-forms.input id="server.user" label="User" required />
<x-forms.input type="number" id="server.port" label="Port" required />
</div>
</div>
<div class="flex flex-col w-full gap-2 lg:flex-row">
@if ($server->id === 0)
<x-forms.input id="server.ip" label="IP Address" required />
@else
<x-forms.input id="server.ip" label="IP Address" readonly required />
@endif
<div class="flex gap-2">
<x-forms.input id="server.user" label="User" required />
<x-forms.input type="number" id="server.port" label="Port" required />
</div>
</div>
</div>
@if ($server->settings->is_reachable)
<h3 class="py-4">Settings</h3>
<div class="flex items-center w-64 gap-2">
<x-forms.input id="cleanup_after_percentage" label="Disk Cleanup threshold (%)" required
helper="Disk cleanup job will be executed if disk usage is more than this number." />
</div>
<h3 class="py-4">Actions</h3>
<div class="flex items-center gap-2">
<x-forms.button wire:click.prevent='validateServer'>
@ -61,26 +70,20 @@
</x-forms.button>
@endif
</div>
<div class="container w-full py-4 mx-auto">
<livewire:activity-monitor header="Logs" />
</div>
@isset($uptime)
<h3 class="pb-3">Server Info</h3>
<div class="py-2 pb-4">
<p>Uptime: {{ $uptime }}</p>
@isset($dockerVersion)
<p>Docker Engine {{ $dockerVersion }}</p>
@endisset
</div>
@endisset
</form>
@else
<div class="w-full pb-4">
<div class="cursor-pointer box" wire:click.prevent='validateServer'>
Validate Server
</div>
@endif
<div class="container w-full py-4 mx-auto">
<livewire:activity-monitor header="Logs" />
</div>
@endif
@isset($uptime)
<h3 class="pb-3">Server Info</h3>
<div class="py-2 pb-4">
<p>Uptime: {{ $uptime }}</p>
@isset($dockerVersion)
<p>Docker Engine {{ $dockerVersion }}</p>
@endisset
</div>
@endisset
</form>
<h2>Danger Zone</h2>
<div class="">Woah. I hope you know what are you doing.</div>
<h4 class="pt-4">Delete Server</h4>