Shiloh/lasthourcloud
Shiloh/lasthourcloud
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 4 Wiki Activity

fix: better unreachable/revived server statuses

Browse Source
This commit is contained in:
Andras Bacsai 2023-10-07 00:51:01 +02:00
parent c58706e3e4
commit 9e81416fef
11 changed files with 174 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
app/Console/Commands/Cloud.php Normal file
View File

@ -0,0 +1,33 @@
<?php
namespace App\Console\Commands;
use App\Models\Server;
use Illuminate\Console\Command;
class Cloud extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'cloud:unused-servers';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Get Unused Servers from Cloud';
/**
* Execute the console command.
*/
public function handle()
{
Server::all()->whereNotNull('team.subscription')->where('team.subscription.stripe_trial_already_ended',true)->each(function($server){
$this->info($server->name);
});
}
}

6
app/Console/Kernel.php
View File

@ -48,7 +48,11 @@ private function cleanup_servers($schedule)
}
private function check_resources($schedule)
{
$servers = Server::all()->where('settings.is_usable', true)->where('settings.is_reachable', true);
if (isCloud()) {
$servers = Server::all()->whereNotNull('team.subscription')->where('team.subscription.stripe_trial_already_ended', false);
} else {
$servers = Server::all();
}
foreach ($servers as $server) {
$schedule->job(new ContainerStatusJob($server))->everyMinute()->onOneServer();
}

25
app/Jobs/ContainerStatusJob.php
View File

@ -7,6 +7,7 @@
use App\Models\Server;
use App\Notifications\Container\ContainerRestarted;
use App\Notifications\Container\ContainerStopped;
use App\Notifications\Server\Revived;
use App\Notifications\Server\Unreachable;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
@ -51,14 +52,20 @@ private function checkServerConnection()
public function handle(): void
{
try {
ray("checking server status for {$this->server->name}");
// ray()->clearAll();
$serverUptimeCheckNumber = 0;
$serverUptimeCheckNumberMax = 3;
while (true) {
if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) {
send_internal_notification('Server unreachable: ' . $this->server->name);
// $this->server->settings()->update(['is_reachable' => false]);
// $this->server->team->notify(new Unreachable($this->server));
if ($this->server->unreachable_email_sent === false) {
$this->server->team->notify(new Unreachable($this->server));
}
$this->server->settings()->update([
'is_reachable' => false,
]);
$this->server->update(['unreachable_email_sent' => true]);
return;
}
$result = $this->checkServerConnection();
@ -68,6 +75,20 @@ public function handle(): void
$serverUptimeCheckNumber++;
sleep(5);
}
if (data_get($this->server, 'unreachable_email_sent') === true) {
$this->server->team->notify(new Revived($this->server));
$this->server->update(['unreachable_email_sent' => false]);
}
if (
data_get($this->server, 'settings.is_reachable') === false ||
data_get($this->server, 'settings.is_usable') === false
) {
$this->server->settings()->update([
'is_reachable' => true,
'is_usable' => true
]);
}
$containers = instant_remote_process(["docker container ls -q"], $this->server);
if (!$containers) {
return;

6
app/Models/Server.php
View File

@ -8,6 +8,7 @@
use Illuminate\Database\Eloquent\Casts\Attribute;
use Spatie\SchemalessAttributes\Casts\SchemalessAttributes;
use Spatie\SchemalessAttributes\SchemalessAttributesTrait;
use Illuminate\Support\Str;
class Server extends BaseModel
{
@ -15,6 +16,11 @@ class Server extends BaseModel
protected static function booted()
{
static::saved(function ($server) {
$server->ip = Str::of($server->ip)->trim();
$server->user = Str::of($server->user)->trim();
});
static::created(function ($server) {
ServerSetting::create([
'server_id' => $server->id,

49
app/Notifications/Server/Revived.php Normal file
View File

@ -0,0 +1,49 @@
<?php
namespace App\Notifications\Server;
use App\Models\Server;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Notifications\Messages\MailMessage;
use Illuminate\Notifications\Notification;
class Revived extends Notification implements ShouldQueue
{
use Queueable;
public $tries = 1;
public function __construct(public Server $server)
{
if ($this->server->unreachable_email_sent === false) {
return;
}
}
public function via(object $notifiable): array
{
return setNotificationChannels($notifiable, 'status_changes');
}
public function toMail(): MailMessage
{
$mail = new MailMessage();
$mail->subject("✅ Server ({$this->server->name}) revived.");
$mail->view('emails.server-revived', [
'name' => $this->server->name,
]);
return $mail;
}
public function toDiscord(): string
{
$message = "✅ Server '{$this->server->name}' revived. All automations & integrations are turned on again!";
return $message;
}
public function toTelegram(): array
{
return [
"message" => "✅ Server '{$this->server->name}' revived. All automations & integrations are turned on again!"
];
}
}

4
app/Notifications/Server/Unreachable.php
View File

@ -35,13 +35,13 @@ public function toMail(): MailMessage
public function toDiscord(): string
{
$message = "⛔ Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: You have to validate your server again after you fix the issue.";
$message = "⛔ Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: We automatically try to revive your server. If your server is back online, we will automatically turn on all automations & integrations.";
return $message;
}
public function toTelegram(): array
{
return [
"message" => "⛔ Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: You have to validate your server again after you fix the issue."
"message" => "⛔ Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: We automatically try to revive your server. If your server is back online, we will automatically turn on all automations & integrations."
];
}
}

21
bootstrap/helpers/remoteProcess.php
View File

@ -7,6 +7,8 @@
use App\Models\ApplicationDeploymentQueue;
use App\Models\PrivateKey;
use App\Models\Server;
use App\Notifications\Server\Revived;
use App\Notifications\Server\Unreachable;
use Carbon\Carbon;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Support\Collection;
@ -85,7 +87,7 @@ function generateSshCommand(Server $server, string $command, bool $isMux = true)
if ($isMux && config('coolify.mux_enabled')) {
$ssh_command .= '-o ControlMaster=auto -o ControlPersist=1m -o ControlPath=/var/www/html/storage/app/ssh/mux/%h_%p_%r ';
}
if (data_get($server,'settings.is_cloudflare_tunnel')) {
if (data_get($server, 'settings.is_cloudflare_tunnel')) {
$ssh_command .= '-o ProxyCommand="/usr/local/bin/cloudflared access ssh --hostname %h" ';
}
$command = "PATH=\$PATH:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/host/usr/local/sbin:/host/usr/local/bin:/host/usr/sbin:/host/usr/bin:/host/sbin:/host/bin && $command";
@ -122,13 +124,14 @@ function instant_remote_process(Collection|array $command, Server $server, $thro
}
return $output;
}
function excludeCertainErrors(string $errorOutput, ?int $exitCode = null) {
function excludeCertainErrors(string $errorOutput, ?int $exitCode = null)
{
$ignoredErrors = collect([
'Permission denied (publickey',
'Could not resolve hostname',
]);
$ignored = false;
foreach ($ignoredErrors as $ignoredError) {
foreach ($ignoredErrors as $ignoredError) {
if (Str::contains($errorOutput, $ignoredError)) {
$ignored = true;
break;
@ -183,6 +186,9 @@ function validateServer(Server $server, bool $throwError = false)
$uptime = instant_remote_process(['uptime'], $server, $throwError);
if (!$uptime) {
$server->settings->is_reachable = false;
$server->team->notify(new Unreachable($server));
$server->unreachable_email_sent = true;
$server->save();
return [
"uptime" => null,
"dockerVersion" => null,
@ -203,6 +209,11 @@ function validateServer(Server $server, bool $throwError = false)
$server->settings->is_usable = false;
} else {
$server->settings->is_usable = true;
if (data_get($server, 'unreachable_email_sent') === true) {
$server->team->notify(new Revived($server));
$server->unreachable_email_sent = false;
$server->save();
}
}
return [
"uptime" => $uptime,
@ -213,7 +224,9 @@ function validateServer(Server $server, bool $throwError = false)
$server->settings->is_usable = false;
throw $e;
} finally {
if (data_get($server, 'settings')) $server->settings->save();
if (data_get($server, 'settings')) {
$server->settings->save();
}
}
}

1
bootstrap/helpers/shared.php
View File

@ -310,6 +310,7 @@ function send_internal_notification(string $message): void
$baseUrl = config('app.name');
$team = Team::find(0);
$team->notify(new GeneralNotification("👀 {$baseUrl}: " . $message));
ray("👀 {$baseUrl}: " . $message);
} catch (\Throwable $e) {
ray($e->getMessage());
}

31
database/migrations/2023_09_23_111819_add_server_emails.php Normal file
View File

@ -0,0 +1,31 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('servers', function (Blueprint $table) {
$table->boolean('unreachable_email_sent')->default(false);
$table->dropColumn('unreachable_count');
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('servers', function (Blueprint $table) {
$table->dropColumn('unreachable_email_sent');
$table->integer('unreachable_count')->default(0);
});
}
};

2
resources/views/emails/server-lost-connection.blade.php
View File

@ -4,7 +4,7 @@
All automations & integrations are turned off!
IMPORTANT: You have to validate your server again after you fix the issue.
IMPORTANT: We automatically try to revive your server. If your server is back online, we will automatically turn on all automations & integrations.
If you have any questions, please contact us.

6
resources/views/emails/server-revived.blade.php Normal file
View File

@ -0,0 +1,6 @@
<x-emails.layout>
Your server ({{$name}}) was offline for a while, but it is back online now. All automations & integrations are turned on again.
</x-emails.layout>