Shiloh/lasthourcloud
Shiloh/lasthourcloud
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 4 Wiki Activity

fix: server update schedule

Browse Source
This commit is contained in:
Andras Bacsai 2023-12-19 15:16:08 +01:00
parent 7952202435
commit 0126286731
3 changed files with 35 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
app/Jobs/ContainerStatusJob.php
View File

@ -17,33 +17,37 @@ use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping; use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels; use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Arr; use Illuminate\Support\Arr;
use Illuminate\Support\Sleep;
class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted
{ {
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $tries = 5;
public function backoff(): int
{
return isDev() ? 1 : 5;
}
public function middleware(): array public function middleware(): array
{ {
return [(new WithoutOverlapping($this->server->id))->dontRelease()]; return [(new WithoutOverlapping($this->server->uuid))];
} }
public function uniqueId(): int public function uniqueId(): int
{ {
return $this->server->id; return $this->server->uuid;
} }
public function __construct(public Server $server) public function __construct(public Server $server)
{ {
$this->handle(); // $this->handle();
} }
public function handle() public function handle()
{ {
try { if (!$this->server->isServerReady($this->tries)) {
if (!$this->server->isServerReady()) { throw new \RuntimeException('Server is not ready.');
return;
}; };
try {
if ($this->server->isSwarm()) { if ($this->server->isSwarm()) {
$containers = instant_remote_process(["docker service inspect $(docker service ls -q) --format '{{json .}}'"], $this->server, false); $containers = instant_remote_process(["docker service inspect $(docker service ls -q) --format '{{json .}}'"], $this->server, false);
$containerReplicates = instant_remote_process(["docker service ls --format '{{json .}}'"], $this->server, false); $containerReplicates = instant_remote_process(["docker service ls --format '{{json .}}'"], $this->server, false);

46
app/Models/Server.php
View File

@ -12,7 +12,6 @@ use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Support\Carbon; use Illuminate\Support\Carbon;
use Illuminate\Support\Collection; use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Process; use Illuminate\Support\Facades\Process;
use Illuminate\Support\Sleep;
use Spatie\SchemalessAttributes\Casts\SchemalessAttributes; use Spatie\SchemalessAttributes\Casts\SchemalessAttributes;
use Spatie\SchemalessAttributes\SchemalessAttributesTrait; use Spatie\SchemalessAttributes\SchemalessAttributesTrait;
use Illuminate\Support\Str; use Illuminate\Support\Str;
@ -150,30 +149,35 @@ class Server extends BaseModel
} }
return false; return false;
} }
public function isServerReady() public function isServerReady($tries)
{ {
$serverUptimeCheckNumber = $this->unreachable_count; $serverUptimeCheckNumber = $this->unreachable_count + 1;
$serverUptimeCheckNumberMax = 8; $serverUptimeCheckNumberMax = $tries ?? 3;
$currentTime = now()->timestamp; ray('server: ' . $this->name);
$runtime = 50;
$isReady = false;
// Run for 50 seconds max and check every 5 seconds for 8 times
while ($currentTime + $runtime > now()->timestamp) {
ray('serverUptimeCheckNumber: ' . $serverUptimeCheckNumber); ray('serverUptimeCheckNumber: ' . $serverUptimeCheckNumber);
ray('serverUptimeCheckNumberMax: ' . $serverUptimeCheckNumberMax);
$result = $this->validateConnection();
if ($result) {
if ($this->unreachable_notification_sent === true) {
$this->update(['unreachable_notification_sent' => false]);
}
return true;
} else {
if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) { if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) {
// Reached max number of retries
if ($this->unreachable_notification_sent === false) { if ($this->unreachable_notification_sent === false) {
ray('Server unreachable, sending notification...'); ray('Server unreachable, sending notification...');
$this->team?->notify(new Unreachable($this)); $this->team?->notify(new Unreachable($this));
$this->update(['unreachable_notification_sent' => true]); $this->update(['unreachable_notification_sent' => true]);
} }
if ($this->settings->is_reachable === true) {
$this->settings()->update([ $this->settings()->update([
'is_reachable' => false, 'is_reachable' => false,
]); ]);
$this->update([ }
'unreachable_count' => 0,
]);
foreach ($this->applications() as $application) { foreach ($this->applications() as $application) {
$application->update(['status' => 'exited']); $application->update(['status' => 'exited']);
} }
@ -190,23 +194,13 @@ class Server extends BaseModel
$db->update(['status' => 'exited']); $db->update(['status' => 'exited']);
} }
} }
$isReady = false; } else {
break;
}
$result = $this->validateConnection();
// ray('validateConnection: ' . $result);
if (!$result) {
$serverUptimeCheckNumber++;
$this->update([ $this->update([
'unreachable_count' => $serverUptimeCheckNumber, 'unreachable_count' => $this->unreachable_count + 1,
]); ]);
Sleep::for(5)->seconds();
continue;
} }
$isReady = true; return false;
break;
} }
return $isReady;
} }
public function getDiskUsage() public function getDiskUsage()
{ {