fix: server update schedule

This commit is contained in:
Andras Bacsai 2023-12-19 15:16:08 +01:00
parent 7952202435
commit 0126286731
3 changed files with 35 additions and 37 deletions

View File

@ -17,33 +17,37 @@ use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Arr;
use Illuminate\Support\Sleep;
class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $tries = 5;
public function backoff(): int
{
return isDev() ? 1 : 5;
}
public function middleware(): array
{
return [(new WithoutOverlapping($this->server->id))->dontRelease()];
return [(new WithoutOverlapping($this->server->uuid))];
}
public function uniqueId(): int
{
return $this->server->id;
return $this->server->uuid;
}
public function __construct(public Server $server)
{
$this->handle();
// $this->handle();
}
public function handle()
{
if (!$this->server->isServerReady($this->tries)) {
throw new \RuntimeException('Server is not ready.');
};
try {
if (!$this->server->isServerReady()) {
return;
};
if ($this->server->isSwarm()) {
$containers = instant_remote_process(["docker service inspect $(docker service ls -q) --format '{{json .}}'"], $this->server, false);
$containerReplicates = instant_remote_process(["docker service ls --format '{{json .}}'"], $this->server, false);

View File

@ -12,7 +12,6 @@ use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Support\Carbon;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Process;
use Illuminate\Support\Sleep;
use Spatie\SchemalessAttributes\Casts\SchemalessAttributes;
use Spatie\SchemalessAttributes\SchemalessAttributesTrait;
use Illuminate\Support\Str;
@ -150,30 +149,35 @@ class Server extends BaseModel
}
return false;
}
public function isServerReady()
public function isServerReady($tries)
{
$serverUptimeCheckNumber = $this->unreachable_count;
$serverUptimeCheckNumberMax = 8;
$serverUptimeCheckNumber = $this->unreachable_count + 1;
$serverUptimeCheckNumberMax = $tries ?? 3;
$currentTime = now()->timestamp;
$runtime = 50;
ray('server: ' . $this->name);
ray('serverUptimeCheckNumber: ' . $serverUptimeCheckNumber);
ray('serverUptimeCheckNumberMax: ' . $serverUptimeCheckNumberMax);
$isReady = false;
// Run for 50 seconds max and check every 5 seconds for 8 times
while ($currentTime + $runtime > now()->timestamp) {
ray('serverUptimeCheckNumber: ' . $serverUptimeCheckNumber);
$result = $this->validateConnection();
if ($result) {
if ($this->unreachable_notification_sent === true) {
$this->update(['unreachable_notification_sent' => false]);
}
return true;
} else {
if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) {
// Reached max number of retries
if ($this->unreachable_notification_sent === false) {
ray('Server unreachable, sending notification...');
$this->team?->notify(new Unreachable($this));
$this->update(['unreachable_notification_sent' => true]);
}
$this->settings()->update([
'is_reachable' => false,
]);
$this->update([
'unreachable_count' => 0,
]);
if ($this->settings->is_reachable === true) {
$this->settings()->update([
'is_reachable' => false,
]);
}
foreach ($this->applications() as $application) {
$application->update(['status' => 'exited']);
}
@ -190,23 +194,13 @@ class Server extends BaseModel
$db->update(['status' => 'exited']);
}
}
$isReady = false;
break;
}
$result = $this->validateConnection();
// ray('validateConnection: ' . $result);
if (!$result) {
$serverUptimeCheckNumber++;
} else {
$this->update([
'unreachable_count' => $serverUptimeCheckNumber,
'unreachable_count' => $this->unreachable_count + 1,
]);
Sleep::for(5)->seconds();
continue;
}
$isReady = true;
break;
return false;
}
return $isReady;
}
public function getDiskUsage()
{

View File

@ -54,7 +54,7 @@ class Revived extends Notification implements ShouldQueue
public function toDiscord(): string
{
$message = "Coolify: Server '{$this->server->name}' revived. All automations & integrations are turned on again!";
$message = "Coolify: Server '{$this->server->name}' revived. All automations & integrations are turned on again!";
return $message;
}
public function toTelegram(): array