fix: server update schedule
This commit is contained in:
parent
7952202435
commit
0126286731
@ -17,33 +17,37 @@ use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\Middleware\WithoutOverlapping;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Arr;
|
||||
use Illuminate\Support\Sleep;
|
||||
|
||||
class ContainerStatusJob implements ShouldQueue, ShouldBeEncrypted
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
public $tries = 5;
|
||||
public function backoff(): int
|
||||
{
|
||||
return isDev() ? 1 : 5;
|
||||
}
|
||||
public function middleware(): array
|
||||
{
|
||||
return [(new WithoutOverlapping($this->server->id))->dontRelease()];
|
||||
return [(new WithoutOverlapping($this->server->uuid))];
|
||||
}
|
||||
|
||||
public function uniqueId(): int
|
||||
{
|
||||
return $this->server->id;
|
||||
return $this->server->uuid;
|
||||
}
|
||||
|
||||
public function __construct(public Server $server)
|
||||
{
|
||||
$this->handle();
|
||||
// $this->handle();
|
||||
}
|
||||
|
||||
public function handle()
|
||||
{
|
||||
if (!$this->server->isServerReady($this->tries)) {
|
||||
throw new \RuntimeException('Server is not ready.');
|
||||
};
|
||||
try {
|
||||
if (!$this->server->isServerReady()) {
|
||||
return;
|
||||
};
|
||||
if ($this->server->isSwarm()) {
|
||||
$containers = instant_remote_process(["docker service inspect $(docker service ls -q) --format '{{json .}}'"], $this->server, false);
|
||||
$containerReplicates = instant_remote_process(["docker service ls --format '{{json .}}'"], $this->server, false);
|
||||
|
@ -12,7 +12,6 @@ use Illuminate\Database\Eloquent\Casts\Attribute;
|
||||
use Illuminate\Support\Carbon;
|
||||
use Illuminate\Support\Collection;
|
||||
use Illuminate\Support\Facades\Process;
|
||||
use Illuminate\Support\Sleep;
|
||||
use Spatie\SchemalessAttributes\Casts\SchemalessAttributes;
|
||||
use Spatie\SchemalessAttributes\SchemalessAttributesTrait;
|
||||
use Illuminate\Support\Str;
|
||||
@ -150,30 +149,35 @@ class Server extends BaseModel
|
||||
}
|
||||
return false;
|
||||
}
|
||||
public function isServerReady()
|
||||
public function isServerReady($tries)
|
||||
{
|
||||
$serverUptimeCheckNumber = $this->unreachable_count;
|
||||
$serverUptimeCheckNumberMax = 8;
|
||||
$serverUptimeCheckNumber = $this->unreachable_count + 1;
|
||||
$serverUptimeCheckNumberMax = $tries ?? 3;
|
||||
|
||||
$currentTime = now()->timestamp;
|
||||
$runtime = 50;
|
||||
ray('server: ' . $this->name);
|
||||
ray('serverUptimeCheckNumber: ' . $serverUptimeCheckNumber);
|
||||
ray('serverUptimeCheckNumberMax: ' . $serverUptimeCheckNumberMax);
|
||||
|
||||
$isReady = false;
|
||||
// Run for 50 seconds max and check every 5 seconds for 8 times
|
||||
while ($currentTime + $runtime > now()->timestamp) {
|
||||
ray('serverUptimeCheckNumber: ' . $serverUptimeCheckNumber);
|
||||
$result = $this->validateConnection();
|
||||
if ($result) {
|
||||
if ($this->unreachable_notification_sent === true) {
|
||||
$this->update(['unreachable_notification_sent' => false]);
|
||||
}
|
||||
return true;
|
||||
} else {
|
||||
if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) {
|
||||
// Reached max number of retries
|
||||
if ($this->unreachable_notification_sent === false) {
|
||||
ray('Server unreachable, sending notification...');
|
||||
$this->team?->notify(new Unreachable($this));
|
||||
$this->update(['unreachable_notification_sent' => true]);
|
||||
}
|
||||
$this->settings()->update([
|
||||
'is_reachable' => false,
|
||||
]);
|
||||
$this->update([
|
||||
'unreachable_count' => 0,
|
||||
]);
|
||||
if ($this->settings->is_reachable === true) {
|
||||
$this->settings()->update([
|
||||
'is_reachable' => false,
|
||||
]);
|
||||
}
|
||||
|
||||
foreach ($this->applications() as $application) {
|
||||
$application->update(['status' => 'exited']);
|
||||
}
|
||||
@ -190,23 +194,13 @@ class Server extends BaseModel
|
||||
$db->update(['status' => 'exited']);
|
||||
}
|
||||
}
|
||||
$isReady = false;
|
||||
break;
|
||||
}
|
||||
$result = $this->validateConnection();
|
||||
// ray('validateConnection: ' . $result);
|
||||
if (!$result) {
|
||||
$serverUptimeCheckNumber++;
|
||||
} else {
|
||||
$this->update([
|
||||
'unreachable_count' => $serverUptimeCheckNumber,
|
||||
'unreachable_count' => $this->unreachable_count + 1,
|
||||
]);
|
||||
Sleep::for(5)->seconds();
|
||||
continue;
|
||||
}
|
||||
$isReady = true;
|
||||
break;
|
||||
return false;
|
||||
}
|
||||
return $isReady;
|
||||
}
|
||||
public function getDiskUsage()
|
||||
{
|
||||
|
@ -54,7 +54,7 @@ class Revived extends Notification implements ShouldQueue
|
||||
|
||||
public function toDiscord(): string
|
||||
{
|
||||
$message = "Coolify: Server '{$this->server->name}' revived. All automations & integrations are turned on again!";
|
||||
$message = "Coolify: Server '{$this->server->name}' revived. All automations & integrations are turned on again!";
|
||||
return $message;
|
||||
}
|
||||
public function toTelegram(): array
|
||||
|
Loading…
x
Reference in New Issue
Block a user