2023-11-17 20:16:25 +00:00
|
|
|
<?php
|
|
|
|
|
|
|
|
namespace App\Jobs;
|
|
|
|
|
|
|
|
use App\Actions\Server\InstallLogDrain;
|
|
|
|
use App\Models\Server;
|
|
|
|
use App\Notifications\Container\ContainerRestarted;
|
|
|
|
use App\Notifications\Container\ContainerStopped;
|
|
|
|
use Illuminate\Bus\Queueable;
|
|
|
|
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
|
|
|
|
use Illuminate\Contracts\Queue\ShouldQueue;
|
|
|
|
use Illuminate\Foundation\Bus\Dispatchable;
|
|
|
|
use Illuminate\Queue\InteractsWithQueue;
|
|
|
|
use Illuminate\Queue\Middleware\WithoutOverlapping;
|
|
|
|
use Illuminate\Queue\SerializesModels;
|
|
|
|
use Illuminate\Support\Sleep;
|
|
|
|
|
|
|
|
class CheckLogDrainContainerJob implements ShouldQueue, ShouldBeEncrypted
|
|
|
|
{
|
|
|
|
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
|
|
|
|
|
|
|
public function __construct(public Server $server)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
public function middleware(): array
|
|
|
|
{
|
|
|
|
return [(new WithoutOverlapping($this->server->id))->dontRelease()];
|
|
|
|
}
|
|
|
|
|
|
|
|
public function uniqueId(): int
|
|
|
|
{
|
|
|
|
return $this->server->id;
|
|
|
|
}
|
|
|
|
public function healthcheck()
|
|
|
|
{
|
|
|
|
$status = instant_remote_process(["docker inspect --format='{{json .State.Status}}' coolify-log-drain"], $this->server, false);
|
|
|
|
if (str($status)->contains('running')) {
|
|
|
|
return true;
|
|
|
|
} else {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
public function handle(): void
|
|
|
|
{
|
|
|
|
// ray("checking log drain statuses for {$this->server->id}");
|
|
|
|
try {
|
2023-12-14 14:33:25 +00:00
|
|
|
if (!$this->server->isFunctional()) {
|
2023-11-17 20:16:25 +00:00
|
|
|
return;
|
|
|
|
};
|
2023-11-29 16:03:04 +00:00
|
|
|
$containers = instant_remote_process(["docker container ls -q"], $this->server, false);
|
2023-11-17 20:16:25 +00:00
|
|
|
if (!$containers) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
$containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this->server);
|
|
|
|
$containers = format_docker_command_output_to_json($containers);
|
|
|
|
|
|
|
|
$foundLogDrainContainer = $containers->filter(function ($value, $key) {
|
|
|
|
return data_get($value, 'Name') === '/coolify-log-drain';
|
|
|
|
})->first();
|
|
|
|
if (!$foundLogDrainContainer || !$this->healthcheck()) {
|
|
|
|
ray('Log drain container not found or unhealthy. Restarting...');
|
|
|
|
InstallLogDrain::run($this->server);
|
|
|
|
Sleep::for(10)->seconds();
|
|
|
|
if ($this->healthcheck()) {
|
|
|
|
if ($this->server->log_drain_notification_sent) {
|
2023-12-13 11:01:27 +00:00
|
|
|
$this->server->team?->notify(new ContainerRestarted('Coolify Log Drainer', $this->server));
|
2023-11-17 20:16:25 +00:00
|
|
|
$this->server->update(['log_drain_notification_sent' => false]);
|
|
|
|
}
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
if (!$this->server->log_drain_notification_sent) {
|
|
|
|
ray('Log drain container still unhealthy. Sending notification...');
|
2023-12-13 11:01:27 +00:00
|
|
|
$this->server->team?->notify(new ContainerStopped('Coolify Log Drainer', $this->server, null));
|
2023-11-17 20:16:25 +00:00
|
|
|
$this->server->update(['log_drain_notification_sent' => true]);
|
|
|
|
}
|
2023-11-17 20:24:22 +00:00
|
|
|
} else {
|
|
|
|
if ($this->server->log_drain_notification_sent) {
|
2023-12-13 11:01:27 +00:00
|
|
|
$this->server->team?->notify(new ContainerRestarted('Coolify Log Drainer', $this->server));
|
2023-11-17 20:24:22 +00:00
|
|
|
$this->server->update(['log_drain_notification_sent' => false]);
|
|
|
|
}
|
2023-11-17 20:16:25 +00:00
|
|
|
}
|
|
|
|
} catch (\Throwable $e) {
|
|
|
|
send_internal_notification("CheckLogDrainContainerJob failed on ({$this->server->id}) with: " . $e->getMessage());
|
|
|
|
ray($e->getMessage());
|
|
|
|
handleError($e);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|