feat: delay container/server jobs

This commit is contained in:
Andras Bacsai 2024-02-23 21:51:43 +01:00
parent b8b76dfa40
commit 61dbc81765
2 changed files with 28 additions and 5 deletions

View File

@ -8,15 +8,16 @@
class CleanupUnreachableServers extends Command
{
protected $signature = 'cleanup:unreachable-servers';
protected $description = 'Cleanup Unreachable Servers (3 days)';
protected $description = 'Cleanup Unreachable Servers (7 days)';
public function handle()
{
echo "Running unreachable server cleanup...\n";
$servers = Server::where('unreachable_count', 3)->where('unreachable_notification_sent', true)->where('updated_at', '<', now()->subDays(3))->get();
$servers = Server::where('unreachable_count', 3)->where('unreachable_notification_sent', true)->where('updated_at', '<', now()->subDays(7))->get();
if ($servers->count() > 0) {
foreach ($servers as $server) {
echo "Cleanup unreachable server ($server->id) with name $server->name";
send_internal_notification("Server $server->name is unreachable for 7 days. Cleaning up...");
$server->update([
'ip' => '1.2.3.4'
]);

View File

@ -68,13 +68,35 @@ private function check_resources($schedule)
$containerServers = $servers->where('settings.is_swarm_worker', false)->where('settings.is_build_server', false);
}
foreach ($containerServers as $server) {
$schedule->job(new ContainerStatusJob($server))->everyMinute()->onOneServer();
// $schedule->job(new ContainerStatusJob($server))->everyMinute()->onOneServer();
$schedule
->call(function () use ($server) {
$randomSeconds = rand(1, 40);
$job = new ContainerStatusJob($server);
$job->delay($randomSeconds);
ray('dispatching container status job in ' . $randomSeconds . ' seconds');
dispatch($job);
})->name('container-status-' . $server->id)->everyMinute()->onOneServer();
if ($server->isLogDrainEnabled()) {
$schedule->job(new CheckLogDrainContainerJob($server))->everyMinute()->onOneServer();
$schedule
->call(function () use ($server) {
$randomSeconds = rand(1, 40);
$job = new CheckLogDrainContainerJob($server);
$job->delay($randomSeconds);
dispatch($job);
})->name('log-drain-container-check-' . $server->id)->everyMinute()->onOneServer();
// $schedule->job(new CheckLogDrainContainerJob($server))->everyMinute()->onOneServer();
}
}
foreach ($servers as $server) {
$schedule->job(new ServerStatusJob($server))->everyMinute()->onOneServer();
$schedule
->call(function () use ($server) {
$randomSeconds = rand(1, 40);
$job = new ServerStatusJob($server);
$job->delay($randomSeconds);
dispatch($job);
})->name('server-status-job-' . $server->id)->everyMinute()->onOneServer();
// $schedule->job(new ServerStatusJob($server))->everyMinute()->onOneServer();
}
}
private function instance_auto_update($schedule)