Shiloh/lasthourcloud
Shiloh/lasthourcloud
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 4 Wiki Activity

feat: new container status checks

Browse Source
This commit is contained in:
Andras Bacsai 2023-09-14 12:45:50 +02:00
parent 3c8c8e20b1
commit 53c20e1e99
18 changed files with 369 additions and 189 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
app/Actions/Proxy/StartProxy.php
View File

@ -8,7 +8,7 @@
class StartProxy
{
public function __invoke(Server $server): Activity
public function __invoke(Server $server, bool $async = true): Activity|string
{
$proxy_path = get_proxy_path();
$networks = collect($server->standaloneDockers)->map(function ($docker) {
@ -26,8 +26,7 @@ public function __invoke(Server $server): Activity
$docker_compose_yml_base64 = base64_encode($configuration);
$server->proxy->last_applied_settings = Str::of($docker_compose_yml_base64)->pipe('md5')->value;
$server->save();
$activity = remote_process([
$commands = [
"echo '####### Creating required Docker networks...'",
...$create_networks_command,
"cd $proxy_path",
@ -44,8 +43,13 @@ public function __invoke(Server $server): Activity
"echo '####### Starting coolify-proxy...'",
'docker compose up -d --remove-orphans',
"echo '####### Proxy installed successfully...'"
], $server);
return $activity;
];
if (!$async) {
instant_remote_process($commands, $server);
return 'OK';
} else {
$activity = remote_process($commands, $server);
return $activity;
}
}
}

32
app/Console/Kernel.php
View File

@ -2,21 +2,15 @@
namespace App\Console;
use App\Enums\ProxyTypes;
use App\Jobs\ApplicationContainerStatusJob;
use App\Jobs\CheckResaleLicenseJob;
use App\Jobs\CleanupInstanceStuffsJob;
use App\Jobs\DatabaseBackupJob;
use App\Jobs\DatabaseContainerStatusJob;
use App\Jobs\DockerCleanupJob;
use App\Jobs\InstanceAutoUpdateJob;
use App\Jobs\ProxyContainerStatusJob;
use App\Jobs\ServerDetailsCheckJob;
use App\Models\Application;
use App\Jobs\ContainerStatusJob;
use App\Models\InstanceSettings;
use App\Models\ScheduledDatabaseBackup;
use App\Models\Server;
use App\Models\StandalonePostgresql;
use Illuminate\Console\Scheduling\Schedule;
use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
@ -25,15 +19,14 @@ class Kernel extends ConsoleKernel
protected function schedule(Schedule $schedule): void
{
if (isDev()) {
$schedule->job(new ServerDetailsCheckJob(Server::find(0)))->everyTenMinutes()->onOneServer();
// $schedule->job(new ContainerStatusJob(Server::find(0)))->everyTenMinutes()->onOneServer();
// $schedule->command('horizon:snapshot')->everyMinute();
// $schedule->job(new CleanupInstanceStuffsJob)->everyMinute();
// $schedule->job(new CheckResaleLicenseJob)->hourly();
// $schedule->job(new DockerCleanupJob)->everyOddHour();
// $this->instance_auto_update($schedule);
// $this->check_scheduled_backups($schedule);
// $this->check_resources($schedule);
// $this->check_proxies($schedule);
$this->check_resources($schedule);
} else {
$schedule->command('horizon:snapshot')->everyFiveMinutes();
$schedule->job(new CleanupInstanceStuffsJob)->everyTenMinutes()->onOneServer();
@ -42,26 +35,15 @@ protected function schedule(Schedule $schedule): void
$this->instance_auto_update($schedule);
$this->check_scheduled_backups($schedule);
$this->check_resources($schedule);
$this->check_proxies($schedule);
}
}
private function check_proxies($schedule)
{
$servers = Server::all()->where('settings.is_usable', true)->where('settings.is_reachable', true)->whereNotNull('proxy.type')->where('proxy.type', '!=', ProxyTypes::NONE->value);
foreach ($servers as $server) {
$schedule->job(new ProxyContainerStatusJob($server))->everyMinute()->onOneServer();
}
}
private function check_resources($schedule)
{
$applications = Application::all();
foreach ($applications as $application) {
$schedule->job(new ApplicationContainerStatusJob($application))->everyMinute()->onOneServer();
}
$servers = Server::all()->where('settings.is_usable', true)->where('settings.is_reachable', true);
ray($servers);
$postgresqls = StandalonePostgresql::all();
foreach ($postgresqls as $postgresql) {
$schedule->job(new DatabaseContainerStatusJob($postgresql))->everyMinute()->onOneServer();
foreach ($servers as $server) {
$schedule->job(new ContainerStatusJob($server))->everyMinute()->onOneServer();
}
}
private function instance_auto_update($schedule)

6
app/Http/Livewire/Project/Application/Heading.php
View File

@ -3,6 +3,7 @@
namespace App\Http\Livewire\Project\Application;
use App\Jobs\ApplicationContainerStatusJob;
use App\Jobs\ContainerStatusJob;
use App\Models\Application;
use App\Notifications\Application\StatusChanged;
use Livewire\Component;
@ -22,9 +23,8 @@ public function mount()
public function check_status()
{
dispatch_sync(new ApplicationContainerStatusJob(
application: $this->application,
));
ray($this->application->destination->server);
dispatch_sync(new ContainerStatusJob($this->application->destination->server));
$this->application->refresh();
}

7
app/Http/Livewire/Project/Database/Heading.php
View File

@ -3,8 +3,7 @@
namespace App\Http\Livewire\Project\Database;
use App\Actions\Database\StartPostgresql;
use App\Jobs\DatabaseContainerStatusJob;
use App\Notifications\Application\StatusChanged;
use App\Jobs\ContainerStatusJob;
use Livewire\Component;
class Heading extends Component
@ -25,9 +24,7 @@ public function activityFinished()
public function check_status()
{
dispatch_sync(new DatabaseContainerStatusJob(
database: $this->database,
));
dispatch_sync(new ContainerStatusJob($this->database->destination->server));
$this->database->refresh();
}

16
app/Http/Livewire/Server/Form.php
View File

@ -88,17 +88,11 @@ public function delete()
public function submit()
{
$this->validate();
// $validation = Validator::make($this->server->toArray(), [
// 'ip' => [
// 'ip'
// ],
// ]);
// if ($validation->fails()) {
// foreach ($validation->errors()->getMessages() as $key => $value) {
// $this->addError("server.{$key}", $value[0]);
// }
// return;
// }
$uniqueIPs = Server::all()->pluck('ip')->toArray();
if (in_array($this->server->ip, $uniqueIPs)) {
$this->emit('error', 'IP address is already in use by another team.');
return;
}
$this->server->settings->wildcard_domain = $this->wildcard_domain;
$this->server->settings->cleanup_after_percentage = $this->cleanup_after_percentage;
$this->server->settings->save();

4
app/Http/Livewire/Settings/Configuration.php
View File

@ -2,7 +2,7 @@
namespace App\Http\Livewire\Settings;
use App\Jobs\ProxyContainerStatusJob;
use App\Jobs\ContainerStatusJob;
use App\Models\InstanceSettings as ModelsInstanceSettings;
use App\Models\Server;
use Livewire\Component;
@ -124,7 +124,7 @@ private function setup_instance_fqdn()
];
}
$this->save_configuration_to_disk($traefik_dynamic_conf, $file);
dispatch(new ProxyContainerStatusJob($this->server));
dispatch(new ContainerStatusJob($this->server));
}
}

1
app/Jobs/ApplicationContainerStatusJob.php
View File

@ -4,7 +4,6 @@
use App\Models\Application;
use App\Models\ApplicationPreview;
use App\Notifications\Application\StatusChanged;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldBeUnique;

5
app/Jobs/ApplicationDeploymentJob.php
View File

@ -132,7 +132,9 @@ public function handle(): void
$this->deploy();
}
}
if ($this->application->fqdn) dispatch(new ProxyContainerStatusJob($this->server));
if ($this->server->isProxyShouldRun()) {
dispatch(new ContainerStatusJob($this->server));
}
$this->next(ApplicationDeploymentStatus::FINISHED->value);
} catch (Exception $e) {
ray($e);
@ -267,6 +269,7 @@ private function health_check()
"echo 'Rolling update completed.'"
],
);
$this->application->update(['status' => 'running']);
break;
}
$counter++;

139
app/Jobs/ContainerStatusJob.php Normal file
View File

@ -0,0 +1,139 @@
<?php
namespace App\Jobs;
use App\Actions\Proxy\StartProxy;
use App\Models\Server;
use App\Notifications\Container\ContainerRestarted;
use App\Notifications\Container\ContainerStopped;
use App\Notifications\Server\Unreachable;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
use Str;
class ContainerStatusJob implements ShouldQueue, ShouldBeUnique, ShouldBeEncrypted
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $tries = 1;
public $timeout = 120;
public function __construct(public Server $server)
{
}
public function middleware(): array
{
return [new WithoutOverlapping($this->server->uuid)];
}
public function uniqueId(): string
{
return $this->server->uuid;
}
private function checkServerConnection() {
ray("Checking server connection to {$this->server->ip}");
$uptime = instant_remote_process(['uptime'], $this->server, false);
if (!is_null($uptime)) {
ray('Server is up');
return true;
}
}
public function handle(): void
{
try {
ray()->clearAll();
$serverUptimeCheckNumber = 0;
$serverUptimeCheckNumberMax = 5;
while (true) {
if ($serverUptimeCheckNumber >= $serverUptimeCheckNumberMax) {
$this->server->settings()->update(['is_reachable' => false]);
$this->server->team->notify(new Unreachable($this->server));
return;
}
$result = $this->checkServerConnection();
if ($result) {
break;
}
$serverUptimeCheckNumber++;
sleep(5);
}
$containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this->server);
$containers = format_docker_command_output_to_json($containers);
$applications = $this->server->applications();
$databases = $this->server->databases();
if ($this->server->isProxyShouldRun()) {
$foundProxyContainer = $containers->filter(function ($value, $key) {
return data_get($value, 'Name') === '/coolify-proxy';
})->first();
if (!$foundProxyContainer) {
resolve(StartProxy::class)($this->server, false);
$this->server->team->notify(new ContainerRestarted('coolify-proxy', $this->server));
}
}
foreach ($applications as $application) {
$uuid = data_get($application, 'uuid');
$foundContainer = $containers->filter(function ($value, $key) use ($uuid) {
return Str::startsWith(data_get($value, 'Name'), "/$uuid");
})->first();
if ($foundContainer) {
$containerStatus = data_get($foundContainer, 'State.Status');
$databaseStatus = data_get($application, 'status');
if ($containerStatus !== $databaseStatus) {
$application->update(['status' => $containerStatus]);
}
} else {
$databaseStatus = data_get($application, 'status');
if ($databaseStatus !== 'exited') {
$application->update(['status' => 'exited']);
$name = data_get($application, 'name');
$fqdn = data_get($application, 'fqdn');
$containerName = $name ? "$name ($fqdn)" : $fqdn;
$project = data_get($application, 'environment.project');
$environment = data_get($application, 'environment');
$url = base_url() . '/project/' . $project->uuid . "/" . $environment->name . "/application/" . $application->uuid;
$this->server->team->notify(new ContainerStopped($containerName, $this->server, $url));
}
}
}
foreach ($databases as $database) {
$uuid = data_get($database, 'uuid');
$foundContainer = $containers->filter(function ($value, $key) use ($uuid) {
return Str::startsWith(data_get($value, 'Name'), "/$uuid");
})->first();
if ($foundContainer) {
$containerStatus = data_get($foundContainer, 'State.Status');
$databaseStatus = data_get($database, 'status');
if ($containerStatus !== $databaseStatus) {
$database->update(['status' => $containerStatus]);
}
} else {
$databaseStatus = data_get($database, 'status');
if ($databaseStatus !== 'exited') {
$database->update(['status' => 'exited']);
$name = data_get($database, 'name');
$containerName = $name;
$project = data_get($database, 'environment.project');
$environment = data_get($database, 'environment');
$url = base_url() . '/project/' . $project->uuid . "/" . $environment->name . "/database/" . $database->uuid;
$this->server->team->notify(new ContainerStopped($containerName, $this->server, $url));
}
}
}
// TODO Monitor other containers not managed by Coolify
} catch (\Throwable $e) {
send_internal_notification('ContainerStatusJob failed with: ' . $e->getMessage());
ray($e->getMessage());
throw $e;
}
}
}

80
app/Jobs/ServerDetailsCheckJob.php
View File

@ -1,80 +0,0 @@
<?php
namespace App\Jobs;
use App\Models\Server;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
use Str;
class ServerDetailsCheckJob implements ShouldQueue, ShouldBeUnique, ShouldBeEncrypted
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $tries = 1;
public $timeout = 120;
public function __construct(public Server $server)
{
}
public function middleware(): array
{
return [new WithoutOverlapping($this->server->uuid)];
}
public function uniqueId(): string
{
return $this->server->uuid;
}
public function handle(): void
{
try {
ray()->clearAll();
$containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this->server);
$containers = format_docker_command_output_to_json($containers);
$applications = $this->server->applications();
// ray($applications);
// ray(format_docker_command_output_to_json($containers));
foreach ($applications as $application) {
$uuid = data_get($application, 'uuid');
$foundContainer = $containers->filter(function ($value, $key) use ($uuid) {
$image = data_get($value, 'Config.Image');
return Str::startsWith($image, $uuid);
})->first();
if ($foundContainer) {
$containerStatus = data_get($foundContainer, 'State.Status');
$databaseStatus = data_get($application, 'status');
ray($containerStatus, $databaseStatus);
if ($containerStatus !== $databaseStatus) {
// $application->update(['status' => $containerStatus]);
}
}
}
// foreach ($containers as $container) {
// $labels = format_docker_labels_to_json(data_get($container,'Config.Labels'));
// $foundLabel = $labels->filter(fn ($value, $key) => Str::startsWith($key, 'coolify.applicationId'));
// if ($foundLabel->count() > 0) {
// $appFound = $applications->where('id', $foundLabel['coolify.applicationId'])->first();
// if ($appFound) {
// $containerStatus = data_get($container, 'State.Status');
// $databaseStatus = data_get($appFound, 'status');
// ray($containerStatus, $databaseStatus);
// }
// }
// }
} catch (\Throwable $e) {
// send_internal_notification('ServerDetailsCheckJob failed with: ' . $e->getMessage());
ray($e->getMessage());
throw $e;
}
}
}

2
app/Notifications/Application/DeploymentSuccess.php
View File

@ -53,7 +53,7 @@ public function toMail(): MailMessage
$pull_request_id = data_get($this->preview, 'pull_request_id', 0);
$fqdn = $this->fqdn;
if ($pull_request_id === 0) {
$mail->subject("✅New version is deployed of {$this->application_name}");
$mail->subject("✅ New version is deployed of {$this->application_name}");
} else {
$fqdn = $this->preview->fqdn;
$mail->subject("✅ Pull request #{$pull_request_id} of {$this->application_name} deployed successfully");

62
app/Notifications/Container/ContainerRestarted.php Normal file
View File

@ -0,0 +1,62 @@
<?php
namespace App\Notifications\Container;
use App\Models\Server;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Notifications\Messages\MailMessage;
use Illuminate\Notifications\Notification;
class ContainerRestarted extends Notification implements ShouldQueue
{
use Queueable;
public $tries = 5;
public function __construct(public string $name, public Server $server, public ?string $url = null)
{
}
public function via(object $notifiable): array
{
return setNotificationChannels($notifiable, 'status_changes');
}
public function toMail(): MailMessage
{
$mail = new MailMessage();
$mail->subject("✅ Container ({$this->name}) has been restarted automatically on {$this->server->name}");
$mail->view('emails.container-restarted', [
'containerName' => $this->name,
'serverName' => $this->server->name,
'url' => $this->url ,
]);
return $mail;
}
public function toDiscord(): string
{
$message = "✅ Container ({$this->name}) has been restarted automatically on {$this->server->name}";
return $message;
}
public function toTelegram(): array
{
$message = "✅ Container ({$this->name}) has been restarted automatically on {$this->server->name}";
$payload = [
"message" => $message,
];
if ($this->url) {
$payload['buttons'] = [
[
[
"text" => "Check Proxy in Coolify",
"url" => $this->url
]
]
];
};
return $payload;
}
}

61
app/Notifications/Container/ContainerStopped.php Normal file
View File

@ -0,0 +1,61 @@
<?php
namespace App\Notifications\Container;
use App\Models\Server;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Notifications\Messages\MailMessage;
use Illuminate\Notifications\Notification;
class ContainerStopped extends Notification implements ShouldQueue
{
use Queueable;
public $tries = 1;
public function __construct(public string $name, public Server $server, public ?string $url = null)
{
}
public function via(object $notifiable): array
{
return setNotificationChannels($notifiable, 'status_changes');
}
public function toMail(): MailMessage
{
$mail = new MailMessage();
$mail->subject("⛔ Container ({$this->name}) has been stopped on {$this->server->name}");
$mail->view('emails.container-stopped', [
'containerName' => $this->name,
'serverName' => $this->server->name,
'url' => $this->url,
]);
return $mail;
}
public function toDiscord(): string
{
$message = "⛔ Container ({$this->name}) has been stopped on {$this->server->name}";
return $message;
}
public function toTelegram(): array
{
$message = "⛔ Container ({$this->name}) has been stopped on {$this->server->name}";
$payload = [
"message" => $message,
];
if ($this->url) {
$payload['buttons'] = [
[
[
"text" => "Open Application in Coolify",
"url" => $this->url
]
]
];
}
return $payload;
}
}

53
app/Notifications/Server/NotReachable.php
View File

@ -1,53 +0,0 @@
<?php
namespace App\Notifications\Server;
use App\Models\Server;
use App\Notifications\Channels\DiscordChannel;
use App\Notifications\Channels\EmailChannel;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Notifications\Messages\MailMessage;
use Illuminate\Notifications\Notification;
use Illuminate\Support\Str;
class NotReachable extends Notification implements ShouldQueue
{
use Queueable;
public $tries = 5;
public function __construct(public Server $server)
{
}
public function via(object $notifiable): array
{
return setNotificationChannels($notifiable, 'status_changes');
}
public function toMail(): MailMessage
{
$mail = new MailMessage();
// $fqdn = $this->fqdn;
$mail->subject("⛔ Server '{$this->server->name}' is unreachable");
// $mail->view('emails.application-status-changes', [
// 'name' => $this->application_name,
// 'fqdn' => $fqdn,
// 'application_url' => $this->application_url,
// ]);
return $mail;
}
public function toDiscord(): string
{
$message = '⛔ Server \'' . $this->server->name . '\' is unreachable (could be a temporary issue). If you receive this more than twice in a row, please check your server.';
return $message;
}
public function toTelegram(): array
{
return [
"message" => '⛔ Server \'' . $this->server->name . '\' is unreachable (could be a temporary issue). If you receive this more than twice in a row, please check your server.'
];
}
}

47
app/Notifications/Server/Unreachable.php Normal file
View File

@ -0,0 +1,47 @@
<?php
namespace App\Notifications\Server;
use App\Models\Server;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Notifications\Messages\MailMessage;
use Illuminate\Notifications\Notification;
class Unreachable extends Notification implements ShouldQueue
{
use Queueable;
public $tries = 1;
public function __construct(public Server $server)
{
}
public function via(object $notifiable): array
{
return setNotificationChannels($notifiable, 'status_changes');
}
public function toMail(): MailMessage
{
$mail = new MailMessage();
$mail->subject("⛔ Server ({$this->server->name}) is unreachable after trying to connect to it 5 times");
$mail->view('emails.server-lost-connection', [
'name' => $this->server->name,
]);
return $mail;
}
public function toDiscord(): string
{
$message = "⛔ Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: You have to validate your server again after you fix the issue.";
return $message;
}
public function toTelegram(): array
{
return [
"message" => "⛔ Server '{$this->server->name}' is unreachable after trying to connect to it 5 times. All automations & integrations are turned off! Please check your server! IMPORTANT: You have to validate your server again after you fix the issue."
];
}
}

11
resources/views/emails/container-restarted.blade.php Normal file
View File

@ -0,0 +1,11 @@
<x-emails.layout>
Container ({{ $containerName }}) has been restarted automatically on {{$serverName}}, because it was stopped unexpected.
@if ($containerName === 'coolify-proxy')
Coolify Proxy should run on your server as you have FQDN set up in one of your resources. If you don't want to use Coolify Proxy, please remove FQDN from your resources.
Note: The proxy should not stop unexpectedly, so please check what is going on your server.
@endif
</x-emails.layout>

9
resources/views/emails/container-stopped.blade.php Normal file
View File

@ -0,0 +1,9 @@
<x-emails.layout>
Container {{ $containerName }} has been stopped unexpected on {{$serverName}}.
@if ($url)
Please check what is going on [here]({{ $url }}).
@endif
</x-emails.layout>

7
resources/views/emails/server-lost-connection.blade.php
View File

@ -1,5 +1,10 @@
<x-emails.layout>
Coolify Cloud cannot connect to your server ({{$name}}). Please check your server and make sure it is running.
Coolify cannot connect to your server ({{$name}}). Please check your server and make sure it is running.
All automations & integrations are turned off!
IMPORTANT: You have to validate your server again after you fix the issue.
If you have any questions, please contact us.