diff --git a/app/Actions/Server/StartSentinel.php b/app/Actions/Server/StartSentinel.php index 342e72b1b..a2afea3bb 100644 --- a/app/Actions/Server/StartSentinel.php +++ b/app/Actions/Server/StartSentinel.php @@ -12,10 +12,13 @@ class StartSentinel public function handle(Server $server, $version = 'latest', bool $restart = false) { if ($restart) { - instant_remote_process(['docker rm -f coolify-sentinel'], $server, false); + StopSentinel::run($server); } + $metrics_history = $server->settings->metrics_history_days; + $refresh_rate = $server->settings->metrics_refresh_rate_seconds; + $token = $server->settings->metrics_token; instant_remote_process([ - "docker run --rm --pull always -d -e \"SCHEDULER=true\" -e \"METRICS_HISTORY=10\" -e \"REFRESH_RATE=5\" --name coolify-sentinel -v /var/run/docker.sock:/var/run/docker.sock -v /data/coolify/metrics:/app/metrics -v /data/coolify/logs:/app/logs --pid host --health-cmd \"curl --fail http://127.0.0.1:8888/api/health || exit 1\" --health-interval 10s --health-retries 3 ghcr.io/coollabsio/sentinel:$version", + "docker run --rm --pull always -d -e \"TOKEN={$token}\" -e \"SCHEDULER=true\" -e \"METRICS_HISTORY={$metrics_history}\" -e \"REFRESH_RATE={$refresh_rate}\" --name coolify-sentinel -v /var/run/docker.sock:/var/run/docker.sock -v /data/coolify/metrics:/app/metrics -v /data/coolify/logs:/app/logs --pid host --health-cmd \"curl --fail http://127.0.0.1:8888/api/health || exit 1\" --health-interval 10s --health-retries 3 ghcr.io/coollabsio/sentinel:$version", 'chown -R 9999:root /data/coolify/metrics /data/coolify/logs', 'chmod -R 700 /data/coolify/metrics /data/coolify/logs', ], $server, false); diff --git a/app/Actions/Server/StopSentinel.php b/app/Actions/Server/StopSentinel.php new file mode 100644 index 000000000..21ffca3bd --- /dev/null +++ b/app/Actions/Server/StopSentinel.php @@ -0,0 +1,16 @@ +all_servers->where('settings.is_usable', true)->where('settings.is_reachable', true)->where('ip', '!=', '1.2.3.4'); foreach ($servers as $server) { - if ($server->is_metrics_enabled) { + if ($server->isMetricsEnabled()) { $schedule->job(new PullSentinelImageJob($server))->everyFiveMinutes()->onOneServer(); } $schedule->job(new PullHelperImageJob($server))->everyFiveMinutes()->onOneServer(); diff --git a/app/Jobs/PullSentinelImageJob.php b/app/Jobs/PullSentinelImageJob.php index 232d00223..20fe19e99 100644 --- a/app/Jobs/PullSentinelImageJob.php +++ b/app/Jobs/PullSentinelImageJob.php @@ -36,10 +36,7 @@ public function handle(): void { try { $version = get_latest_sentinel_version(); - if (isDev()) { - $version = '0.0.5'; - } - if (! $version) { + if (!$version) { ray('Failed to get latest Sentinel version'); return; @@ -54,7 +51,7 @@ public function handle(): void } ray('Sentinel image is up to date'); } catch (\Throwable $e) { - send_internal_notification('PullSentinelImageJob failed with: '.$e->getMessage()); + send_internal_notification('PullSentinelImageJob failed with: ' . $e->getMessage()); ray($e->getMessage()); throw $e; } diff --git a/app/Jobs/ServerStatusJob.php b/app/Jobs/ServerStatusJob.php index 6dfe2792b..ce8864304 100644 --- a/app/Jobs/ServerStatusJob.php +++ b/app/Jobs/ServerStatusJob.php @@ -48,7 +48,7 @@ public function handle() if ($this->server->isFunctional()) { $this->cleanup(notify: false); $this->remove_unnecessary_coolify_yaml(); - if ($this->server->is_metrics_enabled) { + if ($this->server->isMetricsEnabled()) { $this->server->checkSentinel(); } } diff --git a/app/Livewire/Boarding/Index.php b/app/Livewire/Boarding/Index.php index b787ed0cc..7acf5ed87 100644 --- a/app/Livewire/Boarding/Index.php +++ b/app/Livewire/Boarding/Index.php @@ -12,7 +12,7 @@ class Index extends Component { - protected $listeners = ['serverInstalled' => 'validateServer']; + protected $listeners = ['refreshBoardingIndex' => 'validateServer']; public string $currentState = 'welcome'; diff --git a/app/Livewire/Charts/Server.php b/app/Livewire/Charts/Server.php deleted file mode 100644 index ebd6293c8..000000000 --- a/app/Livewire/Charts/Server.php +++ /dev/null @@ -1,38 +0,0 @@ -loadData(); - } - - public function loadData() - { - $metrics = $this->server->getMetrics(); - $metrics = collect($metrics)->map(function ($metric) { - return [$metric[0], $metric[1]]; - }); - $this->dispatch("refreshChartData-{$this->chartId}", [ - 'seriesData' => $metrics, - ]); - } -} diff --git a/app/Livewire/Charts/ServerCpu.php b/app/Livewire/Charts/ServerCpu.php new file mode 100644 index 000000000..fda2c5227 --- /dev/null +++ b/app/Livewire/Charts/ServerCpu.php @@ -0,0 +1,48 @@ +loadData(); + } + + public function loadData() + { + try { + $metrics = $this->server->getCpuMetrics($this->interval); + $metrics = collect($metrics)->map(function ($metric) { + return [$metric[0], $metric[1]]; + }); + $this->dispatch("refreshChartData-{$this->chartId}", [ + 'seriesData' => $metrics, + ]); + } catch (\Throwable $e) { + return handleError($e, $this); + } + } + public function setInterval() + { + $this->loadData(); + } +} diff --git a/app/Livewire/Charts/ServerMemory.php b/app/Livewire/Charts/ServerMemory.php new file mode 100644 index 000000000..366c87838 --- /dev/null +++ b/app/Livewire/Charts/ServerMemory.php @@ -0,0 +1,45 @@ +loadData(); + } + + public function loadData() + { + try { + $metrics = $this->server->getMemoryMetrics($this->interval); + $metrics = collect($metrics)->map(function ($metric) { + return [$metric[0], $metric[1]]; + }); + $this->dispatch("refreshChartData-{$this->chartId}", [ + 'seriesData' => $metrics, + ]); + } catch (\Throwable $e) { + return handleError($e, $this); + } + } + public function setInterval() { + $this->loadData(); + } +} diff --git a/app/Livewire/Project/Shared/Logs.php b/app/Livewire/Project/Shared/Logs.php index 008d743ed..e646f8a26 100644 --- a/app/Livewire/Project/Shared/Logs.php +++ b/app/Livewire/Project/Shared/Logs.php @@ -64,7 +64,7 @@ public function loadMetrics() return; $server = data_get($this->resource, 'destination.server'); if ($server->isFunctional()) { - $this->cpu = $server->getMetrics(); + $this->cpu = $server->getCpuMetrics(); } } diff --git a/app/Livewire/Server/ConfigureCloudflareTunnels.php b/app/Livewire/Server/ConfigureCloudflareTunnels.php index 7d2103e37..f7306a5b5 100644 --- a/app/Livewire/Server/ConfigureCloudflareTunnels.php +++ b/app/Livewire/Server/ConfigureCloudflareTunnels.php @@ -21,7 +21,7 @@ public function alreadyConfigured() $server->settings->is_cloudflare_tunnel = true; $server->settings->save(); $this->dispatch('success', 'Cloudflare Tunnels configured successfully.'); - $this->dispatch('serverInstalled'); + $this->dispatch('refreshServerShow'); } catch (\Throwable $e) { return handleError($e, $this); } @@ -37,7 +37,7 @@ public function submit() $server->save(); $server->settings->save(); $this->dispatch('success', 'Cloudflare Tunnels configured successfully.'); - $this->dispatch('serverInstalled'); + $this->dispatch('refreshServerShow'); } catch (\Throwable $e) { return handleError($e, $this); } diff --git a/app/Livewire/Server/Form.php b/app/Livewire/Server/Form.php index 263ff6367..7622a7b96 100644 --- a/app/Livewire/Server/Form.php +++ b/app/Livewire/Server/Form.php @@ -2,6 +2,9 @@ namespace App\Livewire\Server; +use App\Actions\Server\StartSentinel; +use App\Actions\Server\StopSentinel; +use App\Jobs\PullSentinelImageJob; use App\Models\Server; use Livewire\Component; @@ -36,6 +39,10 @@ class Form extends Component 'server.settings.is_build_server' => 'required|boolean', 'server.settings.concurrent_builds' => 'required|integer|min:1', 'server.settings.dynamic_timeout' => 'required|integer|min:1', + 'server.settings.is_metrics_enabled' => 'required|boolean', + 'server.settings.metrics_token' => 'required', + 'server.settings.metrics_refresh_rate_seconds' => 'required|integer|min:1', + 'server.settings.metrics_history_days' => 'required|integer|min:1', 'wildcard_domain' => 'nullable|url', ]; @@ -52,6 +59,10 @@ class Form extends Component 'server.settings.is_build_server' => 'Build Server', 'server.settings.concurrent_builds' => 'Concurrent Builds', 'server.settings.dynamic_timeout' => 'Dynamic Timeout', + 'server.settings.is_metrics_enabled' => 'Metrics', + 'server.settings.metrics_token' => 'Metrics Token', + 'server.settings.metrics_refresh_rate_seconds' => 'Metrics Interval', + 'server.settings.metrics_history_days' => 'Metrics History', ]; @@ -69,7 +80,7 @@ public function serverInstalled() public function updatedServerSettingsIsBuildServer() { - $this->dispatch('serverInstalled'); + $this->dispatch('refreshServerShow'); $this->dispatch('serverRefresh'); $this->dispatch('proxyStatusUpdated'); } @@ -80,7 +91,24 @@ public function instantSave() refresh_server_connection($this->server->privateKey); $this->validateServer(false); $this->server->settings->save(); + $this->server->save(); $this->dispatch('success', 'Server updated.'); + $this->dispatch('refreshServerShow'); + if ($this->server->isMetricsEnabled()) { + PullSentinelImageJob::dispatchSync($this->server); + $this->dispatch('reloadWindow'); + } else { + StopSentinel::dispatch($this->server); + } + } catch (\Throwable $e) { + return handleError($e, $this); + } + } + public function restartSentinel() { + try { + $version = get_latest_sentinel_version(); + StartSentinel::run($this->server, $version, true); + $this->dispatch('success', 'Sentinel restarted.'); } catch (\Throwable $e) { return handleError($e, $this); } diff --git a/app/Livewire/Server/Show.php b/app/Livewire/Server/Show.php index 7ebf90115..0751b186e 100644 --- a/app/Livewire/Server/Show.php +++ b/app/Livewire/Server/Show.php @@ -14,7 +14,7 @@ class Show extends Component public $parameters = []; - protected $listeners = ['serverInstalled' => '$refresh']; + protected $listeners = ['refreshServerShow' => '$refresh']; public function mount() { diff --git a/app/Livewire/Server/ValidateAndInstall.php b/app/Livewire/Server/ValidateAndInstall.php index bd33937e0..422cae779 100644 --- a/app/Livewire/Server/ValidateAndInstall.php +++ b/app/Livewire/Server/ValidateAndInstall.php @@ -143,7 +143,8 @@ public function validateDockerVersion() } else { $this->docker_version = $this->server->validateDockerEngineVersion(); if ($this->docker_version) { - $this->dispatch('serverInstalled'); + $this->dispatch('refreshServerShow'); + $this->dispatch('refreshBoardingIndex'); $this->dispatch('success', 'Server validated.'); } else { $this->error = 'Docker Engine version is not 22+. Please install Docker manually before continuing: documentation.'; diff --git a/app/Models/Server.php b/app/Models/Server.php index b1419dc0e..4337ce09e 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -131,7 +131,7 @@ public function addInitialNetwork() public function setupDefault404Redirect() { - $dynamic_conf_path = $this->proxyPath().'/dynamic'; + $dynamic_conf_path = $this->proxyPath() . '/dynamic'; $proxy_type = $this->proxyType(); $redirect_url = $this->proxy->redirect_url; if ($proxy_type === 'TRAEFIK_V2') { @@ -145,8 +145,8 @@ public function setupDefault404Redirect() respond 404 }'; $conf = - "# This file is automatically generated by Coolify.\n". - "# Do not edit it manually (only if you know what are you doing).\n\n". + "# This file is automatically generated by Coolify.\n" . + "# Do not edit it manually (only if you know what are you doing).\n\n" . $conf; $base64 = base64_encode($conf); instant_remote_process([ @@ -205,8 +205,8 @@ public function setupDefault404Redirect() ]; $conf = Yaml::dump($dynamic_conf, 12, 2); $conf = - "# This file is automatically generated by Coolify.\n". - "# Do not edit it manually (only if you know what are you doing).\n\n". + "# This file is automatically generated by Coolify.\n" . + "# Do not edit it manually (only if you know what are you doing).\n\n" . $conf; $base64 = base64_encode($conf); @@ -215,8 +215,8 @@ public function setupDefault404Redirect() redir $redirect_url }"; $conf = - "# This file is automatically generated by Coolify.\n". - "# Do not edit it manually (only if you know what are you doing).\n\n". + "# This file is automatically generated by Coolify.\n" . + "# Do not edit it manually (only if you know what are you doing).\n\n" . $conf; $base64 = base64_encode($conf); } @@ -237,7 +237,7 @@ public function setupDefault404Redirect() public function setupDynamicProxyConfiguration() { $settings = InstanceSettings::get(); - $dynamic_config_path = $this->proxyPath().'/dynamic'; + $dynamic_config_path = $this->proxyPath() . '/dynamic'; if ($this->proxyType() === 'TRAEFIK_V2') { $file = "$dynamic_config_path/coolify.yaml"; if (empty($settings->fqdn) || (isCloud() && $this->id !== 0)) { @@ -330,8 +330,8 @@ public function setupDynamicProxyConfiguration() } $yaml = Yaml::dump($traefik_dynamic_conf, 12, 2); $yaml = - "# This file is automatically generated by Coolify.\n". - "# Do not edit it manually (only if you know what are you doing).\n\n". + "# This file is automatically generated by Coolify.\n" . + "# Do not edit it manually (only if you know what are you doing).\n\n" . $yaml; $base64 = base64_encode($yaml); @@ -389,9 +389,9 @@ public function proxyPath() if ($proxyType === ProxyTypes::TRAEFIK_V2->value) { $proxy_path = $proxy_path; } elseif ($proxyType === ProxyTypes::CADDY->value) { - $proxy_path = $proxy_path.'/caddy'; + $proxy_path = $proxy_path . '/caddy'; } elseif ($proxyType === ProxyTypes::NGINX->value) { - $proxy_path = $proxy_path.'/nginx'; + $proxy_path = $proxy_path . '/nginx'; } return $proxy_path; @@ -462,10 +462,14 @@ public function forceDisableServer() Storage::disk('ssh-mux')->delete($this->muxFilename()); } + public function isMetricsEnabled() + { + return $this->settings->is_metrics_enabled; + } public function checkSentinel() { ray("Checking sentinel on server: {$this->name}"); - if ($this->is_metrics_enabled) { + if ($this->isMetricsEnabled()) { $sentinel_found = instant_remote_process(['docker inspect coolify-sentinel'], $this, false); $sentinel_found = json_decode($sentinel_found, true); $status = data_get($sentinel_found, '0.State.Status', 'exited'); @@ -478,11 +482,19 @@ public function checkSentinel() } } - public function getMetrics() + public function getCpuMetrics(int $mins = 5) { - if ($this->is_metrics_enabled) { - $from = now()->subMinutes(5)->toIso8601ZuluString(); - $cpu = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl http://localhost:8888/api/cpu/history?from=$from'"], $this, false); + if ($this->isMetricsEnabled()) { + $from = now()->subMinutes($mins)->toIso8601ZuluString(); + $cpu = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$this->settings->metrics_token}\" http://localhost:8888/api/cpu/history?from=$from'"], $this, false); + if (str($cpu)->contains('error')) { + $error = json_decode($cpu, true); + $error = data_get($error, 'error', 'Something is not okay, are you okay?'); + if ($error == 'Unauthorized') { + $error = 'Unauthorized, please check your metrics token or restart Sentinel to set a new token.'; + } + throw new \Exception($error); + } $cpu = str($cpu)->explode("\n")->skip(1)->all(); $parsedCollection = collect($cpu)->flatMap(function ($item) { return collect(explode("\n", trim($item)))->map(function ($line) { @@ -495,6 +507,31 @@ public function getMetrics() return $parsedCollection; } } + public function getMemoryMetrics(int $mins = 5) + { + if ($this->isMetricsEnabled()) { + $from = now()->subMinutes($mins)->toIso8601ZuluString(); + $memory = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$this->settings->metrics_token}\" http://localhost:8888/api/memory/history?from=$from'"], $this, false); + if (str($memory)->contains('error')) { + $error = json_decode($memory, true); + $error = data_get($error, 'error', 'Something is not okay, are you okay?'); + if ($error == 'Unauthorized') { + $error = 'Unauthorized, please check your metrics token or restart Sentinel to set a new token.'; + } + throw new \Exception($error); + } + $memory = str($memory)->explode("\n")->skip(1)->all(); + $parsedCollection = collect($memory)->flatMap(function ($item) { + return collect(explode("\n", trim($item)))->map(function ($line) { + [$time, $used, $free, $usedPercent] = explode(',', trim($line)); + + return [(int) $time, (float) $usedPercent]; + }); + })->toArray(); + + return $parsedCollection; + } + } public function isServerReady(int $tries = 3) { @@ -609,7 +646,7 @@ public function getContainers(): Collection $containers = instant_remote_process(["docker service inspect $(docker service ls -q) --format '{{json .}}'"], $this, false); } else { $containers = instant_remote_process(['docker container ls -q'], $this, false); - if (! $containers) { + if (!$containers) { return collect([]); } $containers = instant_remote_process(["docker container inspect $(docker container ls -q) --format '{{json .}}'"], $this, false); @@ -629,7 +666,7 @@ public function loadUnmanagedContainers(): Collection $containers = format_docker_command_output_to_json($containers); $containers = $containers->map(function ($container) { $labels = data_get($container, 'Labels'); - if (! str($labels)->contains('coolify.managed')) { + if (!str($labels)->contains('coolify.managed')) { return $container; } @@ -701,7 +738,7 @@ public function dockerComposeBasedPreviewDeployments() return $this->previews()->filter(function ($preview) { $applicationId = data_get($preview, 'application_id'); $application = Application::find($applicationId); - if (! $application) { + if (!$application) { return false; } @@ -785,9 +822,9 @@ public function isProxyShouldRun() public function isFunctional() { - $isFunctional = $this->settings->is_reachable && $this->settings->is_usable && ! $this->settings->force_disabled; + $isFunctional = $this->settings->is_reachable && $this->settings->is_usable && !$this->settings->force_disabled; ['private_key_filename' => $private_key_filename, 'mux_filename' => $mux_filename] = server_ssh_configuration($this); - if (! $isFunctional) { + if (!$isFunctional) { Storage::disk('ssh-keys')->delete($private_key_filename); Storage::disk('ssh-mux')->delete($mux_filename); } @@ -846,7 +883,7 @@ public function validateConnection() config()->set('coolify.mux_enabled', false); $server = Server::find($this->id); - if (! $server) { + if (!$server) { return ['uptime' => false, 'error' => 'Server not found.']; } if ($server->skipServer()) { diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php index 7994c10af..47ed4b9a4 100644 --- a/bootstrap/helpers/shared.php +++ b/bootstrap/helpers/shared.php @@ -157,10 +157,12 @@ function get_route_parameters(): array function get_latest_sentinel_version(): string { + if (isDev()) { + return '0.0.8'; + } try { $response = Http::get('https://cdn.coollabs.io/coolify/versions.json'); $versions = $response->json(); - return data_get($versions, 'coolify.sentinel.version'); } catch (\Throwable $e) { //throw $e; @@ -2282,3 +2284,8 @@ function isAnyDeploymentInprogress() echo "No deployments in progress.\n"; exit(0); } + +function generateSentinelToken() { + $token = Str::random(64); + return $token; +} diff --git a/database/migrations/2024_06_18_105947_disable_server_metrics_by_default.php b/database/migrations/2024_06_18_105947_disable_server_metrics_by_default.php deleted file mode 100644 index 5338f457d..000000000 --- a/database/migrations/2024_06_18_105947_disable_server_metrics_by_default.php +++ /dev/null @@ -1,31 +0,0 @@ -boolean('is_metrics_enabled')->default(false)->change(); - }); - Server::where('is_metrics_enabled', true)->update(['is_metrics_enabled' => false]); - } - - /** - * Reverse the migrations. - */ - public function down(): void - { - Schema::table('servers', function (Blueprint $table) { - $table->boolean('is_metrics_enabled')->default(true)->change(); - }); - Server::where('is_metrics_enabled', false)->update(['is_metrics_enabled' => true]); - } -}; diff --git a/database/migrations/2024_06_18_105948_move_server_metrics.php b/database/migrations/2024_06_18_105948_move_server_metrics.php new file mode 100644 index 000000000..c8cbb3f6f --- /dev/null +++ b/database/migrations/2024_06_18_105948_move_server_metrics.php @@ -0,0 +1,41 @@ +dropColumn('is_metrics_enabled'); + }); + Schema::table('server_settings', function (Blueprint $table) { + $table->boolean('is_metrics_enabled')->default(false); + $table->integer('metrics_refresh_rate_seconds')->default(5); + $table->integer('metrics_history_days')->default(30); + $table->string('metrics_token')->default(generateSentinelToken()); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('servers', function (Blueprint $table) { + $table->boolean('is_metrics_enabled')->default(true); + }); + Schema::table('server_settings', function (Blueprint $table) { + $table->dropColumn('is_metrics_enabled'); + $table->dropColumn('metrics_refresh_rate_seconds'); + $table->dropColumn('metrics_history_days'); + $table->dropColumn('metrics_token'); + }); + } +}; diff --git a/resources/views/components/apex-charts.blade.php b/resources/views/components/apex-charts.blade.php deleted file mode 100644 index 029e16df7..000000000 --- a/resources/views/components/apex-charts.blade.php +++ /dev/null @@ -1,78 +0,0 @@ -
- - diff --git a/resources/views/livewire/charts/server-cpu.blade.php b/resources/views/livewire/charts/server-cpu.blade.php new file mode 100644 index 000000000..a39679fd8 --- /dev/null +++ b/resources/views/livewire/charts/server-cpu.blade.php @@ -0,0 +1,127 @@ +