feat: sentinel + charts

This commit is contained in:
Andras Bacsai 2024-06-18 16:42:42 +02:00
parent 83983bbb32
commit 23ed697b98
24 changed files with 557 additions and 193 deletions

View File

@ -12,10 +12,13 @@ class StartSentinel
public function handle(Server $server, $version = 'latest', bool $restart = false)
{
if ($restart) {
instant_remote_process(['docker rm -f coolify-sentinel'], $server, false);
StopSentinel::run($server);
}
$metrics_history = $server->settings->metrics_history_days;
$refresh_rate = $server->settings->metrics_refresh_rate_seconds;
$token = $server->settings->metrics_token;
instant_remote_process([
"docker run --rm --pull always -d -e \"SCHEDULER=true\" -e \"METRICS_HISTORY=10\" -e \"REFRESH_RATE=5\" --name coolify-sentinel -v /var/run/docker.sock:/var/run/docker.sock -v /data/coolify/metrics:/app/metrics -v /data/coolify/logs:/app/logs --pid host --health-cmd \"curl --fail http://127.0.0.1:8888/api/health || exit 1\" --health-interval 10s --health-retries 3 ghcr.io/coollabsio/sentinel:$version",
"docker run --rm --pull always -d -e \"TOKEN={$token}\" -e \"SCHEDULER=true\" -e \"METRICS_HISTORY={$metrics_history}\" -e \"REFRESH_RATE={$refresh_rate}\" --name coolify-sentinel -v /var/run/docker.sock:/var/run/docker.sock -v /data/coolify/metrics:/app/metrics -v /data/coolify/logs:/app/logs --pid host --health-cmd \"curl --fail http://127.0.0.1:8888/api/health || exit 1\" --health-interval 10s --health-retries 3 ghcr.io/coollabsio/sentinel:$version",
'chown -R 9999:root /data/coolify/metrics /data/coolify/logs',
'chmod -R 700 /data/coolify/metrics /data/coolify/logs',
], $server, false);

View File

@ -0,0 +1,16 @@
<?php
namespace App\Actions\Server;
use App\Models\Server;
use Lorisleiva\Actions\Concerns\AsAction;
class StopSentinel
{
use AsAction;
public function handle(Server $server)
{
instant_remote_process(['docker rm -f coolify-sentinel'], $server, false);
}
}

View File

@ -61,7 +61,7 @@ class Kernel extends ConsoleKernel
{
$servers = $this->all_servers->where('settings.is_usable', true)->where('settings.is_reachable', true)->where('ip', '!=', '1.2.3.4');
foreach ($servers as $server) {
if ($server->is_metrics_enabled) {
if ($server->isMetricsEnabled()) {
$schedule->job(new PullSentinelImageJob($server))->everyFiveMinutes()->onOneServer();
}
$schedule->job(new PullHelperImageJob($server))->everyFiveMinutes()->onOneServer();

View File

@ -36,9 +36,6 @@ class PullSentinelImageJob implements ShouldBeEncrypted, ShouldQueue
{
try {
$version = get_latest_sentinel_version();
if (isDev()) {
$version = '0.0.5';
}
if (!$version) {
ray('Failed to get latest Sentinel version');

View File

@ -48,7 +48,7 @@ class ServerStatusJob implements ShouldBeEncrypted, ShouldQueue
if ($this->server->isFunctional()) {
$this->cleanup(notify: false);
$this->remove_unnecessary_coolify_yaml();
if ($this->server->is_metrics_enabled) {
if ($this->server->isMetricsEnabled()) {
$this->server->checkSentinel();
}
}

View File

@ -12,7 +12,7 @@ use Livewire\Component;
class Index extends Component
{
protected $listeners = ['serverInstalled' => 'validateServer'];
protected $listeners = ['refreshBoardingIndex' => 'validateServer'];
public string $currentState = 'welcome';

View File

@ -1,38 +0,0 @@
<?php
namespace App\Livewire\Charts;
use App\Models\Server as ModelsServer;
use Livewire\Component;
class Server extends Component
{
public ModelsServer $server;
public $chartId = 'server';
public $data;
public $categories;
public function render()
{
return view('livewire.charts.server');
}
public function mount()
{
$this->loadData();
}
public function loadData()
{
$metrics = $this->server->getMetrics();
$metrics = collect($metrics)->map(function ($metric) {
return [$metric[0], $metric[1]];
});
$this->dispatch("refreshChartData-{$this->chartId}", [
'seriesData' => $metrics,
]);
}
}

View File

@ -0,0 +1,48 @@
<?php
namespace App\Livewire\Charts;
use App\Models\Server as ModelsServer;
use Livewire\Component;
class ServerCpu extends Component
{
public ModelsServer $server;
public $chartId = 'server-cpu';
public $data;
public $categories;
public $interval = 5;
public function render()
{
return view('livewire.charts.server-cpu');
}
public function mount()
{
$this->loadData();
}
public function loadData()
{
try {
$metrics = $this->server->getCpuMetrics($this->interval);
$metrics = collect($metrics)->map(function ($metric) {
return [$metric[0], $metric[1]];
});
$this->dispatch("refreshChartData-{$this->chartId}", [
'seriesData' => $metrics,
]);
} catch (\Throwable $e) {
return handleError($e, $this);
}
}
public function setInterval()
{
$this->loadData();
}
}

View File

@ -0,0 +1,45 @@
<?php
namespace App\Livewire\Charts;
use App\Models\Server;
use Livewire\Component;
class ServerMemory extends Component
{
public Server $server;
public $chartId = 'server-memory';
public $data;
public $categories;
public $interval = 5;
public function render()
{
return view('livewire.charts.server-memory');
}
public function mount()
{
$this->loadData();
}
public function loadData()
{
try {
$metrics = $this->server->getMemoryMetrics($this->interval);
$metrics = collect($metrics)->map(function ($metric) {
return [$metric[0], $metric[1]];
});
$this->dispatch("refreshChartData-{$this->chartId}", [
'seriesData' => $metrics,
]);
} catch (\Throwable $e) {
return handleError($e, $this);
}
}
public function setInterval() {
$this->loadData();
}
}

View File

@ -64,7 +64,7 @@ class Logs extends Component
return;
$server = data_get($this->resource, 'destination.server');
if ($server->isFunctional()) {
$this->cpu = $server->getMetrics();
$this->cpu = $server->getCpuMetrics();
}
}

View File

@ -21,7 +21,7 @@ class ConfigureCloudflareTunnels extends Component
$server->settings->is_cloudflare_tunnel = true;
$server->settings->save();
$this->dispatch('success', 'Cloudflare Tunnels configured successfully.');
$this->dispatch('serverInstalled');
$this->dispatch('refreshServerShow');
} catch (\Throwable $e) {
return handleError($e, $this);
}
@ -37,7 +37,7 @@ class ConfigureCloudflareTunnels extends Component
$server->save();
$server->settings->save();
$this->dispatch('success', 'Cloudflare Tunnels configured successfully.');
$this->dispatch('serverInstalled');
$this->dispatch('refreshServerShow');
} catch (\Throwable $e) {
return handleError($e, $this);
}

View File

@ -2,6 +2,9 @@
namespace App\Livewire\Server;
use App\Actions\Server\StartSentinel;
use App\Actions\Server\StopSentinel;
use App\Jobs\PullSentinelImageJob;
use App\Models\Server;
use Livewire\Component;
@ -36,6 +39,10 @@ class Form extends Component
'server.settings.is_build_server' => 'required|boolean',
'server.settings.concurrent_builds' => 'required|integer|min:1',
'server.settings.dynamic_timeout' => 'required|integer|min:1',
'server.settings.is_metrics_enabled' => 'required|boolean',
'server.settings.metrics_token' => 'required',
'server.settings.metrics_refresh_rate_seconds' => 'required|integer|min:1',
'server.settings.metrics_history_days' => 'required|integer|min:1',
'wildcard_domain' => 'nullable|url',
];
@ -52,6 +59,10 @@ class Form extends Component
'server.settings.is_build_server' => 'Build Server',
'server.settings.concurrent_builds' => 'Concurrent Builds',
'server.settings.dynamic_timeout' => 'Dynamic Timeout',
'server.settings.is_metrics_enabled' => 'Metrics',
'server.settings.metrics_token' => 'Metrics Token',
'server.settings.metrics_refresh_rate_seconds' => 'Metrics Interval',
'server.settings.metrics_history_days' => 'Metrics History',
];
@ -69,7 +80,7 @@ class Form extends Component
public function updatedServerSettingsIsBuildServer()
{
$this->dispatch('serverInstalled');
$this->dispatch('refreshServerShow');
$this->dispatch('serverRefresh');
$this->dispatch('proxyStatusUpdated');
}
@ -80,7 +91,24 @@ class Form extends Component
refresh_server_connection($this->server->privateKey);
$this->validateServer(false);
$this->server->settings->save();
$this->server->save();
$this->dispatch('success', 'Server updated.');
$this->dispatch('refreshServerShow');
if ($this->server->isMetricsEnabled()) {
PullSentinelImageJob::dispatchSync($this->server);
$this->dispatch('reloadWindow');
} else {
StopSentinel::dispatch($this->server);
}
} catch (\Throwable $e) {
return handleError($e, $this);
}
}
public function restartSentinel() {
try {
$version = get_latest_sentinel_version();
StartSentinel::run($this->server, $version, true);
$this->dispatch('success', 'Sentinel restarted.');
} catch (\Throwable $e) {
return handleError($e, $this);
}

View File

@ -14,7 +14,7 @@ class Show extends Component
public $parameters = [];
protected $listeners = ['serverInstalled' => '$refresh'];
protected $listeners = ['refreshServerShow' => '$refresh'];
public function mount()
{

View File

@ -143,7 +143,8 @@ class ValidateAndInstall extends Component
} else {
$this->docker_version = $this->server->validateDockerEngineVersion();
if ($this->docker_version) {
$this->dispatch('serverInstalled');
$this->dispatch('refreshServerShow');
$this->dispatch('refreshBoardingIndex');
$this->dispatch('success', 'Server validated.');
} else {
$this->error = 'Docker Engine version is not 22+. Please install Docker manually before continuing: <a target="_blank" class="underline" href="https://docs.docker.com/engine/install/#server">documentation</a>.';

View File

@ -462,10 +462,14 @@ $schema://$host {
Storage::disk('ssh-mux')->delete($this->muxFilename());
}
public function isMetricsEnabled()
{
return $this->settings->is_metrics_enabled;
}
public function checkSentinel()
{
ray("Checking sentinel on server: {$this->name}");
if ($this->is_metrics_enabled) {
if ($this->isMetricsEnabled()) {
$sentinel_found = instant_remote_process(['docker inspect coolify-sentinel'], $this, false);
$sentinel_found = json_decode($sentinel_found, true);
$status = data_get($sentinel_found, '0.State.Status', 'exited');
@ -478,11 +482,19 @@ $schema://$host {
}
}
public function getMetrics()
public function getCpuMetrics(int $mins = 5)
{
if ($this->is_metrics_enabled) {
$from = now()->subMinutes(5)->toIso8601ZuluString();
$cpu = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl http://localhost:8888/api/cpu/history?from=$from'"], $this, false);
if ($this->isMetricsEnabled()) {
$from = now()->subMinutes($mins)->toIso8601ZuluString();
$cpu = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$this->settings->metrics_token}\" http://localhost:8888/api/cpu/history?from=$from'"], $this, false);
if (str($cpu)->contains('error')) {
$error = json_decode($cpu, true);
$error = data_get($error, 'error', 'Something is not okay, are you okay?');
if ($error == 'Unauthorized') {
$error = 'Unauthorized, please check your metrics token or restart Sentinel to set a new token.';
}
throw new \Exception($error);
}
$cpu = str($cpu)->explode("\n")->skip(1)->all();
$parsedCollection = collect($cpu)->flatMap(function ($item) {
return collect(explode("\n", trim($item)))->map(function ($line) {
@ -495,6 +507,31 @@ $schema://$host {
return $parsedCollection;
}
}
public function getMemoryMetrics(int $mins = 5)
{
if ($this->isMetricsEnabled()) {
$from = now()->subMinutes($mins)->toIso8601ZuluString();
$memory = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$this->settings->metrics_token}\" http://localhost:8888/api/memory/history?from=$from'"], $this, false);
if (str($memory)->contains('error')) {
$error = json_decode($memory, true);
$error = data_get($error, 'error', 'Something is not okay, are you okay?');
if ($error == 'Unauthorized') {
$error = 'Unauthorized, please check your metrics token or restart Sentinel to set a new token.';
}
throw new \Exception($error);
}
$memory = str($memory)->explode("\n")->skip(1)->all();
$parsedCollection = collect($memory)->flatMap(function ($item) {
return collect(explode("\n", trim($item)))->map(function ($line) {
[$time, $used, $free, $usedPercent] = explode(',', trim($line));
return [(int) $time, (float) $usedPercent];
});
})->toArray();
return $parsedCollection;
}
}
public function isServerReady(int $tries = 3)
{

View File

@ -157,10 +157,12 @@ function get_route_parameters(): array
function get_latest_sentinel_version(): string
{
if (isDev()) {
return '0.0.8';
}
try {
$response = Http::get('https://cdn.coollabs.io/coolify/versions.json');
$versions = $response->json();
return data_get($versions, 'coolify.sentinel.version');
} catch (\Throwable $e) {
//throw $e;
@ -2282,3 +2284,8 @@ function isAnyDeploymentInprogress()
echo "No deployments in progress.\n";
exit(0);
}
function generateSentinelToken() {
$token = Str::random(64);
return $token;
}

View File

@ -1,31 +0,0 @@
<?php
use App\Models\Server;
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('servers', function (Blueprint $table) {
$table->boolean('is_metrics_enabled')->default(false)->change();
});
Server::where('is_metrics_enabled', true)->update(['is_metrics_enabled' => false]);
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('servers', function (Blueprint $table) {
$table->boolean('is_metrics_enabled')->default(true)->change();
});
Server::where('is_metrics_enabled', false)->update(['is_metrics_enabled' => true]);
}
};

View File

@ -0,0 +1,41 @@
<?php
use App\Models\Server;
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('servers', function (Blueprint $table) {
$table->dropColumn('is_metrics_enabled');
});
Schema::table('server_settings', function (Blueprint $table) {
$table->boolean('is_metrics_enabled')->default(false);
$table->integer('metrics_refresh_rate_seconds')->default(5);
$table->integer('metrics_history_days')->default(30);
$table->string('metrics_token')->default(generateSentinelToken());
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('servers', function (Blueprint $table) {
$table->boolean('is_metrics_enabled')->default(true);
});
Schema::table('server_settings', function (Blueprint $table) {
$table->dropColumn('is_metrics_enabled');
$table->dropColumn('metrics_refresh_rate_seconds');
$table->dropColumn('metrics_history_days');
$table->dropColumn('metrics_token');
});
}
};

View File

@ -1,78 +0,0 @@
<div wire:ignore id="{!! $chartId !!}"></div>
<script>
const options = {
chart: {
height: '150px',
id: '{!! $chartId !!}',
type: 'area',
stroke: {
curve: 'straight',
},
toolbar: {
show: false,
tools: {
download: true,
selection: false,
zoom: false,
zoomin: false,
zoomout: false,
pan: false,
reset: false
},
},
animations: {
enabled: false,
},
},
grid: {
show: true,
borderColor: '',
},
colors: ['red'],
xaxis: {
type: 'datetime',
labels: {
show: true,
style: {
colors: '#ffffff',
}
}
},
yaxis: {
show: true,
labels: {
show: false,
}
},
series: [{
name: '{!! $seriesName !!}',
data: '{!! $seriesData !!}'
}],
noData: {
text: 'Loading...'
},
tooltip: {
enabled: false
},
legend: {
show: false
}
}
const chart = new ApexCharts(document.getElementById(`{!! $chartId !!}`), options);
chart.render();
document.addEventListener('livewire:init', () => {
Livewire.on('refreshChartData-{!! $chartId !!}', (chartData) => {
chart.updateOptions({
series: [{
data: chartData[0].seriesData,
}],
});
});
});
</script>

View File

@ -0,0 +1,127 @@
<div wire:poll.5000ms='loadData'>
<h3>CPU</h3>
<x-forms.select label="Interval" wire:change="setInterval" id="interval">
<option value="5">5 minutes</option>
<option value="10">10 minutes</option>
<option value="30">30 minutes</option>
<option value="60">1 hour</option>
<option value="720">12 hours</option>
<option value="10080">1 week</option>
<option value="43200">30 days</option>
</x-forms.select>
<div wire:ignore id="{!! $chartId !!}"></div>
<script>
checkTheme();
const optionsServerCpu = {
chart: {
height: '150px',
id: '{!! $chartId !!}',
type: 'area',
stroke: {
curve: 'straight',
},
toolbar: {
show: false,
tools: {
download: true,
selection: false,
zoom: false,
zoomin: false,
zoomout: false,
pan: false,
reset: false
},
},
animations: {
enabled: false,
},
},
fill: {
type: 'gradient',
},
dataLabels: {
enabled: false,
offsetY: -10,
style: {
colors: ['#FCD452'],
},
background: {
enabled: false,
}
},
grid: {
show: true,
borderColor: '',
},
colors: [baseColor],
xaxis: {
type: 'datetime',
labels: {
show: true,
style: {
colors: textColor,
}
}
},
yaxis: {
show: true,
labels: {
show: true,
style: {
colors: textColor,
}
}
},
series: [{
data: '{!! $data !!}'
}],
noData: {
text: 'Loading...',
style: {
color: textColor,
}
},
tooltip: {
enabled: false,
},
legend: {
show: false
}
}
const serverCpuChart = new ApexCharts(document.getElementById(`{!! $chartId !!}`), optionsServerCpu);
serverCpuChart.render();
document.addEventListener('livewire:init', () => {
Livewire.on('refreshChartData-{!! $chartId !!}', (chartData) => {
checkTheme();
serverCpuChart.updateOptions({
series: [{
data: chartData[0].seriesData,
}],
colors: [baseColor],
xaxis: {
labels: {
style: {
colors: textColor,
}
}
},
yaxis: {
labels: {
style: {
colors: textColor,
}
}
},
noData: {
style: {
color: textColor,
}
}
});
});
});
</script>
</div>

View File

@ -0,0 +1,128 @@
<div wire:poll.5000ms='loadData'>
<h3>Memory</h3>
<x-forms.select label="Interval" wire:change="setInterval" id="interval">
<option value="5">5 minutes</option>
<option value="10">10 minutes</option>
<option value="30">30 minutes</option>
<option value="60">1 hour</option>
<option value="720">12 hours</option>
<option value="10080">1 week</option>
<option value="43200">30 days</option>
</x-forms.select>
<div wire:ignore id="{!! $chartId !!}"></div>
<script>
checkTheme();
const optionsServerMemory = {
chart: {
height: '150px',
id: '{!! $chartId !!}',
type: 'area',
stroke: {
curve: 'straight',
},
toolbar: {
show: false,
tools: {
download: true,
selection: false,
zoom: false,
zoomin: false,
zoomout: false,
pan: false,
reset: false
},
},
animations: {
enabled: false,
},
},
fill: {
type: 'gradient',
},
dataLabels: {
enabled: false,
offsetY: -10,
style: {
colors: ['#FCD452'],
},
background: {
enabled: false,
}
},
grid: {
show: true,
borderColor: '',
},
colors: [baseColor],
xaxis: {
type: 'datetime',
labels: {
show: true,
style: {
colors: textColor,
}
}
},
yaxis: {
show: true,
labels: {
show: true,
style: {
colors: textColor,
}
}
},
series: [{
data: '{!! $data !!}'
}],
noData: {
text: 'Loading...',
style: {
color: textColor,
}
},
tooltip: {
enabled: false,
},
legend: {
show: false
}
}
const serverMemoryChart = new ApexCharts(document.getElementById(`{!! $chartId !!}`), optionsServerMemory);
serverMemoryChart.render();
document.addEventListener('livewire:init', () => {
Livewire.on('refreshChartData-{!! $chartId !!}', (chartData) => {
checkTheme();
serverMemoryChart.updateOptions({
series: [{
data: chartData[0].seriesData,
}],
colors: [baseColor],
xaxis: {
labels: {
style: {
colors: textColor,
}
}
},
yaxis: {
labels: {
style: {
colors: textColor,
}
}
},
noData: {
style: {
color: textColor,
}
}
});
});
});
</script>
</div>

View File

@ -1,4 +0,0 @@
<div wire:poll.5000ms='loadData'>
<h1>CPU Usage</h1>
<x-apex-charts :chart-id="$chartId" :series-data="$data" :categories="$categories" series-name="Total distance this year"/>
</div>

View File

@ -144,6 +144,26 @@
<x-forms.input id="server.settings.dynamic_timeout" label="Deployment timeout (seconds)" required
helper="You can define the maximum duration for a deployment to run before timing it out." />
</div>
<div class="flex items-center gap-2">
<h3 class="py-4">Metrics</h3>
@if ($server->isMetricsEnabled())
<x-forms.button wire:click='restartSentinel'>Restart Collector</x-forms.button>
@endif
</div>
<div class="w-64">
<x-forms.checkbox instantSave id="server.settings.is_metrics_enabled" label="Enable metrics" />
</div>
<div class="pt-4">
<div class="flex flex-wrap gap-2 sm:flex-nowrap">
<x-forms.input type="password" id="server.settings.metrics_token" label="Metrics token" required
helper="Token for collector (Sentinel)." />
<x-forms.input id="server.settings.metrics_refresh_rate_seconds" label="Metrics rate (seconds)"
required
helper="The interval for gathering metrics. Lower means more disk space will be used." />
<x-forms.input id="server.settings.metrics_history_days" label="Metrics history (days)" required
helper="How many days should the metrics data should be reserved." />
</div>
</div>
@endif
</form>
</div>

View File

@ -5,9 +5,26 @@
<x-server.navbar :server="$server" :parameters="$parameters" />
<livewire:server.form :server="$server" />
<livewire:server.delete :server="$server" />
@if (isDev())
@if ($server->isFunctional() && $server->isMetricsEnabled())
<div class="pt-10">
<livewire:charts.server :server="$server" />
<script>
let theme = localStorage.theme
let baseColor = '#FCD452'
let textColor = '#ffffff'
function checkTheme() {
theme = localStorage.theme
if (theme == 'dark') {
baseColor = '#FCD452'
textColor = '#ffffff'
} else {
baseColor = 'black'
textColor = '#000000'
}
}
</script>
<livewire:charts.server-cpu :server="$server" />
<livewire:charts.server-memory :server="$server" />
</div>
@endif
</div>