Shiloh/lasthourcloud
Shiloh/lasthourcloud
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 4 Wiki Activity

feat: log drain (wip)

Browse Source
This commit is contained in:
Andras Bacsai 2023-11-17 00:37:09 +01:00
parent 91e3d33c0b
commit 6c7e091e1b
12 changed files with 315 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

130
app/Actions/Server/InstallLogDrain.php Normal file
View File

@ -0,0 +1,130 @@
<?php
namespace App\Actions\Server;
use Lorisleiva\Actions\Concerns\AsAction;
use App\Models\Server;
class InstallLogDrain
{
use AsAction;
public function handle(Server $server, string $type)
{
if ($type === 'newrelic') {
if (!$server->settings->is_logdrain_newrelic_enabled) {
throw new \Exception('New Relic log drain is not enabled.');
}
$config = base64_encode("
[SERVICE]
Flush 5
Daemon off
Tag container_logs
[INPUT]
Name forward
Buffer_Chunk_Size 1M
Buffer_Max_Size 6M
[FILTER]
Name grep
Match *
Exclude log 127.0.0.1
[FILTER]
Name modify
Match *
Set server_name {$server->name}
[OUTPUT]
Name nrlogs
Match *
license_key \${LICENSE_KEY}
# https://log-api.eu.newrelic.com/log/v1 - EU
# https://log-api.newrelic.com/log/v1 - US
base_uri \${BASE_URI}
");
} else if ($type === 'highlight') {
if (!$server->settings->is_logdrain_highlight_enabled) {
throw new \Exception('Highlight log drain is not enabled.');
}
$config = base64_encode('
[SERVICE]
Flush 5
Daemon off
Tag container_logs
[INPUT]
Name forward
tag ${HIGHLIGHT_PROJECT_ID}
Buffer_Chunk_Size 1M
Buffer_Max_Size 6M
[FILTER]
Name grep
Match *
Exclude log 127.0.0.1
[FILTER]
Name modify
Match *
Set server_name {$server->name}
[OUTPUT]
Name forward
Match *
Host otel.highlight.io
Port 24224
');
}
$compose = base64_encode("
services:
coolify-log-drain:
image: cr.fluentbit.io/fluent/fluent-bit:2.0
container_name: coolify-log-drain
command: -c /fluent-bit.conf
env_file:
- .env
volumes:
- ./fluent-bit.conf:/fluent-bit.conf
ports:
- 127.0.0.1:24224:24224
");
$readme = base64_encode('# New Relic Log Drain
This log drain is based on [Fluent Bit](https://fluentbit.io/) and New Relic Log Forwarder.
Files:
- `fluent-bit.conf` - configuration file for Fluent Bit
- `docker-compose.yml` - docker-compose file to run Fluent Bit
- `.env` - environment variables for Fluent Bit
');
$license_key = $server->settings->logdrain_newrelic_license_key;
$base_uri = $server->settings->logdrain_newrelic_base_uri;
$base_path = config('coolify.base_config_path');
$config_path = $base_path . '/log-drains';
$fluent_bit_config = $config_path . '/fluent-bit.conf';
$compose_path = $config_path . '/docker-compose.yml';
$readme_path = $config_path . '/README.md';
$command = [
"echo 'Saving configuration'",
"mkdir -p $config_path",
"echo '{$config}' | base64 -d > $fluent_bit_config",
"echo '{$compose}' | base64 -d > $compose_path",
"echo '{$readme}' | base64 -d > $readme_path",
"rm $config_path/.env || true",
];
if ($type === 'newrelic') {
$add_envs_command = [
"echo LICENSE_KEY=$license_key >> $config_path/.env",
"echo BASE_URI=$base_uri >> $config_path/.env",
];
} else if ($type === 'highlight') {
$add_envs_command = [
"echo HIGHLIGHT_PROJECT_ID={$server->settings->logdrain_highlight_project_id} >> $config_path/.env",
];
}
$restart_command = [
"echo 'Stopping old Fluent Bit'",
"cd $config_path && docker rm -f coolify-log-drain || true",
"echo 'Starting Fluent Bit'",
"cd $config_path && docker compose up -d --remove-orphans",
];
$command = array_merge($command, $add_envs_command, $restart_command);
return instant_remote_process($command, $server);
}
}

74
app/Http/Livewire/Server/LogDrains.php Normal file
View File

@ -0,0 +1,74 @@
<?php
namespace App\Http\Livewire\Server;
use App\Models\Server;
use Livewire\Component;
class LogDrains extends Component
{
public Server $server;
public $parameters = [];
protected $rules = [
'server.settings.is_logdrain_newrelic_enabled' => 'required|boolean',
'server.settings.logdrain_newrelic_license_key' => 'required|string',
'server.settings.logdrain_newrelic_base_uri' => 'required|string',
'server.settings.is_logdrain_highlight_enabled' => 'required|boolean',
'server.settings.logdrain_highlight_project_id' => 'required|string',
];
public function mount() {
$this->parameters = get_route_parameters();
try {
$this->server = Server::ownedByCurrentTeam(['name', 'description', 'ip', 'port', 'user', 'proxy'])->whereUuid(request()->server_uuid)->first();
if (is_null($this->server)) {
return redirect()->route('server.all');
}
} catch (\Throwable $e) {
return handleError($e, $this);
}
}
public function configureLogDrain(string $type) {
try {
$this->server->logDrain($type);
$this->emit('serverRefresh');
$this->emit('success', 'Log drain configured successfully.');
} catch (\Throwable $e) {
return handleError($e, $this);
}
}
public function instantSave(string $type) {
$this->submit($type);
}
public function submit(string $type) {
try {
$this->resetErrorBag();
if ($type === 'newrelic') {
$this->validate([
'server.settings.is_logdrain_newrelic_enabled' => 'required|boolean',
'server.settings.logdrain_newrelic_license_key' => 'required|string',
'server.settings.logdrain_newrelic_base_uri' => 'required|string',
]);
$this->server->settings->update([
'is_logdrain_highlight_enabled' => false,
]);
} else if ($type === 'highlight') {
$this->validate([
'server.settings.is_logdrain_highlight_enabled' => 'required|boolean',
'server.settings.logdrain_highlight_project_id' => 'required|string',
]);
$this->server->settings->update([
'is_logdrain_newrelic_enabled' => false,
]);
}
$this->server->settings->save();
$this->emit('success', 'Settings saved successfully.');
} catch (\Throwable $e) {
return handleError($e, $this);
}
}
public function render()
{
return view('livewire.server.log-drains');
}
}

18
app/Jobs/ApplicationDeploymentJob.php
View File

@ -498,7 +498,7 @@ private function health_check()
if ($this->full_healthcheck_url) {
$this->execute_remote_command(
[
"echo 'Healthcheck URL inside your container: {$this->full_healthcheck_url}'"
"echo 'Healthcheck URL (inside the container): {$this->full_healthcheck_url}'"
]
);
}
@ -837,13 +837,15 @@ private function generate_compose_file()
'networks' => [
$this->destination->network,
],
// 'logging' => [
// 'driver' => 'fluentd',
// 'options' => [
// 'fluentd-async' => 'true',
// 'tag' => $this->application->name . '-' . $this->application->uuid
// ]
// ],
'logging' => [
'driver' => 'fluentd',
'options' => [
'fluentd-address' => "tcp://127.0.0.1:24224",
'fluentd-async' => "true",
'fluentd-sub-second-precision' => "true",
]
],
'healthcheck' => [
'test' => [
'CMD-SHELL',

6
app/Models/Server.php
View File

@ -2,6 +2,8 @@
namespace App\Models;
use App\Actions\Server\InstallLogDrain;
use App\Actions\Server\InstallNewRelic;
use App\Enums\ProxyStatus;
use App\Enums\ProxyTypes;
use App\Notifications\Server\Revived;
@ -296,6 +298,10 @@ public function isProxyShouldRun()
// }
return true;
}
public function logDrain($type)
{
InstallLogDrain::run($this, $type);
}
public function isFunctional()
{
return $this->settings->is_reachable && $this->settings->is_usable;

38
database/migrations/2023_11_16_220647_add_log_drains.php Normal file
View File

@ -0,0 +1,38 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('server_settings', function (Blueprint $table) {
$table->boolean('is_logdrain_newrelic_enabled')->default(false);
$table->string('logdrain_newrelic_license_key')->nullable();
$table->string('logdrain_newrelic_base_uri')->nullable();
$table->boolean('is_logdrain_highlight_enabled')->default(false);
$table->string('logdrain_highlight_project_id')->nullable();
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('server_settings', function (Blueprint $table) {
$table->dropColumn('is_logdrain_newrelic_enabled');
$table->dropColumn('logdrain_newrelic_license_key');
$table->dropColumn('logdrain_newrelic_base_uri');
$table->dropColumn('is_logdrain_highlight_enabled');
$table->dropColumn('logdrain_highlight_project_id');
});
}
};

16
examples/fluent-bit/fluent-bit.conf
View File

@ -1,16 +0,0 @@
[SERVICE]
Flush 1
Daemon off
[INPUT]
Name forward
Buffer_Chunk_Size 1M
Buffer_Max_Size 6M
# [OUTPUT]
# Name nrlogs
# Match *
# license_key ${LICENSE_KEY}
# base_uri https://log-api.eu.newrelic.com/log/v1
[OUTPUT]
Name stdout
Match *

9
examples/fluent-bit/fluent-bit.yaml
View File

@ -1,9 +0,0 @@
version: '3'
services:
coolify-fluent-bit:
image: cr.fluentbit.io/fluent/fluent-bit:2.0
command: -c /fluent-bit.conf
volumes:
- ./fluent-bit.conf:/fluent-bit.conf
ports:
- 24224:24224

21
examples/newrelic.yaml
View File

@ -1,21 +0,0 @@
version: '3'
services:
newrelic-infra:
container_name: newrelic-infra
image: newrelic/infrastructure:latest
networks:
- coolify
cap_add:
- SYS_PTRACE
privileged: true
pid: host
volumes:
- "/:/host:ro"
- "/var/run/docker.sock:/var/run/docker.sock"
- "newrelic-infra:/etc/newrelic-infra"
environment:
- NRIA_LICENSE_KEY=${NRIA_LICENSE_KEY}
- NRIA_DISPLAY_NAME=${HOSTNAME}
networks:
coolify:

34
examples/otl/config.yaml
View File

@ -1,34 +0,0 @@
receivers:
hostmetrics:
collection_interval: 5s
scrapers:
cpu:
metrics:
system.cpu.utilization:
enabled: true
processors:
resourcedetection:
detectors: [env, system]
system:
hostname_sources: ["os"]
resource_attributes:
host.id:
enabled: true
batch:
memory_limiter:
check_interval: 1s
limit_mib: 1000
spike_limit_mib: 200
exporters:
debug:
verbosity: detailed
otlp:
endpoint: ${OTLP_ENDPOINT}
headers:
api-key: ${OTLP_API_KEY}
service:
pipelines:
metrics:
receivers: [hostmetrics]
processors: [memory_limiter, resourcedetection, batch]
exporters: [debug, otlp]

6
resources/views/components/server/navbar.blade.php
View File

@ -32,6 +32,12 @@
]) }}">
<button>Destinations</button>
</a>
<a class="{{ request()->routeIs('server.log-drains') ? 'text-white' : '' }}"
href="{{ route('server.log-drains', [
'server_uuid' => data_get($parameters, 'server_uuid'),
]) }}">
<button>Log Drains</button>
</a>
<div class="flex-1"></div>
<livewire:server.proxy.deploy :server="$server" />
</nav>

49
resources/views/livewire/server/log-drains.blade.php Normal file
View File

@ -0,0 +1,49 @@
<div>
<x-server.navbar :server="$server" :parameters="$parameters" />
<h2>Log Drains</h2>
<div class="pb-4">Sends resource logs to external services.</div>
<div class="flex flex-col gap-4">
<div class="p-4 border border-coolgray-500">
<h3>New Relic</h3>
<div class="w-32">
<x-forms.checkbox instantSave='instantSave("newrelic")' id="server.settings.is_logdrain_newrelic_enabled" label="Enabled" />
</div>
<form wire:submit.prevent='submit("newrelic")' class="flex flex-col">
<div class="flex flex-col gap-4">
<div class="flex flex-col w-full gap-2 xl:flex-row">
<x-forms.input type="password" required id="server.settings.logdrain_newrelic_license_key" label="License Key" />
<x-forms.input required id="server.settings.logdrain_newrelic_base_uri" placeholder="https://log-api.eu.newrelic.com/log/v1" label="Endpoint (EU / US)" />
</div>
</div>
<div class="flex justify-end gap-4 pt-6">
<x-forms.button type="submit">
Save
</x-forms.button>
<x-forms.button wire:click="configureLogDrain('newrelic')">
Configure On Server
</x-forms.button>
</div>
</form>
<h3>Highlight.io</h3>
<div class="w-32">
<x-forms.checkbox instantSave='instantSave("highlight")' id="server.settings.is_logdrain_highlight_enabled" label="Enabled" />
</div>
<form wire:submit.prevent='submit("highlight")' class="flex flex-col">
<div class="flex flex-col gap-4">
<div class="flex flex-col w-full gap-2 xl:flex-row">
<x-forms.input type="password" required id="server.settings.logdrain_highlight_project_id" label="Project Id" />
</div>
</div>
<div class="flex justify-end gap-4 pt-6">
<x-forms.button type="submit">
Save
</x-forms.button>
<x-forms.button wire:click="configureLogDrain('highlight')">
Configure On Server
</x-forms.button>
</div>
</form>
</div>
</div>
</div>

2
routes/web.php
View File

@ -16,6 +16,7 @@
use App\Http\Livewire\Server\All;
use App\Http\Livewire\Server\Create;
use App\Http\Livewire\Server\Destination\Show as DestinationShow;
use App\Http\Livewire\Server\LogDrains;
use App\Http\Livewire\Server\PrivateKey\Show as PrivateKeyShow;
use App\Http\Livewire\Server\Proxy\Show as ProxyShow;
use App\Http\Livewire\Server\Proxy\Logs as ProxyLogs;
@ -130,6 +131,7 @@
Route::get('/server/{server_uuid}/proxy/logs', ProxyLogs::class)->name('server.proxy.logs');
Route::get('/server/{server_uuid}/private-key', PrivateKeyShow::class)->name('server.private-key');
Route::get('/server/{server_uuid}/destinations', DestinationShow::class)->name('server.destinations');
Route::get('/server/{server_uuid}/log-drains', LogDrains::class)->name('server.log-drains');
});