feat: log drain (wip)

app/Actions/Server/InstallLogDrain.php

@@ -0,0 +1,130 @@
+<?php
+
+namespace App\Actions\Server;
+
+use Lorisleiva\Actions\Concerns\AsAction;
+use App\Models\Server;
+
+class InstallLogDrain
+{
+    use AsAction;
+    public function handle(Server $server, string $type)
+    {
+        if ($type === 'newrelic') {
+            if (!$server->settings->is_logdrain_newrelic_enabled) {
+                throw new \Exception('New Relic log drain is not enabled.');
+            }
+            $config = base64_encode("
+[SERVICE]
+    Flush     5
+    Daemon    off
+    Tag container_logs
+[INPUT]
+    Name              forward
+    Buffer_Chunk_Size 1M
+    Buffer_Max_Size   6M
+[FILTER]
+    Name grep
+    Match *
+    Exclude log 127.0.0.1
+[FILTER]
+    Name                modify
+    Match               *
+    Set                 server_name {$server->name}
+
+[OUTPUT]
+    Name nrlogs
+    Match *
+    license_key \${LICENSE_KEY}
+    # https://log-api.eu.newrelic.com/log/v1 - EU
+    # https://log-api.newrelic.com/log/v1 - US
+    base_uri \${BASE_URI}
+");
+        } else if ($type === 'highlight') {
+            if (!$server->settings->is_logdrain_highlight_enabled) {
+                throw new \Exception('Highlight log drain is not enabled.');
+            }
+             $config = base64_encode('
+[SERVICE]
+    Flush     5
+    Daemon    off
+    Tag container_logs
+[INPUT]
+    Name              forward
+    tag               ${HIGHLIGHT_PROJECT_ID}
+    Buffer_Chunk_Size 1M
+    Buffer_Max_Size   6M
+[FILTER]
+    Name grep
+    Match *
+    Exclude log 127.0.0.1
+[FILTER]
+    Name                modify
+    Match               *
+    Set                 server_name {$server->name}
+[OUTPUT]
+    Name                forward
+    Match               *
+    Host                otel.highlight.io
+    Port                24224
+');
+        }
+
+        $compose = base64_encode("
+services:
+  coolify-log-drain:
+    image: cr.fluentbit.io/fluent/fluent-bit:2.0
+    container_name: coolify-log-drain
+    command: -c /fluent-bit.conf
+    env_file:
+      - .env
+    volumes:
+      - ./fluent-bit.conf:/fluent-bit.conf
+    ports:
+      - 127.0.0.1:24224:24224
+");
+        $readme = base64_encode('# New Relic Log Drain
+This log drain is based on [Fluent Bit](https://fluentbit.io/) and New Relic Log Forwarder.
+
+Files:
+- `fluent-bit.conf` - configuration file for Fluent Bit
+- `docker-compose.yml` - docker-compose file to run Fluent Bit
+- `.env` - environment variables for Fluent Bit
+');
+        $license_key = $server->settings->logdrain_newrelic_license_key;
+        $base_uri = $server->settings->logdrain_newrelic_base_uri;
+        $base_path = config('coolify.base_config_path');
+
+        $config_path = $base_path . '/log-drains';
+        $fluent_bit_config = $config_path . '/fluent-bit.conf';
+        $compose_path = $config_path . '/docker-compose.yml';
+        $readme_path = $config_path . '/README.md';
+        $command = [
+            "echo 'Saving configuration'",
+            "mkdir -p $config_path",
+            "echo '{$config}' | base64 -d > $fluent_bit_config",
+            "echo '{$compose}' | base64 -d > $compose_path",
+            "echo '{$readme}' | base64 -d > $readme_path",
+            "rm $config_path/.env || true",
+
+        ];
+        if ($type === 'newrelic') {
+            $add_envs_command = [
+                "echo LICENSE_KEY=$license_key >> $config_path/.env",
+                "echo BASE_URI=$base_uri >> $config_path/.env",
+            ];
+        } else if ($type === 'highlight') {
+            $add_envs_command = [
+                "echo HIGHLIGHT_PROJECT_ID={$server->settings->logdrain_highlight_project_id} >> $config_path/.env",
+            ];
+        }
+        $restart_command = [
+            "echo 'Stopping old Fluent Bit'",
+            "cd $config_path && docker rm -f coolify-log-drain || true",
+            "echo 'Starting Fluent Bit'",
+            "cd $config_path && docker compose up -d --remove-orphans",
+        ];
+        $command = array_merge($command, $add_envs_command, $restart_command);
+        return instant_remote_process($command, $server);
+    }
+}

app/Http/Livewire/Server/LogDrains.php

@@ -0,0 +1,74 @@
+<?php
+
+namespace App\Http\Livewire\Server;
+
+use App\Models\Server;
+use Livewire\Component;
+
+class LogDrains extends Component
+{
+    public Server $server;
+    public $parameters = [];
+    protected $rules = [
+        'server.settings.is_logdrain_newrelic_enabled' => 'required|boolean',
+        'server.settings.logdrain_newrelic_license_key' => 'required|string',
+        'server.settings.logdrain_newrelic_base_uri' => 'required|string',
+        'server.settings.is_logdrain_highlight_enabled' => 'required|boolean',
+        'server.settings.logdrain_highlight_project_id' => 'required|string',
+    ];
+
+    public function mount() {
+        $this->parameters = get_route_parameters();
+        try {
+            $this->server = Server::ownedByCurrentTeam(['name', 'description', 'ip', 'port', 'user', 'proxy'])->whereUuid(request()->server_uuid)->first();
+            if (is_null($this->server)) {
+                return redirect()->route('server.all');
+            }
+        } catch (\Throwable $e) {
+            return handleError($e, $this);
+        }
+    }
+    public function configureLogDrain(string $type) {
+        try {
+            $this->server->logDrain($type);
+            $this->emit('serverRefresh');
+            $this->emit('success', 'Log drain configured successfully.');
+        } catch (\Throwable $e) {
+            return handleError($e, $this);
+        }
+    }
+    public function instantSave(string $type) {
+        $this->submit($type);
+    }
+    public function submit(string $type) {
+        try {
+            $this->resetErrorBag();
+            if ($type === 'newrelic') {
+                $this->validate([
+                    'server.settings.is_logdrain_newrelic_enabled' => 'required|boolean',
+                    'server.settings.logdrain_newrelic_license_key' => 'required|string',
+                    'server.settings.logdrain_newrelic_base_uri' => 'required|string',
+                ]);
+                $this->server->settings->update([
+                    'is_logdrain_highlight_enabled' => false,
+                ]);
+            } else if ($type === 'highlight') {
+                $this->validate([
+                    'server.settings.is_logdrain_highlight_enabled' => 'required|boolean',
+                    'server.settings.logdrain_highlight_project_id' => 'required|string',
+                ]);
+                $this->server->settings->update([
+                    'is_logdrain_newrelic_enabled' => false,
+                ]);
+            }
+            $this->server->settings->save();
+            $this->emit('success', 'Settings saved successfully.');
+        } catch (\Throwable $e) {
+            return handleError($e, $this);
+        }
+    }
+    public function render()
+    {
+        return view('livewire.server.log-drains');
+    }
+}

app/Jobs/ApplicationDeploymentJob.php

@@ -498,7 +498,7 @@ private function health_check()
             if ($this->full_healthcheck_url) {
                 $this->execute_remote_command(
                     [
-                        "echo 'Healthcheck URL inside your container: {$this->full_healthcheck_url}'"
+                        "echo 'Healthcheck URL (inside the container): {$this->full_healthcheck_url}'"
                     ]
                 );
             }
@@ -837,13 +837,15 @@ private function generate_compose_file()
                     'networks' => [
                         $this->destination->network,
                     ],
-                    // 'logging' => [
+                    'logging' => [
-                    //     'driver' => 'fluentd',
+                        'driver' => 'fluentd',
-                    //     'options' => [
+                        'options' => [
-                    //         'fluentd-async' => 'true',
+                            'fluentd-address' => "tcp://127.0.0.1:24224",
-                    //         'tag' => $this->application->name . '-' . $this->application->uuid
+                            'fluentd-async' => "true",
-                    //     ]
+                            'fluentd-sub-second-precision' => "true",
-                    // ],
+
+                        ]
+                    ],
                     'healthcheck' => [
                         'test' => [
                             'CMD-SHELL',

app/Models/Server.php

@@ -2,6 +2,8 @@
 
 namespace App\Models;
 
+use App\Actions\Server\InstallLogDrain;
+use App\Actions\Server\InstallNewRelic;
 use App\Enums\ProxyStatus;
 use App\Enums\ProxyTypes;
 use App\Notifications\Server\Revived;
@@ -296,6 +298,10 @@ public function isProxyShouldRun()
         // }
         return true;
     }
+    public function logDrain($type)
+    {
+        InstallLogDrain::run($this, $type);
+    }
     public function isFunctional()
     {
         return $this->settings->is_reachable && $this->settings->is_usable;

database/migrations/2023_11_16_220647_add_log_drains.php

@@ -0,0 +1,38 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        Schema::table('server_settings', function (Blueprint $table) {
+            $table->boolean('is_logdrain_newrelic_enabled')->default(false);
+            $table->string('logdrain_newrelic_license_key')->nullable();
+            $table->string('logdrain_newrelic_base_uri')->nullable();
+
+            $table->boolean('is_logdrain_highlight_enabled')->default(false);
+            $table->string('logdrain_highlight_project_id')->nullable();
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::table('server_settings', function (Blueprint $table) {
+            $table->dropColumn('is_logdrain_newrelic_enabled');
+            $table->dropColumn('logdrain_newrelic_license_key');
+            $table->dropColumn('logdrain_newrelic_base_uri');
+
+            $table->dropColumn('is_logdrain_highlight_enabled');
+            $table->dropColumn('logdrain_highlight_project_id');
+        });
+    }
+};

examples/fluent-bit/fluent-bit.conf

@@ -1,16 +0,0 @@
-[SERVICE]
-    Flush     1
-    Daemon    off
-[INPUT]
-    Name              forward
-    Buffer_Chunk_Size 1M
-    Buffer_Max_Size   6M
-# [OUTPUT]
-#     Name nrlogs
-#     Match *
-#     license_key ${LICENSE_KEY}
-#     base_uri https://log-api.eu.newrelic.com/log/v1
-
-[OUTPUT]
-    Name stdout
-    Match *

examples/fluent-bit/fluent-bit.yaml

@@ -1,9 +0,0 @@
-version: '3'
-services:
-  coolify-fluent-bit:
-    image: cr.fluentbit.io/fluent/fluent-bit:2.0
-    command: -c /fluent-bit.conf
-    volumes:
-      - ./fluent-bit.conf:/fluent-bit.conf
-    ports:
-      - 24224:24224

examples/newrelic.yaml

@@ -1,21 +0,0 @@
-version: '3'
-services:
-  newrelic-infra:
-    container_name: newrelic-infra
-    image: newrelic/infrastructure:latest
-    networks:
-      - coolify
-    cap_add:
-      - SYS_PTRACE
-    privileged: true
-    pid: host
-    volumes:
-      - "/:/host:ro"
-      - "/var/run/docker.sock:/var/run/docker.sock"
-      - "newrelic-infra:/etc/newrelic-infra"
-    environment:
-      - NRIA_LICENSE_KEY=${NRIA_LICENSE_KEY}
-      - NRIA_DISPLAY_NAME=${HOSTNAME}
-
-networks:
-  coolify:

examples/otl/config.yaml

@@ -1,34 +0,0 @@
-receivers:
-  hostmetrics:
-    collection_interval: 5s
-    scrapers:
-      cpu:
-        metrics:
-          system.cpu.utilization:
-            enabled: true
-processors:
-  resourcedetection:
-    detectors: [env, system]
-    system:
-      hostname_sources: ["os"]
-      resource_attributes:
-        host.id:
-          enabled: true
-  batch:
-  memory_limiter:
-    check_interval: 1s
-    limit_mib: 1000
-    spike_limit_mib: 200
-exporters:
-  debug:
-    verbosity: detailed
-  otlp:
-    endpoint: ${OTLP_ENDPOINT}
-    headers:
-      api-key: ${OTLP_API_KEY}
-service:
-  pipelines:
-    metrics:
-      receivers: [hostmetrics]
-      processors: [memory_limiter, resourcedetection, batch]
-      exporters: [debug, otlp]

resources/views/components/server/navbar.blade.php

@@ -32,6 +32,12 @@
             ]) }}">
             <button>Destinations</button>
         </a>
+        <a class="{{ request()->routeIs('server.log-drains') ? 'text-white' : '' }}"
+            href="{{ route('server.log-drains', [
+                'server_uuid' => data_get($parameters, 'server_uuid'),
+            ]) }}">
+            <button>Log Drains</button>
+        </a>
         <div class="flex-1"></div>
         <livewire:server.proxy.deploy :server="$server" />
     </nav>

resources/views/livewire/server/log-drains.blade.php

@@ -0,0 +1,49 @@
+<div>
+    <x-server.navbar :server="$server" :parameters="$parameters" />
+    <h2>Log Drains</h2>
+    <div class="pb-4">Sends resource logs to external services.</div>
+    <div class="flex flex-col gap-4">
+        <div class="p-4 border border-coolgray-500">
+            <h3>New Relic</h3>
+            <div class="w-32">
+                <x-forms.checkbox instantSave='instantSave("newrelic")' id="server.settings.is_logdrain_newrelic_enabled" label="Enabled" />
+            </div>
+            <form wire:submit.prevent='submit("newrelic")' class="flex flex-col">
+                <div class="flex flex-col gap-4">
+                    <div class="flex flex-col w-full gap-2 xl:flex-row">
+                        <x-forms.input type="password" required id="server.settings.logdrain_newrelic_license_key" label="License Key" />
+                        <x-forms.input required id="server.settings.logdrain_newrelic_base_uri" placeholder="https://log-api.eu.newrelic.com/log/v1" label="Endpoint (EU / US)" />
+                    </div>
+
+                </div>
+                <div class="flex justify-end gap-4 pt-6">
+                    <x-forms.button type="submit">
+                        Save
+                    </x-forms.button>
+                    <x-forms.button wire:click="configureLogDrain('newrelic')">
+                        Configure On Server
+                    </x-forms.button>
+                </div>
+            </form>
+            <h3>Highlight.io</h3>
+            <div class="w-32">
+                <x-forms.checkbox instantSave='instantSave("highlight")' id="server.settings.is_logdrain_highlight_enabled" label="Enabled" />
+            </div>
+            <form wire:submit.prevent='submit("highlight")' class="flex flex-col">
+                <div class="flex flex-col gap-4">
+                    <div class="flex flex-col w-full gap-2 xl:flex-row">
+                        <x-forms.input type="password" required id="server.settings.logdrain_highlight_project_id" label="Project Id" />
+                    </div>
+                </div>
+                <div class="flex justify-end gap-4 pt-6">
+                    <x-forms.button type="submit">
+                        Save
+                    </x-forms.button>
+                    <x-forms.button wire:click="configureLogDrain('highlight')">
+                        Configure On Server
+                    </x-forms.button>
+                </div>
+            </form>
+        </div>
+    </div>
+</div>

routes/web.php

@@ -16,6 +16,7 @@
 use App\Http\Livewire\Server\All;
 use App\Http\Livewire\Server\Create;
 use App\Http\Livewire\Server\Destination\Show as DestinationShow;
+use App\Http\Livewire\Server\LogDrains;
 use App\Http\Livewire\Server\PrivateKey\Show as PrivateKeyShow;
 use App\Http\Livewire\Server\Proxy\Show as ProxyShow;
 use App\Http\Livewire\Server\Proxy\Logs as ProxyLogs;
@@ -130,6 +131,7 @@
     Route::get('/server/{server_uuid}/proxy/logs', ProxyLogs::class)->name('server.proxy.logs');
     Route::get('/server/{server_uuid}/private-key', PrivateKeyShow::class)->name('server.private-key');
     Route::get('/server/{server_uuid}/destinations', DestinationShow::class)->name('server.destinations');
+    Route::get('/server/{server_uuid}/log-drains', LogDrains::class)->name('server.log-drains');
 });