feat: fluentbit

.gitignore

@@ -12,4 +12,5 @@ client
 apps/api/db/*.db
 local-serve
 apps/api/db/migration.db-journal
-apps/api/core*
+apps/api/core*
+logs

Dockerfile

@@ -33,6 +33,7 @@ RUN chmod +x ~/.docker/cli-plugins/docker-compose /usr/bin/docker
 RUN (curl -sSL "https://github.com/buildpacks/pack/releases/download/v0.27.0/pack-v0.27.0-linux.tgz" | tar -C /usr/local/bin/ --no-same-owner -xzv pack)
 
 COPY --from=build /app/apps/api/build/ .
+COPY --from=build /app/others/fluentbit/ ./fluentbit
 COPY --from=build /app/apps/ui/build/ ./public
 COPY --from=build /app/apps/api/prisma/ ./prisma
 COPY --from=build /app/apps/api/package.json .

apps/api/package.json

@@ -29,6 +29,8 @@
     "bree": "9.1.2",
     "cabin": "9.1.2",
     "compare-versions": "5.0.1",
+    "csv-parse": "^5.3.0",
+    "csvtojson": "^2.0.10",
     "cuid": "2.1.8",
     "dayjs": "1.11.5",
     "dockerode": "3.3.4",

apps/api/src/lib/buildPacks/common.ts

@@ -1,4 +1,4 @@
-import { base64Encode, executeDockerCmd, generateTimestamp, getDomain, isDev, prisma, version } from "../common";
+import { base64Encode, encrypt, executeDockerCmd, generateTimestamp, getDomain, isDev, prisma, version } from "../common";
 import { promises as fs } from 'fs';
 import { day } from "../dayjs";
 
@@ -461,17 +461,30 @@ export const saveBuildLog = async ({
 	buildId: string;
 	applicationId: string;
 }): Promise<any> => {
+	const { default: got } = await import('got')
+
 	if (line && typeof line === 'string' && line.includes('ghs_')) {
 		const regex = /ghs_.*@/g;
 		line = line.replace(regex, '<SENSITIVE_DATA_DELETED>@');
 	}
 	const addTimestamp = `[${generateTimestamp()}] ${line}`;
-	if (isDev) console.debug(`[${applicationId}] ${addTimestamp}`);
+	const fluentBitUrl = isDev ? 'http://localhost:24224' : 'http://coolify-fluentbit:24224';
-	return await prisma.buildLog.create({
+
-		data: {
+	if (isDev) {
-			line: addTimestamp, buildId, time: Number(day().valueOf()), applicationId
+		console.debug(`[${applicationId}] ${addTimestamp}`);
+	}
+	// return await prisma.buildLog.create({
+	// 	data: {
+	// 		line: addTimestamp, buildId, time: Number(day().valueOf()), applicationId
+	// 	}
+	// });
+
+	return await got.post(`${fluentBitUrl}/${applicationId}_buildlog_${buildId}.csv`, {
+		json: {
+			line: encrypt(line)
 		}
-	});
+	})
+
 };
 
 export async function copyBaseConfigurationFiles(

apps/api/src/routes/api/v1/applications/handlers.ts

@@ -5,6 +5,7 @@ import axios from 'axios';
 import { FastifyReply } from 'fastify';
 import fs from 'fs/promises';
 import yaml from 'js-yaml';
+import csv from 'csvtojson';
 
 import { day } from '../../../../lib/dayjs';
 import { makeLabelForStandaloneApplication, setDefaultBaseImage, setDefaultConfiguration } from '../../../../lib/buildPacks/common';
@@ -14,6 +15,7 @@ import { checkContainer, formatLabelsOnDocker, isContainerExited, removeContaine
 import type { FastifyRequest } from 'fastify';
 import type { GetImages, CancelDeployment, CheckDNS, CheckRepository, DeleteApplication, DeleteSecret, DeleteStorage, GetApplicationLogs, GetBuildIdLogs, GetBuildLogs, SaveApplication, SaveApplicationSettings, SaveApplicationSource, SaveDeployKey, SaveDestination, SaveSecret, SaveStorage, DeployApplication, CheckDomain, StopPreviewApplication, RestartPreviewApplication } from './types';
 import { OnlyId } from '../../../../types';
+import path from 'node:path';
 
 function filterObject(obj, callback) {
     return Object.fromEntries(Object.entries(obj).
@@ -1178,22 +1180,38 @@ export async function getBuildLogs(request: FastifyRequest<GetBuildLogs>) {
 
 export async function getBuildIdLogs(request: FastifyRequest<GetBuildIdLogs>) {
     try {
-        const { buildId } = request.params
+        // TODO: Fluentbit could still hold the logs, so we need to check if the logs are done
+        const { buildId, id } = request.params
         let { sequence = 0 } = request.query
         if (typeof sequence !== 'number') {
             sequence = Number(sequence)
         }
-        let logs = await prisma.buildLog.findMany({
+        let file = `/app/logs/${id}_buildlog_${buildId}.csv`
-            where: { buildId, time: { gt: sequence } },
+        if (isDev) {
-            orderBy: { time: 'asc' }
+            file = `${process.cwd()}/../../logs/${id}_buildlog_${buildId}.csv`
-        });
+        }
         const data = await prisma.build.findFirst({ where: { id: buildId } });
         const createdAt = day(data.createdAt).utc();
+        try {
+            await fs.stat(file)
+        } catch (error) {
+            return {
+                logs: [],
+                took: day().diff(createdAt) / 1000,
+                status: data?.status || 'queued'
+            }
+        }
+        let fileLogs = (await fs.readFile(file)).toString()
+        let decryptedLogs = await csv({ noheader: true }).fromString(fileLogs)
+        let logs = decryptedLogs.map(log => {
+            const parsed = {
+                time: log['field1'],
+                line: decrypt(log['field2'] + '","' + log['field3'])
+            }
+            return parsed
+        }).filter(log => log.time > sequence)
         return {
-            logs: logs.map(log => {
+            logs,
-                log.time = Number(log.time)
-                return log
-            }),
             took: day().diff(createdAt) / 1000,
             status: data?.status || 'queued'
         }

apps/api/src/routes/api/v1/applications/types.ts

@@ -97,6 +97,7 @@ export interface GetBuildLogs extends OnlyId {
 }
 export interface GetBuildIdLogs {
     Params: {
+        id: string,
         buildId: string
     },
     Querystring: {

apps/ui/src/routes/applications/[id]/logs/_BuildLog.svelte

@@ -11,7 +11,7 @@
 	import LoadingLogs from '$lib/components/LoadingLogs.svelte';
 	import { errorNotification } from '$lib/common';
 	import Tooltip from '$lib/components/Tooltip.svelte';
-
+	import { day } from '$lib/dayjs';
 	let logs: any = [];
 	let currentStatus: any;
 	let streamInterval: any;
@@ -159,7 +159,7 @@
 				bind:this={logsEl}
 			>
 				{#each logs as log}
-					<div>{log.line + '\n'}</div>
+					<div>[{day.unix(log.time).format('HH:mm:ss.SSS')}] {log.line + '\n'}</div>
 				{/each}
 			</div>
 		{:else}

docker-compose-dev.yaml

@@ -1,38 +1,18 @@
 version: '3.8'
 
 services:
-  postgres:
+ fluent-bit:
-    image: postgres:14.5-alpine
+    image: fluent/fluent-bit:1.9.8
-    restart: always
+    container_name: coolify-fluentbit
-    container_name: coolify-pg
+    volumes:
+      - ./logs:/logs
+      - ./others/fluentbit/fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf
+      - ./others/fluentbit/parser.conf:/fluent-bit/etc/parser.conf
+    ports:
+      - "24224:24224"
     networks:
       - coolify-infra
-    command:
-      - "postgres"
-      - "-c"
-      - "log_connections=yes"
-      - "-c"
-      - "max_connections=1337"
-    environment:
-       POSTGRES_DB: coolify
-       POSTGRES_USER: coolify
-       POSTGRES_PASSWORD: coolify
-       POSTGRES_HOST_AUTH_METHOD: "trust"
-    volumes:
-      - 'coolify-pgdb:/var/lib/postgresql/data'
 networks:
   coolify-infra:
     attachable: true
     name: coolify-infra
-
-volumes:
-  coolify-db:
-    name: coolify-db
-  coolify-pgdb:
-    name: coolify-pgdb
-  coolify-ssl-certs:
-    name: coolify-ssl-certs
-  coolify-letsencrypt:
-    name: coolify-letsencrypt
-  coolify-traefik-letsencrypt:
-    name: coolify-traefik-letsencrypt

docker-compose.yaml

@@ -20,28 +20,22 @@ services:
       - '.env'
     networks:
       - coolify-infra
-  # postgres:
+  fluent-bit:
-  #   image: postgres:14.5-alpine
+    image: fluent/fluent-bit:1.9.8
-  #   restart: always
+    container_name: coolify-fluentbit
-  #   container_name: coolify-pg
+    volumes:
-  #   command:
+      - 'coolify-logs:/app/logs'
-  #     - "postgres"
+      - /app/fluentbit/:/fluent-bit/etc/
-  #     - "-c"
+    networks:
-  #     - "log_connections=yes"
+      - coolify-infra
-  #     - "-c"
-  #     - "max_connections=1337"
-  #   volumes:
-  #     - 'coolify-pgdb:/var/lib/postgresql/data'
-  #   ports:
-  #     - target: 5432
-  #       published: 5432
-  #       protocol: tcp
 networks:
   coolify-infra:
     attachable: true
     name: coolify-infra
 
 volumes:
+  coolify-logs:
+    name: coolify-logs
   coolify-db:
     name: coolify-db
   coolify-pgdb:

others/docker-compose-dev.yaml

@@ -1,35 +0,0 @@
-version: '3.8'
-
-services:
-  redis:
-    image: redis:6.2-alpine
-    container_name: coolify-redis
-    networks:
-      - coolify-infra
-    ports:
-      - target: 6379
-        published: 6379
-        protocol: tcp
-        mode: host
-  # fluentbit:
-  #     container_name: coolify-fluentbit
-  #     build:
-  #       context: ./data/fluentd
-  #       dockerfile: Dockerfile-dev
-  #     ports:
-  #       - target: 24224
-  #         published: 24224
-  #         protocol: tcp
-  #         mode: host
-  #       - target: 24224
-  #         published: 24224
-  #         protocol: udp
-  #         mode: host
-  #     networks:
-  #       - coolify-infra
-  #     extra_hosts:
-  #       - 'host.docker.internal:host-gateway'
-networks:
-  coolify-infra:
-    attachable: true
-    name: coolify-infra

others/docker-compose-traefik.yaml

@@ -1,29 +0,0 @@
-version: '3.8'
-
-services:
-  proxy:
-    image: traefik:v2.6
-    command:
-      - --api.insecure=true
-      - --entrypoints.web.address=:80
-      - --entrypoints.websecure.address=:443
-      - --providers.docker=false
-      - --providers.docker.exposedbydefault=false
-      - --providers.http.endpoint=http://host.docker.internal:3000/traefik.json
-      - --providers.http.pollTimeout=5s
-      - --log.level=error
-    ports:
-      - '80:80'
-      - '443:443'
-      - '8080:8080'
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
-    extra_hosts:
-      - 'host.docker.internal:host-gateway'
-    networks:
-      - coolify-infra
-
-networks:
-  coolify-infra:
-    attachable: true
-    name: coolify-infra

others/fluentbit/fluent-bit.conf

@@ -0,0 +1,30 @@
+[SERVICE]
+ Parsers_file /fluent-bit/etc/parser.conf
+ Flush 1
+ Grace 30
+[INPUT]
+ Name http
+ Host 0.0.0.0
+ Port 24224
+[FILTER]
+ Name parser
+ Match *
+ Key_Name log
+ Parser docker
+ Reserve_Data True
+[OUTPUT]
+ Name file
+ Match *
+ Path /logs
+ Mkdir true
+ Format csv
+# [OUTPUT]
+#  Name influxdb 
+#  match *
+#  Host coolify-influxdb
+#  Port 8086
+#  Database coolify
+#  Bucket coolify
+#  Org coolify
+#  HTTP_Token 12345678
+#  Sequence_Tag  _seq

others/fluentbit/parser.conf

@@ -0,0 +1,6 @@
+[PARSER]
+    Name        docker
+    Format      json
+    Time_Key    time
+    Time_Format %Y-%m-%dT%H:%M:%S.%L
+    Time_Keep   On

others/fluentd/Dockerfile-dev

@@ -1,6 +0,0 @@
-FROM fluent/fluent-bit:1.9.0
-COPY fluentbit-dev.conf /tmp/fluentbit.conf
-ENTRYPOINT ["/fluent-bit/bin/fluent-bit", "-c", "/tmp/fluentbit.conf"]
-# USER root
-# RUN ["gem", "install", "fluent-plugin-mongo"]
-# USER fluent

others/fluentd/fluentbit-dev.conf

@@ -1,24 +0,0 @@
-[INPUT]
-    Name              forward
-    Listen            0.0.0.0
-    Port              24224
-    Buffer_Chunk_Size 32KB
-    Buffer_Max_Size   64KB
-
-[OUTPUT]
-    Name          influxdb
-    Match         *
-    Host          coolify-influxdb
-    Port          8086
-    Bucket        containerlogs
-    Org           organization
-    HTTP_Token    supertoken
-    Sequence_Tag  _seq
-    Tag_Keys      container_name
-[OUTPUT]
-    Name  http
-    Match *
-    Host  host.docker.internal
-    Port  3000
-    URI   /logs.json
-    Format json

others/fluentd/fluentd-dev.conf

@@ -1,28 +0,0 @@
-<source>
-  @type forward
-  port 24224
-  bind 0.0.0.0
-</source>
-
-<match **>
-  @type http
-  endpoint http://host.docker.internal:3000/logs.json
-  <buffer>
-    flush_at_shutdown true
-    flush_mode immediate
-    flush_thread_count 8
-    flush_thread_interval 1
-    flush_thread_burst_interval 1
-    retry_forever true
-    retry_type exponential_backoff
-   </buffer>
-</match>
-
-<filter docker.**>
-  @type parser
-  key_name log
-  reserve_data true
-  <parse>
-    @type json
-  </parse>
-</filter>

others/traefik/docker-compose-tcp.yaml

@@ -1,23 +0,0 @@
-version: '3.5'
-
-services:
-  ${ID}:
-    container_name: proxy-for-${PORT}
-    image: traefik:v2.6
-    command:
-      - --api.insecure=true
-      - --entrypoints.web.address=:${PORT}
-      - --providers.docker=false
-      - --providers.docker.exposedbydefault=false
-      - --providers.http.endpoint=http://host.docker.internal:3000/traefik.json?id=${ID}
-      - --providers.http.pollTimeout=5s
-      - --log.level=error
-    ports:
-      - '${PORT}:${PORT}'
-    networks:
-      - ${NETWORK}
-
-networks:
-  net:
-    external: false
-    name: ${NETWORK}

pnpm-lock.yaml

@@ -33,6 +33,8 @@ importers:
       bree: 9.1.2
       cabin: 9.1.2
       compare-versions: 5.0.1
+      csv-parse: ^5.3.0
+      csvtojson: ^2.0.10
       cuid: 2.1.8
       dayjs: 1.11.5
       dockerode: 3.3.4
@@ -65,7 +67,7 @@ importers:
       typescript: 4.8.2
       unique-names-generator: 4.7.1
     dependencies:
-      '@breejs/ts-worker': 2.0.0_d3un4r7p64mpe4ydkpns6lvpxy
+      '@breejs/ts-worker': 2.0.0_zx7xfusupi724hd5vcuaoj6jni
       '@fastify/autoload': 5.3.1
       '@fastify/cookie': 8.1.0
       '@fastify/cors': 8.1.0
@@ -80,6 +82,8 @@ importers:
       bree: 9.1.2
       cabin: 9.1.2
       compare-versions: 5.0.1
+      csv-parse: 5.3.0
+      csvtojson: 2.0.10
       cuid: 2.1.8
       dayjs: 1.11.5
       dockerode: 3.3.4
@@ -241,11 +245,12 @@ packages:
     engines: {node: '>= 10'}
     dev: false
 
-  /@breejs/ts-worker/2.0.0_d3un4r7p64mpe4ydkpns6lvpxy:
+  /@breejs/ts-worker/2.0.0_zx7xfusupi724hd5vcuaoj6jni:
     resolution: {integrity: sha512-6anHRcmgYlF7mrm/YVRn6rx2cegLuiY3VBxkkimOTWC/dVQeH336imVSuIKEGKTwiuNTPr2hswVdDSneNuXg3A==}
     engines: {node: '>= 12.11'}
     peerDependencies:
       bree: '>=9.0.0'
+      tsconfig-paths: '>= 4'
     dependencies:
       bree: 9.1.2
       ts-node: 10.8.2_r4hqq7vrw4pxsipnb7ha25ylfe
@@ -1870,6 +1875,10 @@ packages:
       readable-stream: 3.6.0
     dev: false
 
+  /bluebird/3.7.2:
+    resolution: {integrity: sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==}
+    dev: false
+
   /bn.js/4.12.0:
     resolution: {integrity: sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==}
     dev: false
@@ -2292,6 +2301,20 @@ packages:
     engines: {node: '>=4'}
     hasBin: true
 
+  /csv-parse/5.3.0:
+    resolution: {integrity: sha512-UXJCGwvJ2fep39purtAn27OUYmxB1JQto+zhZ4QlJpzsirtSFbzLvip1aIgziqNdZp/TptvsKEV5BZSxe10/DQ==}
+    dev: false
+
+  /csvtojson/2.0.10:
+    resolution: {integrity: sha512-lUWFxGKyhraKCW8Qghz6Z0f2l/PqB1W3AO0HKJzGIQ5JRSlR651ekJDiGJbBT4sRNNv5ddnSGVEnsxP9XRCVpQ==}
+    engines: {node: '>=4.0.0'}
+    hasBin: true
+    dependencies:
+      bluebird: 3.7.2
+      lodash: 4.17.21
+      strip-bom: 2.0.0
+    dev: false
+
   /cuid/2.1.8:
     resolution: {integrity: sha512-xiEMER6E7TlTPnDxrM4eRiC6TRgjNX9xzEZ5U/Se2YJKr7Mq4pJn/2XEHjl3STcSh96GmkHPcBXLES8M29wyyg==}
     dev: false
@@ -3899,6 +3922,10 @@ packages:
       has-symbols: 1.0.3
     dev: true
 
+  /is-utf8/0.2.1:
+    resolution: {integrity: sha512-rMYPYvCzsXywIsldgLaSoPlw5PfoB/ssr7hY4pLfcodrA5M/eArza1a9VmTiNIBNMjOGr1Ow9mTyU2o69U6U9Q==}
+    dev: false
+
   /is-uuid/1.0.2:
     resolution: {integrity: sha512-tCByphFcJgf2qmiMo5hMCgNAquNSagOetVetDvBXswGkNfoyEMvGH1yDlF8cbZbKnbVBr4Y5/rlpMz9umxyBkQ==}
     dev: false
@@ -5595,6 +5622,13 @@ packages:
       ansi-regex: 6.0.1
     dev: false
 
+  /strip-bom/2.0.0:
+    resolution: {integrity: sha512-kwrX1y7czp1E69n2ajbG65mIo9dqvJ+8aBQXOGVxqwvNbsXdFM6Lq37dLAY3mknUwru8CfcCbfOLL/gMo+fi3g==}
+    engines: {node: '>=0.10.0'}
+    dependencies:
+      is-utf8: 0.2.1
+    dev: false
+
   /strip-bom/3.0.0:
     resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==}
     engines: {node: '>=4'}