Shiloh/lasthourcloud
Shiloh/lasthourcloud
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 4 Wiki Activity

feat: fluentbit

Browse Source
This commit is contained in:
Andras Bacsai 2022-09-15 09:34:39 +02:00
parent 0940309600
commit 69891a64a0
18 changed files with 145 additions and 210 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -13,3 +13,4 @@ apps/api/db/*.db
local-serve
apps/api/db/migration.db-journal
apps/api/core*
logs

1
Dockerfile
View File

@ -33,6 +33,7 @@ RUN chmod +x ~/.docker/cli-plugins/docker-compose /usr/bin/docker
RUN (curl -sSL "https://github.com/buildpacks/pack/releases/download/v0.27.0/pack-v0.27.0-linux.tgz" | tar -C /usr/local/bin/ --no-same-owner -xzv pack)
COPY --from=build /app/apps/api/build/ .
COPY --from=build /app/others/fluentbit/ ./fluentbit
COPY --from=build /app/apps/ui/build/ ./public
COPY --from=build /app/apps/api/prisma/ ./prisma
COPY --from=build /app/apps/api/package.json .

2
apps/api/package.json
View File

@ -29,6 +29,8 @@
"bree": "9.1.2",
"cabin": "9.1.2",
"compare-versions": "5.0.1",
"csv-parse": "^5.3.0",
"csvtojson": "^2.0.10",
"cuid": "2.1.8",
"dayjs": "1.11.5",
"dockerode": "3.3.4",

25
apps/api/src/lib/buildPacks/common.ts
View File

@ -1,4 +1,4 @@
import { base64Encode, executeDockerCmd, generateTimestamp, getDomain, isDev, prisma, version } from "../common";
import { base64Encode, encrypt, executeDockerCmd, generateTimestamp, getDomain, isDev, prisma, version } from "../common";
import { promises as fs } from 'fs';
import { day } from "../dayjs";
@ -461,17 +461,30 @@ export const saveBuildLog = async ({
buildId: string;
applicationId: string;
}): Promise<any> => {
const { default: got } = await import('got')
if (line && typeof line === 'string' && line.includes('ghs_')) {
const regex = /ghs_.*@/g;
line = line.replace(regex, '<SENSITIVE_DATA_DELETED>@');
}
const addTimestamp = `[${generateTimestamp()}] ${line}`;
if (isDev) console.debug(`[${applicationId}] ${addTimestamp}`);
return await prisma.buildLog.create({
data: {
line: addTimestamp, buildId, time: Number(day().valueOf()), applicationId
const fluentBitUrl = isDev ? 'http://localhost:24224' : 'http://coolify-fluentbit:24224';
if (isDev) {
console.debug(`[${applicationId}] ${addTimestamp}`);
}
});
// return await prisma.buildLog.create({
// data: {
// line: addTimestamp, buildId, time: Number(day().valueOf()), applicationId
// }
// });
return await got.post(`${fluentBitUrl}/${applicationId}_buildlog_${buildId}.csv`, {
json: {
line: encrypt(line)
}
})
};
export async function copyBaseConfigurationFiles(

36
apps/api/src/routes/api/v1/applications/handlers.ts
View File

@ -5,6 +5,7 @@ import axios from 'axios';
import { FastifyReply } from 'fastify';
import fs from 'fs/promises';
import yaml from 'js-yaml';
import csv from 'csvtojson';
import { day } from '../../../../lib/dayjs';
import { makeLabelForStandaloneApplication, setDefaultBaseImage, setDefaultConfiguration } from '../../../../lib/buildPacks/common';
@ -14,6 +15,7 @@ import { checkContainer, formatLabelsOnDocker, isContainerExited, removeContaine
import type { FastifyRequest } from 'fastify';
import type { GetImages, CancelDeployment, CheckDNS, CheckRepository, DeleteApplication, DeleteSecret, DeleteStorage, GetApplicationLogs, GetBuildIdLogs, GetBuildLogs, SaveApplication, SaveApplicationSettings, SaveApplicationSource, SaveDeployKey, SaveDestination, SaveSecret, SaveStorage, DeployApplication, CheckDomain, StopPreviewApplication, RestartPreviewApplication } from './types';
import { OnlyId } from '../../../../types';
import path from 'node:path';
function filterObject(obj, callback) {
return Object.fromEntries(Object.entries(obj).
@ -1178,22 +1180,38 @@ export async function getBuildLogs(request: FastifyRequest<GetBuildLogs>) {
export async function getBuildIdLogs(request: FastifyRequest<GetBuildIdLogs>) {
try {
const { buildId } = request.params
// TODO: Fluentbit could still hold the logs, so we need to check if the logs are done
const { buildId, id } = request.params
let { sequence = 0 } = request.query
if (typeof sequence !== 'number') {
sequence = Number(sequence)
}
let logs = await prisma.buildLog.findMany({
where: { buildId, time: { gt: sequence } },
orderBy: { time: 'asc' }
});
let file = `/app/logs/${id}_buildlog_${buildId}.csv`
if (isDev) {
file = `${process.cwd()}/../../logs/${id}_buildlog_${buildId}.csv`
}
const data = await prisma.build.findFirst({ where: { id: buildId } });
const createdAt = day(data.createdAt).utc();
try {
await fs.stat(file)
} catch (error) {
return {
logs: logs.map(log => {
log.time = Number(log.time)
return log
}),
logs: [],
took: day().diff(createdAt) / 1000,
status: data?.status || 'queued'
}
}
let fileLogs = (await fs.readFile(file)).toString()
let decryptedLogs = await csv({ noheader: true }).fromString(fileLogs)
let logs = decryptedLogs.map(log => {
const parsed = {
time: log['field1'],
line: decrypt(log['field2'] + '","' + log['field3'])
}
return parsed
}).filter(log => log.time > sequence)
return {
logs,
took: day().diff(createdAt) / 1000,
status: data?.status || 'queued'
}

1
apps/api/src/routes/api/v1/applications/types.ts
View File

@ -97,6 +97,7 @@ export interface GetBuildLogs extends OnlyId {
}
export interface GetBuildIdLogs {
Params: {
id: string,
buildId: string
},
Querystring: {

4
apps/ui/src/routes/applications/[id]/logs/_BuildLog.svelte
View File

@ -11,7 +11,7 @@
import LoadingLogs from '$lib/components/LoadingLogs.svelte';
import { errorNotification } from '$lib/common';
import Tooltip from '$lib/components/Tooltip.svelte';
import { day } from '$lib/dayjs';
let logs: any = [];
let currentStatus: any;
let streamInterval: any;
@ -159,7 +159,7 @@
bind:this={logsEl}
>
{#each logs as log}
<div>{log.line + '\n'}</div>
<div>[{day.unix(log.time).format('HH:mm:ss.SSS')}] {log.line + '\n'}</div>
{/each}
</div>
{:else}

38
docker-compose-dev.yaml
View File

@ -1,38 +1,18 @@
version: '3.8'
services:
postgres:
image: postgres:14.5-alpine
restart: always
container_name: coolify-pg
fluent-bit:
image: fluent/fluent-bit:1.9.8
container_name: coolify-fluentbit
volumes:
- ./logs:/logs
- ./others/fluentbit/fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf
- ./others/fluentbit/parser.conf:/fluent-bit/etc/parser.conf
ports:
- "24224:24224"
networks:
- coolify-infra
command:
- "postgres"
- "-c"
- "log_connections=yes"
- "-c"
- "max_connections=1337"
environment:
POSTGRES_DB: coolify
POSTGRES_USER: coolify
POSTGRES_PASSWORD: coolify
POSTGRES_HOST_AUTH_METHOD: "trust"
volumes:
- 'coolify-pgdb:/var/lib/postgresql/data'
networks:
coolify-infra:
attachable: true
name: coolify-infra
volumes:
coolify-db:
name: coolify-db
coolify-pgdb:
name: coolify-pgdb
coolify-ssl-certs:
name: coolify-ssl-certs
coolify-letsencrypt:
name: coolify-letsencrypt
coolify-traefik-letsencrypt:
name: coolify-traefik-letsencrypt

26
docker-compose.yaml
View File

@ -20,28 +20,22 @@ services:
- '.env'
networks:
- coolify-infra
# postgres:
# image: postgres:14.5-alpine
# restart: always
# container_name: coolify-pg
# command:
# - "postgres"
# - "-c"
# - "log_connections=yes"
# - "-c"
# - "max_connections=1337"
# volumes:
# - 'coolify-pgdb:/var/lib/postgresql/data'
# ports:
# - target: 5432
# published: 5432
# protocol: tcp
fluent-bit:
image: fluent/fluent-bit:1.9.8
container_name: coolify-fluentbit
volumes:
- 'coolify-logs:/app/logs'
- /app/fluentbit/:/fluent-bit/etc/
networks:
- coolify-infra
networks:
coolify-infra:
attachable: true
name: coolify-infra
volumes:
coolify-logs:
name: coolify-logs
coolify-db:
name: coolify-db
coolify-pgdb:

35
others/docker-compose-dev.yaml
View File

@ -1,35 +0,0 @@
version: '3.8'
services:
redis:
image: redis:6.2-alpine
container_name: coolify-redis
networks:
- coolify-infra
ports:
- target: 6379
published: 6379
protocol: tcp
mode: host
# fluentbit:
# container_name: coolify-fluentbit
# build:
# context: ./data/fluentd
# dockerfile: Dockerfile-dev
# ports:
# - target: 24224
# published: 24224
# protocol: tcp
# mode: host
# - target: 24224
# published: 24224
# protocol: udp
# mode: host
# networks:
# - coolify-infra
# extra_hosts:
# - 'host.docker.internal:host-gateway'
networks:
coolify-infra:
attachable: true
name: coolify-infra

29
others/docker-compose-traefik.yaml
View File

@ -1,29 +0,0 @@
version: '3.8'
services:
proxy:
image: traefik:v2.6
command:
- --api.insecure=true
- --entrypoints.web.address=:80
- --entrypoints.websecure.address=:443
- --providers.docker=false
- --providers.docker.exposedbydefault=false
- --providers.http.endpoint=http://host.docker.internal:3000/traefik.json
- --providers.http.pollTimeout=5s
- --log.level=error
ports:
- '80:80'
- '443:443'
- '8080:8080'
volumes:
- /var/run/docker.sock:/var/run/docker.sock
extra_hosts:
- 'host.docker.internal:host-gateway'
networks:
- coolify-infra
networks:
coolify-infra:
attachable: true
name: coolify-infra

30
others/fluentbit/fluent-bit.conf Normal file
View File

@ -0,0 +1,30 @@
[SERVICE]
Parsers_file /fluent-bit/etc/parser.conf
Flush 1
Grace 30
[INPUT]
Name http
Host 0.0.0.0
Port 24224
[FILTER]
Name parser
Match *
Key_Name log
Parser docker
Reserve_Data True
[OUTPUT]
Name file
Match *
Path /logs
Mkdir true
Format csv
# [OUTPUT]
# Name influxdb
# match *
# Host coolify-influxdb
# Port 8086
# Database coolify
# Bucket coolify
# Org coolify
# HTTP_Token 12345678
# Sequence_Tag _seq

6
others/fluentbit/parser.conf Normal file
View File

@ -0,0 +1,6 @@
[PARSER]
Name docker
Format json
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L
Time_Keep On

6
others/fluentd/Dockerfile-dev
View File

@ -1,6 +0,0 @@
FROM fluent/fluent-bit:1.9.0
COPY fluentbit-dev.conf /tmp/fluentbit.conf
ENTRYPOINT ["/fluent-bit/bin/fluent-bit", "-c", "/tmp/fluentbit.conf"]
# USER root
# RUN ["gem", "install", "fluent-plugin-mongo"]
# USER fluent

24
others/fluentd/fluentbit-dev.conf
View File

@ -1,24 +0,0 @@
[INPUT]
Name forward
Listen 0.0.0.0
Port 24224
Buffer_Chunk_Size 32KB
Buffer_Max_Size 64KB
[OUTPUT]
Name influxdb
Match *
Host coolify-influxdb
Port 8086
Bucket containerlogs
Org organization
HTTP_Token supertoken
Sequence_Tag _seq
Tag_Keys container_name
[OUTPUT]
Name http
Match *
Host host.docker.internal
Port 3000
URI /logs.json
Format json

28
others/fluentd/fluentd-dev.conf
View File

@ -1,28 +0,0 @@
<source>
@type forward
port 24224
bind 0.0.0.0
</source>
<match **>
@type http
endpoint http://host.docker.internal:3000/logs.json
<buffer>
flush_at_shutdown true
flush_mode immediate
flush_thread_count 8
flush_thread_interval 1
flush_thread_burst_interval 1
retry_forever true
retry_type exponential_backoff
</buffer>
</match>
<filter docker.**>
@type parser
key_name log
reserve_data true
<parse>
@type json
</parse>
</filter>

23
others/traefik/docker-compose-tcp.yaml
View File

@ -1,23 +0,0 @@
version: '3.5'
services:
${ID}:
container_name: proxy-for-${PORT}
image: traefik:v2.6
command:
- --api.insecure=true
- --entrypoints.web.address=:${PORT}
- --providers.docker=false
- --providers.docker.exposedbydefault=false
- --providers.http.endpoint=http://host.docker.internal:3000/traefik.json?id=${ID}
- --providers.http.pollTimeout=5s
- --log.level=error
ports:
- '${PORT}:${PORT}'
networks:
- ${NETWORK}
networks:
net:
external: false
name: ${NETWORK}

38
pnpm-lock.yaml
View File

@ -33,6 +33,8 @@ importers:
bree: 9.1.2
cabin: 9.1.2
compare-versions: 5.0.1
csv-parse: ^5.3.0
csvtojson: ^2.0.10
cuid: 2.1.8
dayjs: 1.11.5
dockerode: 3.3.4
@ -65,7 +67,7 @@ importers:
typescript: 4.8.2
unique-names-generator: 4.7.1
dependencies:
'@breejs/ts-worker': 2.0.0_d3un4r7p64mpe4ydkpns6lvpxy
'@breejs/ts-worker': 2.0.0_zx7xfusupi724hd5vcuaoj6jni
'@fastify/autoload': 5.3.1
'@fastify/cookie': 8.1.0
'@fastify/cors': 8.1.0
@ -80,6 +82,8 @@ importers:
bree: 9.1.2
cabin: 9.1.2
compare-versions: 5.0.1
csv-parse: 5.3.0
csvtojson: 2.0.10
cuid: 2.1.8
dayjs: 1.11.5
dockerode: 3.3.4
@ -241,11 +245,12 @@ packages:
engines: {node: '>= 10'}
dev: false
/@breejs/ts-worker/2.0.0_d3un4r7p64mpe4ydkpns6lvpxy:
/@breejs/ts-worker/2.0.0_zx7xfusupi724hd5vcuaoj6jni:
resolution: {integrity: sha512-6anHRcmgYlF7mrm/YVRn6rx2cegLuiY3VBxkkimOTWC/dVQeH336imVSuIKEGKTwiuNTPr2hswVdDSneNuXg3A==}
engines: {node: '>= 12.11'}
peerDependencies:
bree: '>=9.0.0'
tsconfig-paths: '>= 4'
dependencies:
bree: 9.1.2
ts-node: 10.8.2_r4hqq7vrw4pxsipnb7ha25ylfe
@ -1870,6 +1875,10 @@ packages:
readable-stream: 3.6.0
dev: false
/bluebird/3.7.2:
resolution: {integrity: sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==}
dev: false
/bn.js/4.12.0:
resolution: {integrity: sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==}
dev: false
@ -2292,6 +2301,20 @@ packages:
engines: {node: '>=4'}
hasBin: true
/csv-parse/5.3.0:
resolution: {integrity: sha512-UXJCGwvJ2fep39purtAn27OUYmxB1JQto+zhZ4QlJpzsirtSFbzLvip1aIgziqNdZp/TptvsKEV5BZSxe10/DQ==}
dev: false
/csvtojson/2.0.10:
resolution: {integrity: sha512-lUWFxGKyhraKCW8Qghz6Z0f2l/PqB1W3AO0HKJzGIQ5JRSlR651ekJDiGJbBT4sRNNv5ddnSGVEnsxP9XRCVpQ==}
engines: {node: '>=4.0.0'}
hasBin: true
dependencies:
bluebird: 3.7.2
lodash: 4.17.21
strip-bom: 2.0.0
dev: false
/cuid/2.1.8:
resolution: {integrity: sha512-xiEMER6E7TlTPnDxrM4eRiC6TRgjNX9xzEZ5U/Se2YJKr7Mq4pJn/2XEHjl3STcSh96GmkHPcBXLES8M29wyyg==}
dev: false
@ -3899,6 +3922,10 @@ packages:
has-symbols: 1.0.3
dev: true
/is-utf8/0.2.1:
resolution: {integrity: sha512-rMYPYvCzsXywIsldgLaSoPlw5PfoB/ssr7hY4pLfcodrA5M/eArza1a9VmTiNIBNMjOGr1Ow9mTyU2o69U6U9Q==}
dev: false
/is-uuid/1.0.2:
resolution: {integrity: sha512-tCByphFcJgf2qmiMo5hMCgNAquNSagOetVetDvBXswGkNfoyEMvGH1yDlF8cbZbKnbVBr4Y5/rlpMz9umxyBkQ==}
dev: false
@ -5595,6 +5622,13 @@ packages:
ansi-regex: 6.0.1
dev: false
/strip-bom/2.0.0:
resolution: {integrity: sha512-kwrX1y7czp1E69n2ajbG65mIo9dqvJ+8aBQXOGVxqwvNbsXdFM6Lq37dLAY3mknUwru8CfcCbfOLL/gMo+fi3g==}
engines: {node: '>=0.10.0'}
dependencies:
is-utf8: 0.2.1
dev: false
/strip-bom/3.0.0:
resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==}
engines: {node: '>=4'}