diff --git a/data/fluentd/Dockerfile-dev b/data/fluentd/Dockerfile-dev new file mode 100644 index 000000000..bb5524df9 --- /dev/null +++ b/data/fluentd/Dockerfile-dev @@ -0,0 +1,6 @@ +FROM fluent/fluent-bit:1.9.0 +COPY fluentbit-dev.conf /tmp/fluentbit.conf +ENTRYPOINT ["/fluent-bit/bin/fluent-bit", "-c", "/tmp/fluentbit.conf"] +# USER root +# RUN ["gem", "install", "fluent-plugin-mongo"] +# USER fluent \ No newline at end of file diff --git a/data/fluentd/fluentbit-dev.conf b/data/fluentd/fluentbit-dev.conf new file mode 100644 index 000000000..8d06fd0af --- /dev/null +++ b/data/fluentd/fluentbit-dev.conf @@ -0,0 +1,24 @@ +[INPUT] + Name forward + Listen 0.0.0.0 + Port 24224 + Buffer_Chunk_Size 32KB + Buffer_Max_Size 64KB + +[OUTPUT] + Name influxdb + Match * + Host coolify-influxdb + Port 8086 + Bucket containerlogs + Org organization + HTTP_Token supertoken + Sequence_Tag _seq + Tag_Keys container_name +[OUTPUT] + Name http + Match * + Host host.docker.internal + Port 3000 + URI /logs.json + Format json \ No newline at end of file diff --git a/data/fluentd/fluentd-dev.conf b/data/fluentd/fluentd-dev.conf new file mode 100644 index 000000000..9ae4a06c0 --- /dev/null +++ b/data/fluentd/fluentd-dev.conf @@ -0,0 +1,28 @@ + + @type forward + port 24224 + bind 0.0.0.0 + + + + @type http + endpoint http://host.docker.internal:3000/logs.json + + flush_at_shutdown true + flush_mode immediate + flush_thread_count 8 + flush_thread_interval 1 + flush_thread_burst_interval 1 + retry_forever true + retry_type exponential_backoff + + + + + @type parser + key_name log + reserve_data true + + @type json + + \ No newline at end of file diff --git a/docker-compose-dev.yaml b/docker-compose-dev.yaml index 195854327..6c89e87d5 100644 --- a/docker-compose-dev.yaml +++ b/docker-compose-dev.yaml @@ -11,7 +11,24 @@ services: published: 6379 protocol: tcp mode: host - + # fluentbit: + # container_name: coolify-fluentbit + # build: + # context: ./data/fluentd + # dockerfile: Dockerfile-dev + # ports: + # - target: 24224 + # published: 24224 + # protocol: tcp + # mode: host + # - target: 24224 + # published: 24224 + # protocol: udp + # mode: host + # networks: + # - coolify-infra + # extra_hosts: + # - 'host.docker.internal:host-gateway' networks: coolify-infra: attachable: true diff --git a/package.json b/package.json index 4cfc61406..bb3348d88 100644 --- a/package.json +++ b/package.json @@ -1,10 +1,10 @@ { "name": "coolify", "description": "An open-source & self-hostable Heroku / Netlify alternative.", - "version": "2.4.10", + "version": "2.4.11", "license": "AGPL-3.0", "scripts": { - "dev": "docker-compose -f docker-compose-dev.yaml up -d && cross-env NODE_ENV=development & svelte-kit dev", + "dev": "docker-compose -f docker-compose-dev.yaml up -d && cross-env NODE_ENV=development & svelte-kit dev --host 0.0.0.0", "dev:stop": "docker-compose -f docker-compose-dev.yaml down", "dev:logs": "docker-compose -f docker-compose-dev.yaml logs -f --tail 10", "studio": "npx prisma studio", @@ -63,7 +63,7 @@ "@iarna/toml": "2.2.5", "@prisma/client": "3.11.1", "@sentry/node": "6.19.6", - "bcryptjs": "^2.4.3", + "bcryptjs": "2.4.3", "bullmq": "1.80.0", "compare-versions": "4.1.3", "cookie": "0.4.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 66d7c480f..99b37c004 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -5,7 +5,7 @@ specifiers: '@prisma/client': 3.11.1 '@sentry/node': 6.19.6 '@sveltejs/adapter-node': 1.0.0-next.73 - '@sveltejs/kit': 1.0.0-next.303 + '@sveltejs/kit': 1.0.0-next.310 '@types/js-cookie': 3.0.1 '@types/js-yaml': 4.0.5 '@types/node': 17.0.23 @@ -14,8 +14,8 @@ specifiers: '@typescript-eslint/parser': 4.31.1 '@zerodevx/svelte-toast': 0.7.1 autoprefixer: 10.4.4 - bcryptjs: ^2.4.3 - bullmq: 1.78.1 + bcryptjs: 2.4.3 + bullmq: 1.80.0 compare-versions: 4.1.3 cookie: 0.4.2 cross-env: 7.0.3 @@ -60,7 +60,7 @@ dependencies: '@prisma/client': 3.11.1_prisma@3.11.1 '@sentry/node': 6.19.6 bcryptjs: 2.4.3 - bullmq: 1.78.1 + bullmq: 1.80.0 compare-versions: 4.1.3 cookie: 0.4.2 cuid: 2.1.8 @@ -82,7 +82,7 @@ dependencies: devDependencies: '@sveltejs/adapter-node': 1.0.0-next.73 - '@sveltejs/kit': 1.0.0-next.303_svelte@3.47.0 + '@sveltejs/kit': 1.0.0-next.310_svelte@3.47.0 '@types/js-cookie': 3.0.1 '@types/js-yaml': 4.0.5 '@types/node': 17.0.23 @@ -374,10 +374,10 @@ packages: tiny-glob: 0.2.9 dev: true - /@sveltejs/kit/1.0.0-next.303_svelte@3.47.0: + /@sveltejs/kit/1.0.0-next.310_svelte@3.47.0: resolution: { - integrity: sha512-WdxDc8OiF1WEd/bEza7CBdzA+3qIcCi1GKBj/gieKX9I3N8iDJt/Cg2POrLo9wQoJ47nZcAd1eOhfr7XEX1aIQ== + integrity: sha512-pTyMyaoyHS+V5cQZIQMfQXmLkhw1VaRwT9avOSgwDc0QBpnNw2LdzwoPYsUr96ca5B6cfT3SMUNolxErTNHmPQ== } engines: { node: '>=14.13' } hasBin: true @@ -1669,10 +1669,10 @@ packages: ieee754: 1.2.1 dev: false - /bullmq/1.78.1: + /bullmq/1.80.0: resolution: { - integrity: sha512-er45mM8nGhgA83EVCJ4PNxPyDSzakvoxeFGU4vdSgYeB+SbeFQAlJYmAC50Ms7YFPstm1LeinbVZ+oX/BmBzOg== + integrity: sha512-oz7GZIg7gAGIIlLQ3KdpYSA5WSz5205pQHyGwOtQof9MmkOf+Kmo6sxqr+BiQrjhFOrB6JLSCqS3EGEbMA34MA== } dependencies: cron-parser: 4.2.1 diff --git a/prisma/migrations/20220418214843_persistent_storage_services/migration.sql b/prisma/migrations/20220418214843_persistent_storage_services/migration.sql new file mode 100644 index 000000000..f85fd31df --- /dev/null +++ b/prisma/migrations/20220418214843_persistent_storage_services/migration.sql @@ -0,0 +1,12 @@ +-- CreateTable +CREATE TABLE "ServicePersistentStorage" ( + "id" TEXT NOT NULL PRIMARY KEY, + "serviceId" TEXT NOT NULL, + "path" TEXT NOT NULL, + "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" DATETIME NOT NULL, + CONSTRAINT "ServicePersistentStorage_serviceId_fkey" FOREIGN KEY ("serviceId") REFERENCES "Service" ("id") ON DELETE RESTRICT ON UPDATE CASCADE +); + +-- CreateIndex +CREATE UNIQUE INDEX "ServicePersistentStorage_serviceId_path_key" ON "ServicePersistentStorage"("serviceId", "path"); diff --git a/prisma/migrations/20220419203408_multiply_dockerfile_locations/migration.sql b/prisma/migrations/20220419203408_multiply_dockerfile_locations/migration.sql new file mode 100644 index 000000000..ce32f0844 --- /dev/null +++ b/prisma/migrations/20220419203408_multiply_dockerfile_locations/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "Application" ADD COLUMN "dockerFileLocation" TEXT; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 928562abd..668f4588a 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -1,6 +1,6 @@ generator client { - provider = "prisma-client-js" - binaryTargets = ["linux-musl"] + provider = "prisma-client-js" + binaryTargets = ["native", "linux-musl"] } datasource db { @@ -91,6 +91,7 @@ model Application { pythonWSGI String? pythonModule String? pythonVariable String? + dockerFileLocation String? createdAt DateTime @default(now()) updatedAt DateTime @updatedAt settings ApplicationSettings? @@ -118,14 +119,25 @@ model ApplicationSettings { model ApplicationPersistentStorage { id String @id @default(cuid()) application Application @relation(fields: [applicationId], references: [id]) - applicationId String - path String + applicationId String + path String createdAt DateTime @default(now()) updatedAt DateTime @updatedAt @@unique([applicationId, path]) } +model ServicePersistentStorage { + id String @id @default(cuid()) + service Service @relation(fields: [serviceId], references: [id]) + serviceId String + path String + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@unique([serviceId, path]) +} + model Secret { id String @id @default(cuid()) name String @@ -267,17 +279,17 @@ model DatabaseSettings { } model Service { - id String @id @default(cuid()) + id String @id @default(cuid()) name String fqdn String? - dualCerts Boolean @default(false) + dualCerts Boolean @default(false) type String? version String? teams Team[] destinationDockerId String? - destinationDocker DestinationDocker? @relation(fields: [destinationDockerId], references: [id]) - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt + destinationDocker DestinationDocker? @relation(fields: [destinationDockerId], references: [id]) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt plausibleAnalytics PlausibleAnalytics? minio Minio? vscodeserver Vscodeserver? @@ -285,6 +297,7 @@ model Service { ghost Ghost? serviceSecret ServiceSecret[] meiliSearch MeiliSearch? + persistentStorage ServicePersistentStorage[] } model PlausibleAnalytics { diff --git a/src/app.d.ts b/src/app.d.ts index d432aef4a..37f419edf 100644 --- a/src/app.d.ts +++ b/src/app.d.ts @@ -6,7 +6,11 @@ declare namespace App { cookies: Record; } interface Platform {} - interface Session extends SessionData {} + interface Session extends SessionData { + whiteLabelDetails: { + icon: string | null; + }; + } interface Stuff { service: any; application: any; diff --git a/src/hooks.ts b/src/hooks.ts index c72c7fd6c..fef98375f 100644 --- a/src/hooks.ts +++ b/src/hooks.ts @@ -8,6 +8,9 @@ import cookie from 'cookie'; import { dev } from '$app/env'; const whiteLabeled = process.env['COOLIFY_WHITE_LABELED'] === 'true'; +const whiteLabelDetails = { + icon: (whiteLabeled && process.env['COOLIFY_WHITE_LABELED_ICON']) || null +}; export const handle = handleSession( { @@ -74,6 +77,7 @@ export const getSession: GetSession = function ({ locals }) { return { version, whiteLabeled, + whiteLabelDetails, ...locals.session.data }; }; diff --git a/src/lib/buildPacks/common.ts b/src/lib/buildPacks/common.ts index 65f4d6903..e6f62bd23 100644 --- a/src/lib/buildPacks/common.ts +++ b/src/lib/buildPacks/common.ts @@ -91,7 +91,8 @@ export const setDefaultConfiguration = async (data) => { startCommand, buildCommand, publishDirectory, - baseDirectory + baseDirectory, + dockerFileLocation } = data; const template = scanningTemplates[buildPack]; if (!port) { @@ -110,6 +111,12 @@ export const setDefaultConfiguration = async (data) => { if (!baseDirectory.startsWith('/')) baseDirectory = `/${baseDirectory}`; if (!baseDirectory.endsWith('/')) baseDirectory = `${baseDirectory}/`; } + if (dockerFileLocation) { + if (!dockerFileLocation.startsWith('/')) dockerFileLocation = `/${dockerFileLocation}`; + if (dockerFileLocation.endsWith('/')) dockerFileLocation = dockerFileLocation.slice(0, -1); + } else { + dockerFileLocation = '/Dockerfile'; + } return { buildPack, @@ -118,7 +125,8 @@ export const setDefaultConfiguration = async (data) => { startCommand, buildCommand, publishDirectory, - baseDirectory + baseDirectory, + dockerFileLocation }; }; diff --git a/src/lib/buildPacks/docker.ts b/src/lib/buildPacks/docker.ts index 2db5d567c..1ddf81a60 100644 --- a/src/lib/buildPacks/docker.ts +++ b/src/lib/buildPacks/docker.ts @@ -10,15 +10,16 @@ export default async function ({ buildId, baseDirectory, secrets, - pullmergeRequestId + pullmergeRequestId, + dockerFileLocation }) { try { - let file = `${workdir}/Dockerfile`; + const file = `${workdir}${dockerFileLocation}`; + let dockerFileOut = `${workdir}`; if (baseDirectory) { - file = `${workdir}/${baseDirectory}/Dockerfile`; - workdir = `${workdir}/${baseDirectory}`; + dockerFileOut = `${workdir}${baseDirectory}`; + workdir = `${workdir}${baseDirectory}`; } - const Dockerfile: Array = (await fs.readFile(`${file}`, 'utf8')) .toString() .trim() @@ -41,8 +42,8 @@ export default async function ({ } }); } - await fs.writeFile(`${file}`, Dockerfile.join('\n')); - await buildImage({ applicationId, tag, workdir, docker, buildId, debug }); + await fs.writeFile(`${dockerFileOut}${dockerFileLocation}`, Dockerfile.join('\n')); + await buildImage({ applicationId, tag, workdir, docker, buildId, debug, dockerFileLocation }); } catch (error) { throw error; } diff --git a/src/lib/common.ts b/src/lib/common.ts index 1b5754917..8bcc48b99 100644 --- a/src/lib/common.ts +++ b/src/lib/common.ts @@ -61,14 +61,12 @@ export const saveBuildLog = async ({ buildId: string; applicationId: string; }): Promise => { - if (line) { - if (line.includes('ghs_')) { - const regex = /ghs_.*@/g; - line = line.replace(regex, '@'); - } - const addTimestamp = `${generateTimestamp()} ${line}`; - return await buildLogQueue.add(buildId, { buildId, line: addTimestamp, applicationId }); + if (line && typeof line === 'string' && line.includes('ghs_')) { + const regex = /ghs_.*@/g; + line = line.replace(regex, '@'); } + const addTimestamp = `${generateTimestamp()} ${line}`; + return await buildLogQueue.add(buildId, { buildId, line: addTimestamp, applicationId }); }; export const getTeam = (event: RequestEvent): string | null => { diff --git a/src/lib/database/applications.ts b/src/lib/database/applications.ts index 3ce6286d7..020b9ba75 100644 --- a/src/lib/database/applications.ts +++ b/src/lib/database/applications.ts @@ -263,7 +263,8 @@ export async function configureApplication({ publishDirectory, pythonWSGI, pythonModule, - pythonVariable + pythonVariable, + dockerFileLocation }: { id: string; buildPack: string; @@ -278,6 +279,7 @@ export async function configureApplication({ pythonWSGI: string; pythonModule: string; pythonVariable: string; + dockerFileLocation: string; }): Promise { return await prisma.application.update({ where: { id }, @@ -293,7 +295,8 @@ export async function configureApplication({ publishDirectory, pythonWSGI, pythonModule, - pythonVariable + pythonVariable, + dockerFileLocation } }); } diff --git a/src/lib/database/services.ts b/src/lib/database/services.ts index 7beaaf76a..314cf4548 100644 --- a/src/lib/database/services.ts +++ b/src/lib/database/services.ts @@ -27,25 +27,35 @@ export async function newService({ export async function getService({ id, teamId }: { id: string; teamId: string }): Promise { let body; - const include = { - destinationDocker: true, - plausibleAnalytics: true, - minio: true, - vscodeserver: true, - wordpress: true, - ghost: true, - serviceSecret: true, - meiliSearch: true - }; if (teamId === '0') { body = await prisma.service.findFirst({ where: { id }, - include + include: { + destinationDocker: true, + plausibleAnalytics: true, + minio: true, + vscodeserver: true, + wordpress: true, + ghost: true, + serviceSecret: true, + meiliSearch: true, + persistentStorage: true + } }); } else { body = await prisma.service.findFirst({ where: { id, teams: { some: { id: teamId } } }, - include + include: { + destinationDocker: true, + plausibleAnalytics: true, + minio: true, + vscodeserver: true, + wordpress: true, + ghost: true, + serviceSecret: true, + meiliSearch: true, + persistentStorage: true + } }); } @@ -362,6 +372,7 @@ export async function updateGhostService({ } export async function removeService({ id }: { id: string }): Promise { + await prisma.servicePersistentStorage.deleteMany({ where: { serviceId: id } }); await prisma.meiliSearch.deleteMany({ where: { serviceId: id } }); await prisma.ghost.deleteMany({ where: { serviceId: id } }); await prisma.plausibleAnalytics.deleteMany({ where: { serviceId: id } }); diff --git a/src/lib/docker.ts b/src/lib/docker.ts index 04f68ac48..c2642623f 100644 --- a/src/lib/docker.ts +++ b/src/lib/docker.ts @@ -85,7 +85,8 @@ export async function buildImage({ docker, buildId, isCache = false, - debug = false + debug = false, + dockerFileLocation = '/Dockerfile' }) { if (isCache) { await saveBuildLog({ line: `Building cache image started.`, buildId, applicationId }); @@ -103,7 +104,7 @@ export async function buildImage({ const stream = await docker.engine.buildImage( { src: ['.'], context: workdir }, { - dockerfile: isCache ? 'Dockerfile-cache' : 'Dockerfile', + dockerfile: isCache ? `${dockerFileLocation}-cache` : dockerFileLocation, t: `${applicationId}:${tag}${isCache ? '-cache' : ''}` } ); diff --git a/src/lib/letsencrypt/index.ts b/src/lib/letsencrypt/index.ts index f9de8bc1d..da923c3d5 100644 --- a/src/lib/letsencrypt/index.ts +++ b/src/lib/letsencrypt/index.ts @@ -10,6 +10,9 @@ import { promises as dns } from 'dns'; export async function letsEncrypt(domain: string, id?: string, isCoolify = false): Promise { try { + const certbotImage = + process.arch === 'x64' ? 'certbot/certbot' : 'certbot/certbot:arm64v8-latest'; + const data = await db.prisma.setting.findFirst(); const { minPort, maxPort } = data; @@ -63,7 +66,7 @@ export async function letsEncrypt(domain: string, id?: string, isCoolify = false if (found) return; await asyncExecShell( - `DOCKER_HOST=${host} docker run --rm --name certbot-${randomCuid} -p 9080:${randomPort} -v "coolify-letsencrypt:/etc/letsencrypt" certbot/certbot --logs-dir /etc/letsencrypt/logs certonly --standalone --preferred-challenges http --http-01-address 0.0.0.0 --http-01-port ${randomPort} -d ${nakedDomain} -d ${wwwDomain} --expand --agree-tos --non-interactive --register-unsafely-without-email ${ + `DOCKER_HOST=${host} docker run --rm --name certbot-${randomCuid} -p 9080:${randomPort} -v "coolify-letsencrypt:/etc/letsencrypt" ${certbotImage} --logs-dir /etc/letsencrypt/logs certonly --standalone --preferred-challenges http --http-01-address 0.0.0.0 --http-01-port ${randomPort} -d ${nakedDomain} -d ${wwwDomain} --expand --agree-tos --non-interactive --register-unsafely-without-email ${ dev ? '--test-cert' : '' }` ); @@ -83,7 +86,7 @@ export async function letsEncrypt(domain: string, id?: string, isCoolify = false } if (found) return; await asyncExecShell( - `DOCKER_HOST=${host} docker run --rm --name certbot-${randomCuid} -p 9080:${randomPort} -v "coolify-letsencrypt:/etc/letsencrypt" certbot/certbot --logs-dir /etc/letsencrypt/logs certonly --standalone --preferred-challenges http --http-01-address 0.0.0.0 --http-01-port ${randomPort} -d ${domain} --expand --agree-tos --non-interactive --register-unsafely-without-email ${ + `DOCKER_HOST=${host} docker run --rm --name certbot-${randomCuid} -p 9080:${randomPort} -v "coolify-letsencrypt:/etc/letsencrypt" ${certbotImage} --logs-dir /etc/letsencrypt/logs certonly --standalone --preferred-challenges http --http-01-address 0.0.0.0 --http-01-port ${randomPort} -d ${domain} --expand --agree-tos --non-interactive --register-unsafely-without-email ${ dev ? '--test-cert' : '' }` ); diff --git a/src/lib/queues/builder.ts b/src/lib/queues/builder.ts index e76104376..bd66c05c6 100644 --- a/src/lib/queues/builder.ts +++ b/src/lib/queues/builder.ts @@ -56,7 +56,8 @@ export default async function (job: Job): Promise): Promise): Promise): Promise { publishDirectory, pythonWSGI, pythonModule, - pythonVariable + pythonVariable, + dockerFileLocation } = await event.request.json(); if (port) port = Number(port); @@ -68,7 +69,8 @@ export const post: RequestHandler = async (event) => { startCommand, buildCommand, publishDirectory, - baseDirectory + baseDirectory, + dockerFileLocation }); await db.configureApplication({ id, @@ -84,6 +86,7 @@ export const post: RequestHandler = async (event) => { pythonWSGI, pythonModule, pythonVariable, + dockerFileLocation, ...defaultConfiguration }); return { status: 201 }; diff --git a/src/routes/applications/[id]/index.svelte b/src/routes/applications/[id]/index.svelte index 4b7d6b497..b1939913e 100644 --- a/src/routes/applications/[id]/index.svelte +++ b/src/routes/applications/[id]/index.svelte @@ -68,11 +68,6 @@ value: 'Gunicorn', label: 'Gunicorn' } - // }, - // { - // value: 'uWSGI', - // label: 'uWSGI' - // } ]; if (browser && window.location.hostname === 'demo.coolify.io' && !application.fqdn) { @@ -420,6 +415,23 @@ /> {/if} + {#if application.buildPack === 'docker'} +
+ + + +
+ {/if}