From 547ca60c2af502b1966704f2dd48b6d14ab8e149 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Sat, 19 Mar 2022 15:04:52 +0100 Subject: [PATCH] fix: Better queue system + more support on monorepos --- Dockerfile | 1 + src/lib/buildPacks/common.ts | 25 ++++++- src/lib/buildPacks/nextjs.ts | 18 ++--- src/lib/buildPacks/node.ts | 18 ++--- src/lib/buildPacks/nuxtjs.ts | 17 +---- src/lib/buildPacks/php.ts | 2 +- src/lib/buildPacks/static.ts | 2 +- src/lib/database/applications.ts | 3 + src/lib/docker.ts | 14 +--- src/lib/queues/builder.ts | 32 ++++----- src/lib/queues/index.ts | 4 +- src/routes/applications/[id]/__layout.svelte | 2 + src/routes/applications/[id]/deploy.json.ts | 12 ++++ .../[id]/logs/build/_BuildLog.svelte | 72 ++++++++++--------- .../[id]/logs/build/build.json.ts | 2 +- .../applications/[id]/logs/build/index.svelte | 2 + src/routes/webhooks/github/events.ts | 24 +++++++ src/routes/webhooks/gitlab/events.ts | 24 +++++++ 18 files changed, 163 insertions(+), 111 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9ca82df2c..5d0a8912b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ FROM node:16.14.0-alpine +RUN apk add --no-cache g++ cmake make python3 WORKDIR /app COPY package*.json . RUN yarn install diff --git a/src/lib/buildPacks/common.ts b/src/lib/buildPacks/common.ts index 67be90ee5..1d2fef25f 100644 --- a/src/lib/buildPacks/common.ts +++ b/src/lib/buildPacks/common.ts @@ -84,7 +84,15 @@ export function makeLabelForServices(type) { } export const setDefaultConfiguration = async (data) => { - let { buildPack, port, installCommand, startCommand, buildCommand, publishDirectory } = data; + let { + buildPack, + port, + installCommand, + startCommand, + buildCommand, + publishDirectory, + baseDirectory + } = data; const template = scanningTemplates[buildPack]; if (!port) { port = template?.port || 3000; @@ -97,6 +105,10 @@ export const setDefaultConfiguration = async (data) => { if (!startCommand) startCommand = template?.startCommand || 'yarn start'; if (!buildCommand) buildCommand = template?.buildCommand || null; if (!publishDirectory) publishDirectory = template?.publishDirectory || null; + if (baseDirectory) { + if (!baseDirectory.startsWith('/')) baseDirectory = `/${baseDirectory}`; + if (!baseDirectory.endsWith('/')) baseDirectory = `${baseDirectory}/`; + } return { buildPack, @@ -104,7 +116,8 @@ export const setDefaultConfiguration = async (data) => { installCommand, startCommand, buildCommand, - publishDirectory + publishDirectory, + baseDirectory }; }; @@ -175,3 +188,11 @@ export async function copyBaseConfigurationFiles(buildPack, workdir, buildId, ap throw new Error(error); } } + +export function checkPnpm(installCommand = null, buildCommand = null, startCommand = null) { + return ( + installCommand?.includes('pnpm') || + buildCommand?.includes('pnpm') || + startCommand?.includes('pnpm') + ); +} diff --git a/src/lib/buildPacks/nextjs.ts b/src/lib/buildPacks/nextjs.ts index c8d16dab6..c8d6a03bc 100644 --- a/src/lib/buildPacks/nextjs.ts +++ b/src/lib/buildPacks/nextjs.ts @@ -1,5 +1,6 @@ import { buildImage } from '$lib/docker'; import { promises as fs } from 'fs'; +import { checkPnpm } from './common'; const createDockerfile = async (data, image): Promise => { const { @@ -13,10 +14,7 @@ const createDockerfile = async (data, image): Promise => { pullmergeRequestId } = data; const Dockerfile: Array = []; - const isPnpm = - installCommand.includes('pnpm') || - buildCommand.includes('pnpm') || - startCommand.includes('pnpm'); + const isPnpm = checkPnpm(installCommand, buildCommand, startCommand); Dockerfile.push(`FROM ${image}`); Dockerfile.push('WORKDIR /usr/src/app'); Dockerfile.push(`LABEL coolify.image=true`); @@ -39,17 +37,9 @@ const createDockerfile = async (data, image): Promise => { Dockerfile.push('RUN curl -f https://get.pnpm.io/v6.16.js | node - add --global pnpm'); Dockerfile.push('RUN pnpm add -g pnpm'); } - Dockerfile.push(`COPY ./${baseDirectory || ''}package*.json ./`); - try { - await fs.stat(`${workdir}/yarn.lock`); - Dockerfile.push(`COPY ./${baseDirectory || ''}yarn.lock ./`); - } catch (error) {} - try { - await fs.stat(`${workdir}/pnpm-lock.yaml`); - Dockerfile.push(`COPY ./${baseDirectory || ''}pnpm-lock.yaml ./`); - } catch (error) {} + Dockerfile.push(`COPY .${baseDirectory || ''} ./`); Dockerfile.push(`RUN ${installCommand}`); - Dockerfile.push(`COPY ./${baseDirectory || ''} ./`); + if (buildCommand) { Dockerfile.push(`RUN ${buildCommand}`); } diff --git a/src/lib/buildPacks/node.ts b/src/lib/buildPacks/node.ts index c8d16dab6..cb99ecec8 100644 --- a/src/lib/buildPacks/node.ts +++ b/src/lib/buildPacks/node.ts @@ -1,5 +1,6 @@ import { buildImage } from '$lib/docker'; import { promises as fs } from 'fs'; +import { checkPnpm } from './common'; const createDockerfile = async (data, image): Promise => { const { @@ -13,10 +14,8 @@ const createDockerfile = async (data, image): Promise => { pullmergeRequestId } = data; const Dockerfile: Array = []; - const isPnpm = - installCommand.includes('pnpm') || - buildCommand.includes('pnpm') || - startCommand.includes('pnpm'); + const isPnpm = checkPnpm(installCommand, buildCommand, startCommand); + Dockerfile.push(`FROM ${image}`); Dockerfile.push('WORKDIR /usr/src/app'); Dockerfile.push(`LABEL coolify.image=true`); @@ -39,17 +38,8 @@ const createDockerfile = async (data, image): Promise => { Dockerfile.push('RUN curl -f https://get.pnpm.io/v6.16.js | node - add --global pnpm'); Dockerfile.push('RUN pnpm add -g pnpm'); } - Dockerfile.push(`COPY ./${baseDirectory || ''}package*.json ./`); - try { - await fs.stat(`${workdir}/yarn.lock`); - Dockerfile.push(`COPY ./${baseDirectory || ''}yarn.lock ./`); - } catch (error) {} - try { - await fs.stat(`${workdir}/pnpm-lock.yaml`); - Dockerfile.push(`COPY ./${baseDirectory || ''}pnpm-lock.yaml ./`); - } catch (error) {} + Dockerfile.push(`COPY .${baseDirectory || ''} ./`); Dockerfile.push(`RUN ${installCommand}`); - Dockerfile.push(`COPY ./${baseDirectory || ''} ./`); if (buildCommand) { Dockerfile.push(`RUN ${buildCommand}`); } diff --git a/src/lib/buildPacks/nuxtjs.ts b/src/lib/buildPacks/nuxtjs.ts index c8d16dab6..ca7c1809c 100644 --- a/src/lib/buildPacks/nuxtjs.ts +++ b/src/lib/buildPacks/nuxtjs.ts @@ -1,5 +1,6 @@ import { buildImage } from '$lib/docker'; import { promises as fs } from 'fs'; +import { checkPnpm } from './common'; const createDockerfile = async (data, image): Promise => { const { @@ -13,10 +14,7 @@ const createDockerfile = async (data, image): Promise => { pullmergeRequestId } = data; const Dockerfile: Array = []; - const isPnpm = - installCommand.includes('pnpm') || - buildCommand.includes('pnpm') || - startCommand.includes('pnpm'); + const isPnpm = checkPnpm(installCommand, buildCommand, startCommand); Dockerfile.push(`FROM ${image}`); Dockerfile.push('WORKDIR /usr/src/app'); Dockerfile.push(`LABEL coolify.image=true`); @@ -39,17 +37,8 @@ const createDockerfile = async (data, image): Promise => { Dockerfile.push('RUN curl -f https://get.pnpm.io/v6.16.js | node - add --global pnpm'); Dockerfile.push('RUN pnpm add -g pnpm'); } - Dockerfile.push(`COPY ./${baseDirectory || ''}package*.json ./`); - try { - await fs.stat(`${workdir}/yarn.lock`); - Dockerfile.push(`COPY ./${baseDirectory || ''}yarn.lock ./`); - } catch (error) {} - try { - await fs.stat(`${workdir}/pnpm-lock.yaml`); - Dockerfile.push(`COPY ./${baseDirectory || ''}pnpm-lock.yaml ./`); - } catch (error) {} + Dockerfile.push(`COPY .${baseDirectory || ''} ./`); Dockerfile.push(`RUN ${installCommand}`); - Dockerfile.push(`COPY ./${baseDirectory || ''} ./`); if (buildCommand) { Dockerfile.push(`RUN ${buildCommand}`); } diff --git a/src/lib/buildPacks/php.ts b/src/lib/buildPacks/php.ts index 64ac6a141..5d53bd468 100644 --- a/src/lib/buildPacks/php.ts +++ b/src/lib/buildPacks/php.ts @@ -9,7 +9,7 @@ const createDockerfile = async (data, image): Promise => { Dockerfile.push(`LABEL coolify.image=true`); Dockerfile.push('RUN a2enmod rewrite'); Dockerfile.push('WORKDIR /var/www/html'); - Dockerfile.push(`COPY ./${baseDirectory || ''} /var/www/html`); + Dockerfile.push(`COPY .${baseDirectory || ''} /var/www/html`); Dockerfile.push(`EXPOSE 80`); Dockerfile.push('CMD ["apache2-foreground"]'); Dockerfile.push('RUN chown -R www-data /var/www/html'); diff --git a/src/lib/buildPacks/static.ts b/src/lib/buildPacks/static.ts index f9fa4ee96..795f00d7b 100644 --- a/src/lib/buildPacks/static.ts +++ b/src/lib/buildPacks/static.ts @@ -37,7 +37,7 @@ const createDockerfile = async (data, image): Promise => { `COPY --from=${applicationId}:${tag}-cache /usr/src/app/${publishDirectory} ./` ); } else { - Dockerfile.push(`COPY ./${baseDirectory || ''} ./`); + Dockerfile.push(`COPY .${baseDirectory || ''} ./`); } Dockerfile.push(`EXPOSE 80`); Dockerfile.push('CMD ["nginx", "-g", "daemon off;"]'); diff --git a/src/lib/database/applications.ts b/src/lib/database/applications.ts index 70c66144b..e6679d365 100644 --- a/src/lib/database/applications.ts +++ b/src/lib/database/applications.ts @@ -79,6 +79,9 @@ export async function getApplicationWebhook({ projectId, branch }) { secrets: true } }); + if (!application) { + return null; + } if (application?.gitSource?.githubApp?.clientSecret) { application.gitSource.githubApp.clientSecret = decrypt( application.gitSource.githubApp.clientSecret diff --git a/src/lib/docker.ts b/src/lib/docker.ts index a202649e2..ccbc21b2a 100644 --- a/src/lib/docker.ts +++ b/src/lib/docker.ts @@ -1,5 +1,6 @@ import Dockerode from 'dockerode'; import { promises as fs } from 'fs'; +import { checkPnpm } from './buildPacks/common'; import { saveBuildLog } from './common'; export async function buildCacheImageWithNode(data, imageForBuild) { @@ -16,7 +17,7 @@ export async function buildCacheImageWithNode(data, imageForBuild) { secrets, pullmergeRequestId } = data; - const isPnpm = installCommand.includes('pnpm') || buildCommand.includes('pnpm'); + const isPnpm = checkPnpm(installCommand, buildCommand); const Dockerfile: Array = []; Dockerfile.push(`FROM ${imageForBuild}`); Dockerfile.push('WORKDIR /usr/src/app'); @@ -40,19 +41,10 @@ export async function buildCacheImageWithNode(data, imageForBuild) { Dockerfile.push('RUN curl -f https://get.pnpm.io/v6.16.js | node - add --global pnpm'); Dockerfile.push('RUN pnpm add -g pnpm'); } + Dockerfile.push(`COPY .${baseDirectory || ''} ./`); if (installCommand) { - Dockerfile.push(`COPY ./${baseDirectory || ''}package*.json ./`); - try { - await fs.stat(`${workdir}/yarn.lock`); - Dockerfile.push(`COPY ./${baseDirectory || ''}yarn.lock ./`); - } catch (error) {} - try { - await fs.stat(`${workdir}/pnpm-lock.yaml`); - Dockerfile.push(`COPY ./${baseDirectory || ''}pnpm-lock.yaml ./`); - } catch (error) {} Dockerfile.push(`RUN ${installCommand}`); } - Dockerfile.push(`COPY ./${baseDirectory || ''} ./`); Dockerfile.push(`RUN ${buildCommand}`); await fs.writeFile(`${workdir}/Dockerfile-cache`, Dockerfile.join('\n')); await buildImage({ applicationId, tag, workdir, docker, buildId, isCache: true, debug }); diff --git a/src/lib/queues/builder.ts b/src/lib/queues/builder.ts index cefcfc6ed..0430276e3 100644 --- a/src/lib/queues/builder.ts +++ b/src/lib/queues/builder.ts @@ -3,7 +3,14 @@ import fs from 'fs/promises'; import * as buildpacks from '../buildPacks'; import * as importers from '../importers'; import { dockerInstance } from '../docker'; -import { asyncExecShell, createDirectories, getDomain, getEngine, saveBuildLog } from '../common'; +import { + asyncExecShell, + asyncSleep, + createDirectories, + getDomain, + getEngine, + saveBuildLog +} from '../common'; import * as db from '$lib/database'; import { decrypt } from '$lib/crypto'; import { sentry } from '$lib/common'; @@ -12,7 +19,6 @@ import { makeLabelForStandaloneApplication, setDefaultConfiguration } from '$lib/buildPacks/common'; -import { letsEncrypt } from '$lib/letsencrypt'; export default async function (job) { /* @@ -45,7 +51,7 @@ export default async function (job) { settings } = job.data; const { debug } = settings; - + await asyncSleep(1000); let imageId = applicationId; let domain = getDomain(fqdn); const isHttps = fqdn.startsWith('https://'); @@ -67,17 +73,8 @@ export default async function (job) { const docker = dockerInstance({ destinationDocker }); const host = getEngine(destinationDocker.engine); - const build = await db.createBuild({ - id: buildId, - applicationId, - destinationDockerId: destinationDocker.id, - gitSourceId: gitSource.id, - githubAppId: gitSource.githubApp?.id, - gitlabAppId: gitSource.gitlabApp?.id, - type - }); - - const { workdir, repodir } = await createDirectories({ repository, buildId: build.id }); + await db.prisma.build.update({ where: { id: buildId }, data: { status: 'running' } }); + const { workdir, repodir } = await createDirectories({ repository, buildId }); const configuration = await setDefaultConfiguration(job.data); @@ -87,6 +84,7 @@ export default async function (job) { startCommand = configuration.startCommand; buildCommand = configuration.buildCommand; publishDirectory = configuration.publishDirectory; + baseDirectory = configuration.baseDirectory; let commit = await importers[gitSource.type]({ applicationId, @@ -97,7 +95,7 @@ export default async function (job) { gitlabAppId: gitSource.gitlabApp?.id, repository, branch, - buildId: build.id, + buildId, apiUrl: gitSource.apiUrl, projectId, deployKeyId: gitSource.gitlabApp?.deployKeyId || null, @@ -109,7 +107,7 @@ export default async function (job) { } try { - await db.prisma.build.update({ where: { id: build.id }, data: { commit } }); + db.prisma.build.update({ where: { id: buildId }, data: { commit } }); } catch (err) { console.log(err); } @@ -160,7 +158,7 @@ export default async function (job) { await copyBaseConfigurationFiles(buildPack, workdir, buildId, applicationId); if (buildpacks[buildPack]) await buildpacks[buildPack]({ - buildId: build.id, + buildId, applicationId, domain, name, diff --git a/src/lib/queues/index.ts b/src/lib/queues/index.ts index 8e4cc2479..55d7ca51b 100644 --- a/src/lib/queues/index.ts +++ b/src/lib/queues/index.ts @@ -87,7 +87,7 @@ const cron = async () => { await queue.proxy.add('proxy', {}, { repeat: { every: 10000 } }); await queue.ssl.add('ssl', {}, { repeat: { every: dev ? 10000 : 60000 } }); - await queue.cleanup.add('cleanup', {}, { repeat: { every: dev ? 10000 : 300000 } }); + if (!dev) await queue.cleanup.add('cleanup', {}, { repeat: { every: 300000 } }); await queue.sslRenew.add('sslRenew', {}, { repeat: { every: 1800000 } }); const events = { @@ -110,7 +110,7 @@ cron().catch((error) => { const buildQueueName = 'build_queue'; const buildQueue = new Queue(buildQueueName, connectionOptions); const buildWorker = new Worker(buildQueueName, async (job) => await builder(job), { - concurrency: 2, + concurrency: 1, ...connectionOptions }); diff --git a/src/routes/applications/[id]/__layout.svelte b/src/routes/applications/[id]/__layout.svelte index 041be4a82..fb944519b 100644 --- a/src/routes/applications/[id]/__layout.svelte +++ b/src/routes/applications/[id]/__layout.svelte @@ -76,6 +76,7 @@ import { del, post } from '$lib/api'; import { goto } from '$app/navigation'; import { gitTokens } from '$lib/store'; + import { toast } from '@zerodevx/svelte-toast'; if (githubToken) $gitTokens.githubToken = githubToken; if (gitlabToken) $gitTokens.gitlabToken = gitlabToken; @@ -86,6 +87,7 @@ async function handleDeploySubmit() { try { const { buildId } = await post(`/applications/${id}/deploy.json`, { ...application }); + toast.push('Deployment queued.'); return await goto(`/applications/${id}/logs/build?buildId=${buildId}`); } catch ({ error }) { return errorNotification(error); diff --git a/src/routes/applications/[id]/deploy.json.ts b/src/routes/applications/[id]/deploy.json.ts index ce5182c44..0e33c7ce0 100644 --- a/src/routes/applications/[id]/deploy.json.ts +++ b/src/routes/applications/[id]/deploy.json.ts @@ -30,6 +30,18 @@ export const post: RequestHandler = async (event) => { await db.prisma.application.update({ where: { id }, data: { configHash } }); } await db.prisma.application.update({ where: { id }, data: { updatedAt: new Date() } }); + await db.prisma.build.create({ + data: { + id: buildId, + applicationId: id, + destinationDockerId: applicationFound.destinationDocker.id, + gitSourceId: applicationFound.gitSource.id, + githubAppId: applicationFound.gitSource.githubApp?.id, + gitlabAppId: applicationFound.gitSource.gitlabApp?.id, + status: 'queued', + type: 'manual' + } + }); await buildQueue.add(buildId, { build_id: buildId, type: 'manual', ...applicationFound }); return { status: 200, diff --git a/src/routes/applications/[id]/logs/build/_BuildLog.svelte b/src/routes/applications/[id]/logs/build/_BuildLog.svelte index 6d3cced39..1ac0ab83a 100644 --- a/src/routes/applications/[id]/logs/build/_BuildLog.svelte +++ b/src/routes/applications/[id]/logs/build/_BuildLog.svelte @@ -43,11 +43,11 @@ logs = logs.concat(responseLogs.map((log) => ({ ...log, line: cleanAnsiCodes(log.line) }))); loading = false; streamInterval = setInterval(async () => { - if (status !== 'running') { + if (status !== 'running' && status !== 'queued') { clearInterval(streamInterval); return; } - const nextSequence = logs[logs.length - 1].time; + const nextSequence = logs[logs.length - 1]?.time || 0; try { const data = await get( `/applications/${id}/logs/build/build.json?buildId=${buildId}&sequence=${nextSequence}` @@ -83,38 +83,42 @@ {#if currentStatus === 'running'} {/if} -
-
+ {:else} +
+ -
-
- {#each logs as log} -
{log.line + '\n'}
- {/each} -
+ + + + + + + + + +
+ {#each logs as log} +
{log.line + '\n'}
+ {/each} +
+ {/if} {/if} diff --git a/src/routes/applications/[id]/logs/build/build.json.ts b/src/routes/applications/[id]/logs/build/build.json.ts index d6b79ceeb..08779c001 100644 --- a/src/routes/applications/[id]/logs/build/build.json.ts +++ b/src/routes/applications/[id]/logs/build/build.json.ts @@ -19,7 +19,7 @@ export const get: RequestHandler = async (event) => { return { body: { logs, - status: data?.status || 'running' + status: data?.status } }; } catch (error) { diff --git a/src/routes/applications/[id]/logs/build/index.svelte b/src/routes/applications/[id]/logs/build/index.svelte index d81159d9d..eba0e1e54 100644 --- a/src/routes/applications/[id]/logs/build/index.svelte +++ b/src/routes/applications/[id]/logs/build/index.svelte @@ -121,6 +121,8 @@
{#if build.status === 'running'}
Running
+ {:else if build.status === 'queued'} +
Queued
{:else}
{build.since}
Finished in {build.took}s
diff --git a/src/routes/webhooks/github/events.ts b/src/routes/webhooks/github/events.ts index 21eead6c2..9f278122d 100644 --- a/src/routes/webhooks/github/events.ts +++ b/src/routes/webhooks/github/events.ts @@ -88,6 +88,18 @@ export const post: RequestHandler = async (event) => { where: { id: applicationFound.id }, data: { updatedAt: new Date() } }); + await db.prisma.build.create({ + data: { + id: buildId, + applicationId: applicationFound.id, + destinationDockerId: applicationFound.destinationDocker.id, + gitSourceId: applicationFound.gitSource.id, + githubAppId: applicationFound.gitSource.githubApp?.id, + gitlabAppId: applicationFound.gitSource.gitlabApp?.id, + status: 'queued', + type: 'webhook_commit' + } + }); await buildQueue.add(buildId, { build_id: buildId, type: 'webhook_commit', @@ -136,6 +148,18 @@ export const post: RequestHandler = async (event) => { where: { id: applicationFound.id }, data: { updatedAt: new Date() } }); + await db.prisma.build.create({ + data: { + id: buildId, + applicationId: applicationFound.id, + destinationDockerId: applicationFound.destinationDocker.id, + gitSourceId: applicationFound.gitSource.id, + githubAppId: applicationFound.gitSource.githubApp?.id, + gitlabAppId: applicationFound.gitSource.gitlabApp?.id, + status: 'queued', + type: 'webhook_pr' + } + }); await buildQueue.add(buildId, { build_id: buildId, type: 'webhook_pr', diff --git a/src/routes/webhooks/gitlab/events.ts b/src/routes/webhooks/gitlab/events.ts index 1c125ec02..4a1eeafcc 100644 --- a/src/routes/webhooks/gitlab/events.ts +++ b/src/routes/webhooks/gitlab/events.ts @@ -52,6 +52,18 @@ export const post: RequestHandler = async (event) => { where: { id: applicationFound.id }, data: { updatedAt: new Date() } }); + await db.prisma.build.create({ + data: { + id: buildId, + applicationId: applicationFound.id, + destinationDockerId: applicationFound.destinationDocker.id, + gitSourceId: applicationFound.gitSource.id, + githubAppId: applicationFound.gitSource.githubApp?.id, + gitlabAppId: applicationFound.gitSource.gitlabApp?.id, + status: 'queued', + type: 'webhook_commit' + } + }); await buildQueue.add(buildId, { build_id: buildId, type: 'webhook_commit', @@ -133,6 +145,18 @@ export const post: RequestHandler = async (event) => { where: { id: applicationFound.id }, data: { updatedAt: new Date() } }); + await db.prisma.build.create({ + data: { + id: buildId, + applicationId: applicationFound.id, + destinationDockerId: applicationFound.destinationDocker.id, + gitSourceId: applicationFound.gitSource.id, + githubAppId: applicationFound.gitSource.githubApp?.id, + gitlabAppId: applicationFound.gitSource.gitlabApp?.id, + status: 'queued', + type: 'webhook_mr' + } + }); await buildQueue.add(buildId, { build_id: buildId, type: 'webhook_mr',