fix: Better queue system + more support on monorepos

This commit is contained in:
Andras Bacsai 2022-03-19 15:04:52 +01:00
parent 376f6f7455
commit 547ca60c2a
18 changed files with 163 additions and 111 deletions

View File

@ -1,4 +1,5 @@
FROM node:16.14.0-alpine
RUN apk add --no-cache g++ cmake make python3
WORKDIR /app
COPY package*.json .
RUN yarn install

View File

@ -84,7 +84,15 @@ export function makeLabelForServices(type) {
}
export const setDefaultConfiguration = async (data) => {
let { buildPack, port, installCommand, startCommand, buildCommand, publishDirectory } = data;
let {
buildPack,
port,
installCommand,
startCommand,
buildCommand,
publishDirectory,
baseDirectory
} = data;
const template = scanningTemplates[buildPack];
if (!port) {
port = template?.port || 3000;
@ -97,6 +105,10 @@ export const setDefaultConfiguration = async (data) => {
if (!startCommand) startCommand = template?.startCommand || 'yarn start';
if (!buildCommand) buildCommand = template?.buildCommand || null;
if (!publishDirectory) publishDirectory = template?.publishDirectory || null;
if (baseDirectory) {
if (!baseDirectory.startsWith('/')) baseDirectory = `/${baseDirectory}`;
if (!baseDirectory.endsWith('/')) baseDirectory = `${baseDirectory}/`;
}
return {
buildPack,
@ -104,7 +116,8 @@ export const setDefaultConfiguration = async (data) => {
installCommand,
startCommand,
buildCommand,
publishDirectory
publishDirectory,
baseDirectory
};
};
@ -175,3 +188,11 @@ export async function copyBaseConfigurationFiles(buildPack, workdir, buildId, ap
throw new Error(error);
}
}
export function checkPnpm(installCommand = null, buildCommand = null, startCommand = null) {
return (
installCommand?.includes('pnpm') ||
buildCommand?.includes('pnpm') ||
startCommand?.includes('pnpm')
);
}

View File

@ -1,5 +1,6 @@
import { buildImage } from '$lib/docker';
import { promises as fs } from 'fs';
import { checkPnpm } from './common';
const createDockerfile = async (data, image): Promise<void> => {
const {
@ -13,10 +14,7 @@ const createDockerfile = async (data, image): Promise<void> => {
pullmergeRequestId
} = data;
const Dockerfile: Array<string> = [];
const isPnpm =
installCommand.includes('pnpm') ||
buildCommand.includes('pnpm') ||
startCommand.includes('pnpm');
const isPnpm = checkPnpm(installCommand, buildCommand, startCommand);
Dockerfile.push(`FROM ${image}`);
Dockerfile.push('WORKDIR /usr/src/app');
Dockerfile.push(`LABEL coolify.image=true`);
@ -39,17 +37,9 @@ const createDockerfile = async (data, image): Promise<void> => {
Dockerfile.push('RUN curl -f https://get.pnpm.io/v6.16.js | node - add --global pnpm');
Dockerfile.push('RUN pnpm add -g pnpm');
}
Dockerfile.push(`COPY ./${baseDirectory || ''}package*.json ./`);
try {
await fs.stat(`${workdir}/yarn.lock`);
Dockerfile.push(`COPY ./${baseDirectory || ''}yarn.lock ./`);
} catch (error) {}
try {
await fs.stat(`${workdir}/pnpm-lock.yaml`);
Dockerfile.push(`COPY ./${baseDirectory || ''}pnpm-lock.yaml ./`);
} catch (error) {}
Dockerfile.push(`COPY .${baseDirectory || ''} ./`);
Dockerfile.push(`RUN ${installCommand}`);
Dockerfile.push(`COPY ./${baseDirectory || ''} ./`);
if (buildCommand) {
Dockerfile.push(`RUN ${buildCommand}`);
}

View File

@ -1,5 +1,6 @@
import { buildImage } from '$lib/docker';
import { promises as fs } from 'fs';
import { checkPnpm } from './common';
const createDockerfile = async (data, image): Promise<void> => {
const {
@ -13,10 +14,8 @@ const createDockerfile = async (data, image): Promise<void> => {
pullmergeRequestId
} = data;
const Dockerfile: Array<string> = [];
const isPnpm =
installCommand.includes('pnpm') ||
buildCommand.includes('pnpm') ||
startCommand.includes('pnpm');
const isPnpm = checkPnpm(installCommand, buildCommand, startCommand);
Dockerfile.push(`FROM ${image}`);
Dockerfile.push('WORKDIR /usr/src/app');
Dockerfile.push(`LABEL coolify.image=true`);
@ -39,17 +38,8 @@ const createDockerfile = async (data, image): Promise<void> => {
Dockerfile.push('RUN curl -f https://get.pnpm.io/v6.16.js | node - add --global pnpm');
Dockerfile.push('RUN pnpm add -g pnpm');
}
Dockerfile.push(`COPY ./${baseDirectory || ''}package*.json ./`);
try {
await fs.stat(`${workdir}/yarn.lock`);
Dockerfile.push(`COPY ./${baseDirectory || ''}yarn.lock ./`);
} catch (error) {}
try {
await fs.stat(`${workdir}/pnpm-lock.yaml`);
Dockerfile.push(`COPY ./${baseDirectory || ''}pnpm-lock.yaml ./`);
} catch (error) {}
Dockerfile.push(`COPY .${baseDirectory || ''} ./`);
Dockerfile.push(`RUN ${installCommand}`);
Dockerfile.push(`COPY ./${baseDirectory || ''} ./`);
if (buildCommand) {
Dockerfile.push(`RUN ${buildCommand}`);
}

View File

@ -1,5 +1,6 @@
import { buildImage } from '$lib/docker';
import { promises as fs } from 'fs';
import { checkPnpm } from './common';
const createDockerfile = async (data, image): Promise<void> => {
const {
@ -13,10 +14,7 @@ const createDockerfile = async (data, image): Promise<void> => {
pullmergeRequestId
} = data;
const Dockerfile: Array<string> = [];
const isPnpm =
installCommand.includes('pnpm') ||
buildCommand.includes('pnpm') ||
startCommand.includes('pnpm');
const isPnpm = checkPnpm(installCommand, buildCommand, startCommand);
Dockerfile.push(`FROM ${image}`);
Dockerfile.push('WORKDIR /usr/src/app');
Dockerfile.push(`LABEL coolify.image=true`);
@ -39,17 +37,8 @@ const createDockerfile = async (data, image): Promise<void> => {
Dockerfile.push('RUN curl -f https://get.pnpm.io/v6.16.js | node - add --global pnpm');
Dockerfile.push('RUN pnpm add -g pnpm');
}
Dockerfile.push(`COPY ./${baseDirectory || ''}package*.json ./`);
try {
await fs.stat(`${workdir}/yarn.lock`);
Dockerfile.push(`COPY ./${baseDirectory || ''}yarn.lock ./`);
} catch (error) {}
try {
await fs.stat(`${workdir}/pnpm-lock.yaml`);
Dockerfile.push(`COPY ./${baseDirectory || ''}pnpm-lock.yaml ./`);
} catch (error) {}
Dockerfile.push(`COPY .${baseDirectory || ''} ./`);
Dockerfile.push(`RUN ${installCommand}`);
Dockerfile.push(`COPY ./${baseDirectory || ''} ./`);
if (buildCommand) {
Dockerfile.push(`RUN ${buildCommand}`);
}

View File

@ -9,7 +9,7 @@ const createDockerfile = async (data, image): Promise<void> => {
Dockerfile.push(`LABEL coolify.image=true`);
Dockerfile.push('RUN a2enmod rewrite');
Dockerfile.push('WORKDIR /var/www/html');
Dockerfile.push(`COPY ./${baseDirectory || ''} /var/www/html`);
Dockerfile.push(`COPY .${baseDirectory || ''} /var/www/html`);
Dockerfile.push(`EXPOSE 80`);
Dockerfile.push('CMD ["apache2-foreground"]');
Dockerfile.push('RUN chown -R www-data /var/www/html');

View File

@ -37,7 +37,7 @@ const createDockerfile = async (data, image): Promise<void> => {
`COPY --from=${applicationId}:${tag}-cache /usr/src/app/${publishDirectory} ./`
);
} else {
Dockerfile.push(`COPY ./${baseDirectory || ''} ./`);
Dockerfile.push(`COPY .${baseDirectory || ''} ./`);
}
Dockerfile.push(`EXPOSE 80`);
Dockerfile.push('CMD ["nginx", "-g", "daemon off;"]');

View File

@ -79,6 +79,9 @@ export async function getApplicationWebhook({ projectId, branch }) {
secrets: true
}
});
if (!application) {
return null;
}
if (application?.gitSource?.githubApp?.clientSecret) {
application.gitSource.githubApp.clientSecret = decrypt(
application.gitSource.githubApp.clientSecret

View File

@ -1,5 +1,6 @@
import Dockerode from 'dockerode';
import { promises as fs } from 'fs';
import { checkPnpm } from './buildPacks/common';
import { saveBuildLog } from './common';
export async function buildCacheImageWithNode(data, imageForBuild) {
@ -16,7 +17,7 @@ export async function buildCacheImageWithNode(data, imageForBuild) {
secrets,
pullmergeRequestId
} = data;
const isPnpm = installCommand.includes('pnpm') || buildCommand.includes('pnpm');
const isPnpm = checkPnpm(installCommand, buildCommand);
const Dockerfile: Array<string> = [];
Dockerfile.push(`FROM ${imageForBuild}`);
Dockerfile.push('WORKDIR /usr/src/app');
@ -40,19 +41,10 @@ export async function buildCacheImageWithNode(data, imageForBuild) {
Dockerfile.push('RUN curl -f https://get.pnpm.io/v6.16.js | node - add --global pnpm');
Dockerfile.push('RUN pnpm add -g pnpm');
}
Dockerfile.push(`COPY .${baseDirectory || ''} ./`);
if (installCommand) {
Dockerfile.push(`COPY ./${baseDirectory || ''}package*.json ./`);
try {
await fs.stat(`${workdir}/yarn.lock`);
Dockerfile.push(`COPY ./${baseDirectory || ''}yarn.lock ./`);
} catch (error) {}
try {
await fs.stat(`${workdir}/pnpm-lock.yaml`);
Dockerfile.push(`COPY ./${baseDirectory || ''}pnpm-lock.yaml ./`);
} catch (error) {}
Dockerfile.push(`RUN ${installCommand}`);
}
Dockerfile.push(`COPY ./${baseDirectory || ''} ./`);
Dockerfile.push(`RUN ${buildCommand}`);
await fs.writeFile(`${workdir}/Dockerfile-cache`, Dockerfile.join('\n'));
await buildImage({ applicationId, tag, workdir, docker, buildId, isCache: true, debug });

View File

@ -3,7 +3,14 @@ import fs from 'fs/promises';
import * as buildpacks from '../buildPacks';
import * as importers from '../importers';
import { dockerInstance } from '../docker';
import { asyncExecShell, createDirectories, getDomain, getEngine, saveBuildLog } from '../common';
import {
asyncExecShell,
asyncSleep,
createDirectories,
getDomain,
getEngine,
saveBuildLog
} from '../common';
import * as db from '$lib/database';
import { decrypt } from '$lib/crypto';
import { sentry } from '$lib/common';
@ -12,7 +19,6 @@ import {
makeLabelForStandaloneApplication,
setDefaultConfiguration
} from '$lib/buildPacks/common';
import { letsEncrypt } from '$lib/letsencrypt';
export default async function (job) {
/*
@ -45,7 +51,7 @@ export default async function (job) {
settings
} = job.data;
const { debug } = settings;
await asyncSleep(1000);
let imageId = applicationId;
let domain = getDomain(fqdn);
const isHttps = fqdn.startsWith('https://');
@ -67,17 +73,8 @@ export default async function (job) {
const docker = dockerInstance({ destinationDocker });
const host = getEngine(destinationDocker.engine);
const build = await db.createBuild({
id: buildId,
applicationId,
destinationDockerId: destinationDocker.id,
gitSourceId: gitSource.id,
githubAppId: gitSource.githubApp?.id,
gitlabAppId: gitSource.gitlabApp?.id,
type
});
const { workdir, repodir } = await createDirectories({ repository, buildId: build.id });
await db.prisma.build.update({ where: { id: buildId }, data: { status: 'running' } });
const { workdir, repodir } = await createDirectories({ repository, buildId });
const configuration = await setDefaultConfiguration(job.data);
@ -87,6 +84,7 @@ export default async function (job) {
startCommand = configuration.startCommand;
buildCommand = configuration.buildCommand;
publishDirectory = configuration.publishDirectory;
baseDirectory = configuration.baseDirectory;
let commit = await importers[gitSource.type]({
applicationId,
@ -97,7 +95,7 @@ export default async function (job) {
gitlabAppId: gitSource.gitlabApp?.id,
repository,
branch,
buildId: build.id,
buildId,
apiUrl: gitSource.apiUrl,
projectId,
deployKeyId: gitSource.gitlabApp?.deployKeyId || null,
@ -109,7 +107,7 @@ export default async function (job) {
}
try {
await db.prisma.build.update({ where: { id: build.id }, data: { commit } });
db.prisma.build.update({ where: { id: buildId }, data: { commit } });
} catch (err) {
console.log(err);
}
@ -160,7 +158,7 @@ export default async function (job) {
await copyBaseConfigurationFiles(buildPack, workdir, buildId, applicationId);
if (buildpacks[buildPack])
await buildpacks[buildPack]({
buildId: build.id,
buildId,
applicationId,
domain,
name,

View File

@ -87,7 +87,7 @@ const cron = async () => {
await queue.proxy.add('proxy', {}, { repeat: { every: 10000 } });
await queue.ssl.add('ssl', {}, { repeat: { every: dev ? 10000 : 60000 } });
await queue.cleanup.add('cleanup', {}, { repeat: { every: dev ? 10000 : 300000 } });
if (!dev) await queue.cleanup.add('cleanup', {}, { repeat: { every: 300000 } });
await queue.sslRenew.add('sslRenew', {}, { repeat: { every: 1800000 } });
const events = {
@ -110,7 +110,7 @@ cron().catch((error) => {
const buildQueueName = 'build_queue';
const buildQueue = new Queue(buildQueueName, connectionOptions);
const buildWorker = new Worker(buildQueueName, async (job) => await builder(job), {
concurrency: 2,
concurrency: 1,
...connectionOptions
});

View File

@ -76,6 +76,7 @@
import { del, post } from '$lib/api';
import { goto } from '$app/navigation';
import { gitTokens } from '$lib/store';
import { toast } from '@zerodevx/svelte-toast';
if (githubToken) $gitTokens.githubToken = githubToken;
if (gitlabToken) $gitTokens.gitlabToken = gitlabToken;
@ -86,6 +87,7 @@
async function handleDeploySubmit() {
try {
const { buildId } = await post(`/applications/${id}/deploy.json`, { ...application });
toast.push('Deployment queued.');
return await goto(`/applications/${id}/logs/build?buildId=${buildId}`);
} catch ({ error }) {
return errorNotification(error);

View File

@ -30,6 +30,18 @@ export const post: RequestHandler = async (event) => {
await db.prisma.application.update({ where: { id }, data: { configHash } });
}
await db.prisma.application.update({ where: { id }, data: { updatedAt: new Date() } });
await db.prisma.build.create({
data: {
id: buildId,
applicationId: id,
destinationDockerId: applicationFound.destinationDocker.id,
gitSourceId: applicationFound.gitSource.id,
githubAppId: applicationFound.gitSource.githubApp?.id,
gitlabAppId: applicationFound.gitSource.gitlabApp?.id,
status: 'queued',
type: 'manual'
}
});
await buildQueue.add(buildId, { build_id: buildId, type: 'manual', ...applicationFound });
return {
status: 200,

View File

@ -43,11 +43,11 @@
logs = logs.concat(responseLogs.map((log) => ({ ...log, line: cleanAnsiCodes(log.line) })));
loading = false;
streamInterval = setInterval(async () => {
if (status !== 'running') {
if (status !== 'running' && status !== 'queued') {
clearInterval(streamInterval);
return;
}
const nextSequence = logs[logs.length - 1].time;
const nextSequence = logs[logs.length - 1]?.time || 0;
try {
const data = await get(
`/applications/${id}/logs/build/build.json?buildId=${buildId}&sequence=${nextSequence}`
@ -83,38 +83,42 @@
{#if currentStatus === 'running'}
<LoadingLogs />
{/if}
<div class="flex justify-end sticky top-0 p-2">
<button
on:click={followBuild}
class="bg-transparent"
data-tooltip="Follow logs"
class:text-green-500={followingBuild}
>
<svg
xmlns="http://www.w3.org/2000/svg"
class="w-6 h-6"
viewBox="0 0 24 24"
stroke-width="1.5"
stroke="currentColor"
fill="none"
stroke-linecap="round"
stroke-linejoin="round"
{#if currentStatus === 'queued'}
<div class="text-center">Queued and waiting for execution.</div>
{:else}
<div class="flex justify-end sticky top-0 p-2">
<button
on:click={followBuild}
class="bg-transparent"
data-tooltip="Follow logs"
class:text-green-500={followingBuild}
>
<path stroke="none" d="M0 0h24v24H0z" fill="none" />
<circle cx="12" cy="12" r="9" />
<line x1="8" y1="12" x2="12" y2="16" />
<line x1="12" y1="8" x2="12" y2="16" />
<line x1="16" y1="12" x2="12" y2="16" />
</svg>
</button>
</div>
<div
class="font-mono leading-6 text-left text-md tracking-tighter rounded bg-coolgray-200 py-5 px-6 whitespace-pre-wrap break-words overflow-auto max-h-[80vh] -mt-12 overflow-y-scroll scrollbar-w-1 scrollbar-thumb-coollabs scrollbar-track-coolgray-200"
bind:this={logsEl}
>
{#each logs as log}
<div>{log.line + '\n'}</div>
{/each}
</div>
<svg
xmlns="http://www.w3.org/2000/svg"
class="w-6 h-6"
viewBox="0 0 24 24"
stroke-width="1.5"
stroke="currentColor"
fill="none"
stroke-linecap="round"
stroke-linejoin="round"
>
<path stroke="none" d="M0 0h24v24H0z" fill="none" />
<circle cx="12" cy="12" r="9" />
<line x1="8" y1="12" x2="12" y2="16" />
<line x1="12" y1="8" x2="12" y2="16" />
<line x1="16" y1="12" x2="12" y2="16" />
</svg>
</button>
</div>
<div
class="font-mono leading-6 text-left text-md tracking-tighter rounded bg-coolgray-200 py-5 px-6 whitespace-pre-wrap break-words overflow-auto max-h-[80vh] -mt-12 overflow-y-scroll scrollbar-w-1 scrollbar-thumb-coollabs scrollbar-track-coolgray-200"
bind:this={logsEl}
>
{#each logs as log}
<div>{log.line + '\n'}</div>
{/each}
</div>
{/if}
</div>
{/if}

View File

@ -19,7 +19,7 @@ export const get: RequestHandler = async (event) => {
return {
body: {
logs,
status: data?.status || 'running'
status: data?.status
}
};
} catch (error) {

View File

@ -121,6 +121,8 @@
<div class="w-48 text-center text-xs">
{#if build.status === 'running'}
<div class="font-bold">Running</div>
{:else if build.status === 'queued'}
<div class="font-bold">Queued</div>
{:else}
<div>{build.since}</div>
<div>Finished in <span class="font-bold">{build.took}s</span></div>

View File

@ -88,6 +88,18 @@ export const post: RequestHandler = async (event) => {
where: { id: applicationFound.id },
data: { updatedAt: new Date() }
});
await db.prisma.build.create({
data: {
id: buildId,
applicationId: applicationFound.id,
destinationDockerId: applicationFound.destinationDocker.id,
gitSourceId: applicationFound.gitSource.id,
githubAppId: applicationFound.gitSource.githubApp?.id,
gitlabAppId: applicationFound.gitSource.gitlabApp?.id,
status: 'queued',
type: 'webhook_commit'
}
});
await buildQueue.add(buildId, {
build_id: buildId,
type: 'webhook_commit',
@ -136,6 +148,18 @@ export const post: RequestHandler = async (event) => {
where: { id: applicationFound.id },
data: { updatedAt: new Date() }
});
await db.prisma.build.create({
data: {
id: buildId,
applicationId: applicationFound.id,
destinationDockerId: applicationFound.destinationDocker.id,
gitSourceId: applicationFound.gitSource.id,
githubAppId: applicationFound.gitSource.githubApp?.id,
gitlabAppId: applicationFound.gitSource.gitlabApp?.id,
status: 'queued',
type: 'webhook_pr'
}
});
await buildQueue.add(buildId, {
build_id: buildId,
type: 'webhook_pr',

View File

@ -52,6 +52,18 @@ export const post: RequestHandler = async (event) => {
where: { id: applicationFound.id },
data: { updatedAt: new Date() }
});
await db.prisma.build.create({
data: {
id: buildId,
applicationId: applicationFound.id,
destinationDockerId: applicationFound.destinationDocker.id,
gitSourceId: applicationFound.gitSource.id,
githubAppId: applicationFound.gitSource.githubApp?.id,
gitlabAppId: applicationFound.gitSource.gitlabApp?.id,
status: 'queued',
type: 'webhook_commit'
}
});
await buildQueue.add(buildId, {
build_id: buildId,
type: 'webhook_commit',
@ -133,6 +145,18 @@ export const post: RequestHandler = async (event) => {
where: { id: applicationFound.id },
data: { updatedAt: new Date() }
});
await db.prisma.build.create({
data: {
id: buildId,
applicationId: applicationFound.id,
destinationDockerId: applicationFound.destinationDocker.id,
gitSourceId: applicationFound.gitSource.id,
githubAppId: applicationFound.gitSource.githubApp?.id,
gitlabAppId: applicationFound.gitSource.gitlabApp?.id,
status: 'queued',
type: 'webhook_mr'
}
});
await buildQueue.add(buildId, {
build_id: buildId,
type: 'webhook_mr',