fix: escape env vars

This commit is contained in:
Andras Bacsai 2022-12-19 10:22:11 +01:00
parent 4261147fe8
commit bd7d756254

View File

@ -3,8 +3,25 @@ import crypto from 'crypto';
import fs from 'fs/promises';
import yaml from 'js-yaml';
import { copyBaseConfigurationFiles, makeLabelForSimpleDockerfile, makeLabelForStandaloneApplication, saveBuildLog, saveDockerRegistryCredentials, setDefaultConfiguration } from '../lib/buildPacks/common';
import { createDirectories, decrypt, defaultComposeConfiguration, getDomain, prisma, decryptApplication, isDev, pushToRegistry, executeCommand } from '../lib/common';
import {
copyBaseConfigurationFiles,
makeLabelForSimpleDockerfile,
makeLabelForStandaloneApplication,
saveBuildLog,
saveDockerRegistryCredentials,
setDefaultConfiguration
} from '../lib/buildPacks/common';
import {
createDirectories,
decrypt,
defaultComposeConfiguration,
getDomain,
prisma,
decryptApplication,
isDev,
pushToRegistry,
executeCommand
} from '../lib/common';
import * as importers from '../lib/importers';
import * as buildpacks from '../lib/buildPacks';
@ -14,33 +31,56 @@ import * as buildpacks from '../lib/buildPacks';
if (message === 'error') throw new Error('oops');
if (message === 'cancel') {
parentPort.postMessage('cancelled');
await prisma.$disconnect()
await prisma.$disconnect();
process.exit(0);
}
});
const pThrottle = await import('p-throttle')
const pThrottle = await import('p-throttle');
const throttle = pThrottle.default({
limit: 1,
interval: 2000
});
const th = throttle(async () => {
try {
const queuedBuilds = await prisma.build.findMany({ where: { status: { in: ['queued', 'running'] } }, orderBy: { createdAt: 'asc' } });
const { concurrentBuilds } = await prisma.setting.findFirst({})
const queuedBuilds = await prisma.build.findMany({
where: { status: { in: ['queued', 'running'] } },
orderBy: { createdAt: 'asc' }
});
const { concurrentBuilds } = await prisma.setting.findFirst({});
if (queuedBuilds.length > 0) {
parentPort.postMessage({ deploying: true });
const concurrency = concurrentBuilds;
const pAll = await import('p-all');
const actions = []
const actions = [];
for (const queueBuild of queuedBuilds) {
actions.push(async () => {
let application = await prisma.application.findUnique({ where: { id: queueBuild.applicationId }, include: { dockerRegistry: true, destinationDocker: true, gitSource: { include: { githubApp: true, gitlabApp: true } }, persistentStorage: true, secrets: true, settings: true, teams: true } })
const { default: escapeStringRegexp } = await import('escape-string-regexp');
let application = await prisma.application.findUnique({
where: { id: queueBuild.applicationId },
include: {
dockerRegistry: true,
destinationDocker: true,
gitSource: { include: { githubApp: true, gitlabApp: true } },
persistentStorage: true,
secrets: true,
settings: true,
teams: true
}
});
let { id: buildId, type, gitSourceId, sourceBranch = null, pullmergeRequestId = null, previewApplicationId = null, forceRebuild, sourceRepository = null } = queueBuild
application = decryptApplication(application)
let {
id: buildId,
type,
gitSourceId,
sourceBranch = null,
pullmergeRequestId = null,
previewApplicationId = null,
forceRebuild,
sourceRepository = null
} = queueBuild;
application = decryptApplication(application);
if (!gitSourceId && application.simpleDockerfile) {
const {
@ -53,54 +93,81 @@ import * as buildpacks from '../lib/buildPacks';
exposePort,
simpleDockerfile,
dockerRegistry
} = application
} = application;
const { workdir } = await createDirectories({ repository: applicationId, buildId });
try {
if (queueBuild.status === 'running') {
await saveBuildLog({ line: 'Building halted, restarting...', buildId, applicationId: application.id });
await saveBuildLog({
line: 'Building halted, restarting...',
buildId,
applicationId: application.id
});
}
const volumes =
persistentStorage?.map((storage) => {
if (storage.oldPath) {
return `${applicationId}${storage.path.replace(/\//gi, '-').replace('-app', '')}:${storage.path}`;
return `${applicationId}${storage.path
.replace(/\//gi, '-')
.replace('-app', '')}:${storage.path}`;
}
return `${applicationId}${storage.path.replace(/\//gi, '-')}:${storage.path}`;
}) || [];
if (destinationDockerId) {
await prisma.build.update({ where: { id: buildId }, data: { status: 'running' } });
await prisma.build.update({
where: { id: buildId },
data: { status: 'running' }
});
try {
const { stdout: containers } = await executeCommand({
dockerId: destinationDockerId,
command: `docker ps -a --filter 'label=com.docker.compose.service=${applicationId}' --format {{.ID}}`
})
});
if (containers) {
const containerArray = containers.split('\n');
if (containerArray.length > 0) {
for (const container of containerArray) {
await executeCommand({ dockerId: destinationDockerId, command: `docker stop -t 0 ${container}` })
await executeCommand({ dockerId: destinationDockerId, command: `docker rm --force ${container}` })
await executeCommand({
dockerId: destinationDockerId,
command: `docker stop -t 0 ${container}`
});
await executeCommand({
dockerId: destinationDockerId,
command: `docker rm --force ${container}`
});
}
}
}
} catch (error) {
//
}
const envs = [
`PORT=${port}`
];
const envs = [`PORT=${port}`];
if (secrets.length > 0) {
secrets.forEach((secret) => {
if (pullmergeRequestId) {
const isSecretFound = secrets.filter(s => s.name === secret.name && s.isPRMRSecret)
const isSecretFound = secrets.filter(
(s) => s.name === secret.name && s.isPRMRSecret
);
if (isSecretFound.length > 0) {
envs.push(`${secret.name}=${isSecretFound[0].value}`);
envs.push(
`${secret.name}="${escapeStringRegexp(
isSecretFound[0].value.replace(/[\\$'"]/g, '\\$&')
)}"`
);
} else {
envs.push(`${secret.name}=${secret.value}`);
envs.push(
`${secret.name}="${escapeStringRegexp(
secret.value.replace(/[\\$'"]/g, '\\$&')
)}"`
);
}
} else {
if (!secret.isPRMRSecret) {
envs.push(`${secret.name}=${secret.value}`);
envs.push(
`${secret.name}="${escapeStringRegexp(
secret.value.replace(/[\\$'"]/g, '\\$&')
)}"`
);
}
}
});
@ -115,14 +182,14 @@ import * as buildpacks from '../lib/buildPacks';
await fs.writeFile(`${workdir}/Dockerfile`, simpleDockerfile);
if (dockerRegistry) {
const { url, username, password } = dockerRegistry
await saveDockerRegistryCredentials({ url, username, password, workdir })
const { url, username, password } = dockerRegistry;
await saveDockerRegistryCredentials({ url, username, password, workdir });
}
const labels = makeLabelForSimpleDockerfile({
applicationId,
type,
port: exposePort ? `${exposePort}:${port}` : port,
port: exposePort ? `${exposePort}:${port}` : port
});
try {
const composeVolumes = volumes.map((volume) => {
@ -137,7 +204,7 @@ import * as buildpacks from '../lib/buildPacks';
services: {
[applicationId]: {
build: {
context: workdir,
context: workdir
},
image: `${applicationId}:${buildId}`,
container_name: applicationId,
@ -147,7 +214,7 @@ import * as buildpacks from '../lib/buildPacks';
depends_on: [],
expose: [port],
...(exposePort ? { ports: [`${exposePort}:${port}`] } : {}),
...defaultComposeConfiguration(destinationDocker.network),
...defaultComposeConfiguration(destinationDocker.network)
}
},
networks: {
@ -158,11 +225,15 @@ import * as buildpacks from '../lib/buildPacks';
volumes: Object.assign({}, ...composeVolumes)
};
await fs.writeFile(`${workdir}/docker-compose.yml`, yaml.dump(composeFile));
await executeCommand({ debug: true, dockerId: destinationDocker.id, command: `docker compose --project-directory ${workdir} up -d` })
await executeCommand({
debug: true,
dockerId: destinationDocker.id,
command: `docker compose --project-directory ${workdir} up -d`
});
await saveBuildLog({ line: 'Deployed 🎉', buildId, applicationId });
} catch (error) {
await saveBuildLog({ line: error, buildId, applicationId });
const foundBuild = await prisma.build.findUnique({ where: { id: buildId } })
const foundBuild = await prisma.build.findUnique({ where: { id: buildId } });
if (foundBuild) {
await prisma.build.update({
where: { id: buildId },
@ -173,10 +244,9 @@ import * as buildpacks from '../lib/buildPacks';
}
throw new Error(error);
}
}
} catch (error) {
const foundBuild = await prisma.build.findUnique({ where: { id: buildId } })
const foundBuild = await prisma.build.findUnique({ where: { id: buildId } });
if (foundBuild) {
await prisma.build.update({
where: { id: buildId },
@ -189,7 +259,11 @@ import * as buildpacks from '../lib/buildPacks';
await saveBuildLog({ line: error, buildId, applicationId: application.id });
}
if (error instanceof Error) {
await saveBuildLog({ line: error.message, buildId, applicationId: application.id });
await saveBuildLog({
line: error.message,
buildId,
applicationId: application.id
});
}
await fs.rm(workdir, { recursive: true, force: true });
return;
@ -198,9 +272,13 @@ import * as buildpacks from '../lib/buildPacks';
if (application.dockerRegistryImageName) {
const customTag = application.dockerRegistryImageName.split(':')[1] || buildId;
const imageName = application.dockerRegistryImageName.split(':')[0];
await saveBuildLog({ line: `Pushing ${imageName}:${customTag} to Docker Registry... It could take a while...`, buildId, applicationId: application.id });
await pushToRegistry(application, workdir, buildId, imageName, customTag)
await saveBuildLog({ line: "Success", buildId, applicationId: application.id });
await saveBuildLog({
line: `Pushing ${imageName}:${customTag} to Docker Registry... It could take a while...`,
buildId,
applicationId: application.id
});
await pushToRegistry(application, workdir, buildId, imageName, customTag);
await saveBuildLog({ line: 'Success', buildId, applicationId: application.id });
}
} catch (error) {
if (error.stdout) {
@ -211,12 +289,15 @@ import * as buildpacks from '../lib/buildPacks';
}
} finally {
await fs.rm(workdir, { recursive: true, force: true });
await prisma.build.update({ where: { id: buildId }, data: { status: 'success' } });
await prisma.build.update({
where: { id: buildId },
data: { status: 'success' }
});
}
return;
}
const originalApplicationId = application.id
const originalApplicationId = application.id;
const {
id: applicationId,
name,
@ -240,7 +321,7 @@ import * as buildpacks from '../lib/buildPacks';
deploymentType,
gitCommitHash,
dockerRegistry
} = application
} = application;
let {
branch,
@ -256,7 +337,7 @@ import * as buildpacks from '../lib/buildPacks';
dockerComposeFileLocation,
dockerComposeConfiguration,
denoMainFile
} = application
} = application;
let imageId = applicationId;
let domain = getDomain(fqdn);
@ -271,9 +352,11 @@ import * as buildpacks from '../lib/buildPacks';
let imageFoundRemotely = false;
if (pullmergeRequestId) {
const previewApplications = await prisma.previewApplication.findMany({ where: { applicationId: originalApplicationId, pullmergeRequestId } })
const previewApplications = await prisma.previewApplication.findMany({
where: { applicationId: originalApplicationId, pullmergeRequestId }
});
if (previewApplications.length > 0) {
previewApplicationId = previewApplications[0].id
previewApplicationId = previewApplications[0].id;
}
// Previews, we need to get the source branch and set subdomain
branch = sourceBranch;
@ -284,7 +367,11 @@ import * as buildpacks from '../lib/buildPacks';
const { workdir, repodir } = await createDirectories({ repository, buildId });
try {
if (queueBuild.status === 'running') {
await saveBuildLog({ line: 'Building halted, restarting...', buildId, applicationId: application.id });
await saveBuildLog({
line: 'Building halted, restarting...',
buildId,
applicationId: application.id
});
}
const currentHash = crypto
@ -322,15 +409,16 @@ import * as buildpacks from '../lib/buildPacks';
const volumes =
persistentStorage?.map((storage) => {
if (storage.oldPath) {
return `${applicationId}${storage.path.replace(/\//gi, '-').replace('-app', '')}:${storage.path}`;
return `${applicationId}${storage.path
.replace(/\//gi, '-')
.replace('-app', '')}:${storage.path}`;
}
return `${applicationId}${storage.path.replace(/\//gi, '-')}:${storage.path}`;
}) || [];
try {
dockerComposeConfiguration = JSON.parse(dockerComposeConfiguration)
} catch (error) { }
dockerComposeConfiguration = JSON.parse(dockerComposeConfiguration);
} catch (error) {}
let deployNeeded = true;
let destinationType;
@ -338,7 +426,10 @@ import * as buildpacks from '../lib/buildPacks';
destinationType = 'docker';
}
if (destinationType === 'docker') {
await prisma.build.update({ where: { id: buildId }, data: { status: 'running' } });
await prisma.build.update({
where: { id: buildId },
data: { status: 'running' }
});
const configuration = await setDefaultConfiguration(application);
@ -380,10 +471,10 @@ import * as buildpacks from '../lib/buildPacks';
tag = `${commit.slice(0, 7)}-${pullmergeRequestId}`;
}
if (application.dockerRegistryImageName) {
imageName = application.dockerRegistryImageName.split(':')[0]
customTag = application.dockerRegistryImageName.split(':')[1] || tag
imageName = application.dockerRegistryImageName.split(':')[0];
customTag = application.dockerRegistryImageName.split(':')[1] || tag;
} else {
customTag = tag
customTag = tag;
imageName = applicationId;
}
@ -393,13 +484,17 @@ import * as buildpacks from '../lib/buildPacks';
try {
await prisma.build.update({ where: { id: buildId }, data: { commit } });
} catch (err) { }
} catch (err) {}
if (!pullmergeRequestId) {
if (configHash !== currentHash) {
deployNeeded = true;
if (configHash) {
await saveBuildLog({ line: 'Configuration changed', buildId, applicationId });
await saveBuildLog({
line: 'Configuration changed',
buildId,
applicationId
});
}
} else {
deployNeeded = false;
@ -412,31 +507,44 @@ import * as buildpacks from '../lib/buildPacks';
await executeCommand({
dockerId: destinationDocker.id,
command: `docker image inspect ${applicationId}:${tag}`
})
});
imageFoundLocally = true;
} catch (error) {
//
}
if (dockerRegistry) {
const { url, username, password } = dockerRegistry
location = await saveDockerRegistryCredentials({ url, username, password, workdir })
const { url, username, password } = dockerRegistry;
location = await saveDockerRegistryCredentials({
url,
username,
password,
workdir
});
}
try {
await executeCommand({
dockerId: destinationDocker.id,
command: `docker ${location ? `--config ${location}` : ''} pull ${imageName}:${customTag}`
})
command: `docker ${
location ? `--config ${location}` : ''
} pull ${imageName}:${customTag}`
});
imageFoundRemotely = true;
} catch (error) {
//
}
let imageFound = `${applicationId}:${tag}`
let imageFound = `${applicationId}:${tag}`;
if (imageFoundRemotely) {
imageFound = `${imageName}:${customTag}`
imageFound = `${imageName}:${customTag}`;
}
await copyBaseConfigurationFiles(buildPack, workdir, buildId, applicationId, baseImage);
console.log({secrets})
await copyBaseConfigurationFiles(
buildPack,
workdir,
buildId,
applicationId,
baseImage
);
console.log({ secrets });
const labels = makeLabelForStandaloneApplication({
applicationId,
fqdn,
@ -455,7 +563,7 @@ import * as buildpacks from '../lib/buildPacks';
baseDirectory,
publishDirectory
});
if (forceRebuild) deployNeeded = true
if (forceRebuild) deployNeeded = true;
if ((!imageFoundLocally && !imageFoundRemotely) || deployNeeded) {
if (buildpacks[buildPack])
await buildpacks[buildPack]({
@ -495,18 +603,30 @@ import * as buildpacks from '../lib/buildPacks';
denoOptions,
baseImage,
baseBuildImage,
deploymentType,
deploymentType
});
else {
await saveBuildLog({ line: `Build pack ${buildPack} not found`, buildId, applicationId });
await saveBuildLog({
line: `Build pack ${buildPack} not found`,
buildId,
applicationId
});
throw new Error(`Build pack ${buildPack} not found.`);
}
} else {
if (imageFoundRemotely || deployNeeded) {
await saveBuildLog({ line: `Container image ${imageFound} found in Docker Registry - reuising it`, buildId, applicationId });
await saveBuildLog({
line: `Container image ${imageFound} found in Docker Registry - reuising it`,
buildId,
applicationId
});
} else {
if (imageFoundLocally || deployNeeded) {
await saveBuildLog({ line: `Container image ${imageFound} found locally - reuising it`, buildId, applicationId });
await saveBuildLog({
line: `Container image ${imageFound} found locally - reuising it`,
buildId,
applicationId
});
}
}
}
@ -516,13 +636,19 @@ import * as buildpacks from '../lib/buildPacks';
const { stdout: containers } = await executeCommand({
dockerId: destinationDockerId,
command: `docker ps -a --filter 'label=coolify.applicationId=${applicationId}' --format {{.ID}}`
})
});
if (containers) {
const containerArray = containers.split('\n');
if (containerArray.length > 0) {
for (const container of containerArray) {
await executeCommand({ dockerId: destinationDockerId, command: `docker stop -t 0 ${container}` })
await executeCommand({ dockerId: destinationDockerId, command: `docker rm --force ${container}` })
await executeCommand({
dockerId: destinationDockerId,
command: `docker stop -t 0 ${container}`
});
await executeCommand({
dockerId: destinationDockerId,
command: `docker rm --force ${container}`
});
}
}
}
@ -530,16 +656,25 @@ import * as buildpacks from '../lib/buildPacks';
//
}
try {
await executeCommand({ debug, buildId, applicationId, dockerId: destinationDocker.id, command: `docker compose --project-directory ${workdir} up -d` })
await executeCommand({
debug,
buildId,
applicationId,
dockerId: destinationDocker.id,
command: `docker compose --project-directory ${workdir} up -d`
});
await saveBuildLog({ line: 'Deployed 🎉', buildId, applicationId });
await prisma.build.update({ where: { id: buildId }, data: { status: 'success' } });
await prisma.build.update({
where: { id: buildId },
data: { status: 'success' }
});
await prisma.application.update({
where: { id: applicationId },
data: { configHash: currentHash }
});
} catch (error) {
await saveBuildLog({ line: error, buildId, applicationId });
const foundBuild = await prisma.build.findUnique({ where: { id: buildId } })
const foundBuild = await prisma.build.findUnique({ where: { id: buildId } });
if (foundBuild) {
await prisma.build.update({
where: { id: buildId },
@ -550,32 +685,39 @@ import * as buildpacks from '../lib/buildPacks';
}
throw new Error(error);
}
} else {
try {
const { stdout: containers } = await executeCommand({
dockerId: destinationDockerId,
command: `docker ps -a --filter 'label=com.docker.compose.service=${pullmergeRequestId ? imageId : applicationId}' --format {{.ID}}`
})
command: `docker ps -a --filter 'label=com.docker.compose.service=${
pullmergeRequestId ? imageId : applicationId
}' --format {{.ID}}`
});
if (containers) {
const containerArray = containers.split('\n');
if (containerArray.length > 0) {
for (const container of containerArray) {
await executeCommand({ dockerId: destinationDockerId, command: `docker stop -t 0 ${container}` })
await executeCommand({ dockerId: destinationDockerId, command: `docker rm --force ${container}` })
await executeCommand({
dockerId: destinationDockerId,
command: `docker stop -t 0 ${container}`
});
await executeCommand({
dockerId: destinationDockerId,
command: `docker rm --force ${container}`
});
}
}
}
} catch (error) {
//
}
const envs = [
`PORT=${port}`
];
const envs = [`PORT=${port}`];
if (secrets.length > 0) {
secrets.forEach((secret) => {
if (pullmergeRequestId) {
const isSecretFound = secrets.filter(s => s.name === secret.name && s.isPRMRSecret)
const isSecretFound = secrets.filter(
(s) => s.name === secret.name && s.isPRMRSecret
);
if (isSecretFound.length > 0) {
envs.push(`${secret.name}=${isSecretFound[0].value}`);
} else {
@ -588,11 +730,11 @@ import * as buildpacks from '../lib/buildPacks';
}
});
}
console.log({envs})
console.log({ envs });
await fs.writeFile(`${workdir}/.env`, envs.join('\n'));
if (dockerRegistry) {
const { url, username, password } = dockerRegistry
await saveDockerRegistryCredentials({ url, username, password, workdir })
const { url, username, password } = dockerRegistry;
await saveDockerRegistryCredentials({ url, username, password, workdir });
}
let envFound = false;
@ -621,7 +763,7 @@ import * as buildpacks from '../lib/buildPacks';
depends_on: [],
expose: [port],
...(exposePort ? { ports: [`${exposePort}:${port}`] } : {}),
...defaultComposeConfiguration(destinationDocker.network),
...defaultComposeConfiguration(destinationDocker.network)
}
},
networks: {
@ -632,11 +774,15 @@ import * as buildpacks from '../lib/buildPacks';
volumes: Object.assign({}, ...composeVolumes)
};
await fs.writeFile(`${workdir}/docker-compose.yml`, yaml.dump(composeFile));
await executeCommand({ debug, dockerId: destinationDocker.id, command: `docker compose --project-directory ${workdir} up -d` })
await executeCommand({
debug,
dockerId: destinationDocker.id,
command: `docker compose --project-directory ${workdir} up -d`
});
await saveBuildLog({ line: 'Deployed 🎉', buildId, applicationId });
} catch (error) {
await saveBuildLog({ line: error, buildId, applicationId });
const foundBuild = await prisma.build.findUnique({ where: { id: buildId } })
const foundBuild = await prisma.build.findUnique({ where: { id: buildId } });
if (foundBuild) {
await prisma.build.update({
where: { id: buildId },
@ -648,14 +794,15 @@ import * as buildpacks from '../lib/buildPacks';
throw new Error(error);
}
if (!pullmergeRequestId) await prisma.application.update({
if (!pullmergeRequestId)
await prisma.application.update({
where: { id: applicationId },
data: { configHash: currentHash }
});
}
}
} catch (error) {
const foundBuild = await prisma.build.findUnique({ where: { id: buildId } })
const foundBuild = await prisma.build.findUnique({ where: { id: buildId } });
if (foundBuild) {
await prisma.build.update({
where: { id: buildId },
@ -668,16 +815,24 @@ import * as buildpacks from '../lib/buildPacks';
await saveBuildLog({ line: error, buildId, applicationId: application.id });
}
if (error instanceof Error) {
await saveBuildLog({ line: error.message, buildId, applicationId: application.id });
await saveBuildLog({
line: error.message,
buildId,
applicationId: application.id
});
}
await fs.rm(workdir, { recursive: true, force: true });
return;
}
try {
if (application.dockerRegistryImageName && (!imageFoundRemotely || forceRebuild)) {
await saveBuildLog({ line: `Pushing ${imageName}:${customTag} to Docker Registry... It could take a while...`, buildId, applicationId: application.id });
await pushToRegistry(application, workdir, tag, imageName, customTag)
await saveBuildLog({ line: "Success", buildId, applicationId: application.id });
await saveBuildLog({
line: `Pushing ${imageName}:${customTag} to Docker Registry... It could take a while...`,
buildId,
applicationId: application.id
});
await pushToRegistry(application, workdir, tag, imageName, customTag);
await saveBuildLog({ line: 'Success', buildId, applicationId: application.id });
}
} catch (error) {
if (error.stdout) {
@ -686,21 +841,20 @@ import * as buildpacks from '../lib/buildPacks';
if (error.stderr) {
await saveBuildLog({ line: error.stderr, buildId, applicationId });
}
} finally {
await fs.rm(workdir, { recursive: true, force: true });
await prisma.build.update({ where: { id: buildId }, data: { status: 'success' } });
}
});
}
await pAll.default(actions, { concurrency })
await pAll.default(actions, { concurrency });
}
} catch (error) {
console.log(error)
console.log(error);
}
})
});
while (true) {
await th()
await th();
}
} else process.exit(0);
})();