fix: escape secrets

This commit is contained in:
Andras Bacsai 2022-12-19 10:04:28 +01:00
parent a70adc5eb3
commit 4261147fe8
4 changed files with 184 additions and 136 deletions

View File

@ -1,85 +1,86 @@
{
"name": "api",
"description": "Coolify's Fastify API",
"license": "Apache-2.0",
"scripts": {
"db:generate": "prisma generate",
"db:push": "prisma db push && prisma generate",
"db:seed": "prisma db seed",
"db:studio": "prisma studio",
"db:migrate": "COOLIFY_DATABASE_URL=file:../db/migration.db prisma migrate dev --skip-seed --name",
"dev": "nodemon",
"build": "rimraf build && esbuild `find src \\( -name '*.ts' \\)| grep -v client/` --platform=node --outdir=build --format=cjs",
"format": "prettier --write 'src/**/*.{js,ts,json,md}'",
"lint": "prettier --check 'src/**/*.{js,ts,json,md}' && eslint --ignore-path .eslintignore .",
"start": "NODE_ENV=production pnpm prisma migrate deploy && pnpm prisma generate && pnpm prisma db seed && node index.js"
},
"dependencies": {
"@breejs/ts-worker": "2.0.0",
"@fastify/autoload": "5.5.0",
"@fastify/cookie": "8.3.0",
"@fastify/cors": "8.2.0",
"@fastify/env": "4.1.0",
"@fastify/jwt": "6.3.3",
"@fastify/multipart": "7.3.0",
"@fastify/static": "6.5.1",
"@iarna/toml": "2.2.5",
"@ladjs/graceful": "3.0.2",
"@prisma/client": "4.6.1",
"@sentry/node": "7.21.1",
"@sentry/tracing": "7.21.1",
"axe": "11.0.0",
"bcryptjs": "2.4.3",
"bree": "9.1.2",
"cabin": "11.0.1",
"compare-versions": "5.0.1",
"csv-parse": "5.3.2",
"csvtojson": "2.0.10",
"cuid": "2.1.8",
"dayjs": "1.11.6",
"dockerode": "3.3.4",
"dotenv-extended": "2.9.0",
"execa": "6.1.0",
"fastify": "4.10.2",
"fastify-plugin": "4.3.0",
"fastify-socket.io": "4.0.0",
"generate-password": "1.7.0",
"got": "12.5.3",
"is-ip": "5.0.0",
"is-port-reachable": "4.0.0",
"js-yaml": "4.1.0",
"jsonwebtoken": "8.5.1",
"minimist": "^1.2.7",
"node-forge": "1.3.1",
"node-os-utils": "1.3.7",
"p-all": "4.0.0",
"p-throttle": "5.0.0",
"prisma": "4.6.1",
"public-ip": "6.0.1",
"pump": "3.0.0",
"shell-quote": "^1.7.4",
"socket.io": "4.5.3",
"ssh-config": "4.1.6",
"strip-ansi": "7.0.1",
"unique-names-generator": "4.7.1"
},
"devDependencies": {
"@types/node": "18.11.9",
"@types/node-os-utils": "1.3.0",
"@typescript-eslint/eslint-plugin": "5.44.0",
"@typescript-eslint/parser": "5.44.0",
"esbuild": "0.15.15",
"eslint": "8.28.0",
"eslint-config-prettier": "8.5.0",
"eslint-plugin-prettier": "4.2.1",
"nodemon": "2.0.20",
"prettier": "2.7.1",
"rimraf": "3.0.2",
"tsconfig-paths": "4.1.0",
"types-fastify-socket.io": "0.0.1",
"typescript": "4.9.3"
},
"prisma": {
"seed": "node prisma/seed.js"
}
}
"name": "api",
"description": "Coolify's Fastify API",
"license": "Apache-2.0",
"scripts": {
"db:generate": "prisma generate",
"db:push": "prisma db push && prisma generate",
"db:seed": "prisma db seed",
"db:studio": "prisma studio",
"db:migrate": "COOLIFY_DATABASE_URL=file:../db/migration.db prisma migrate dev --skip-seed --name",
"dev": "nodemon",
"build": "rimraf build && esbuild `find src \\( -name '*.ts' \\)| grep -v client/` --platform=node --outdir=build --format=cjs",
"format": "prettier --write 'src/**/*.{js,ts,json,md}'",
"lint": "prettier --check 'src/**/*.{js,ts,json,md}' && eslint --ignore-path .eslintignore .",
"start": "NODE_ENV=production pnpm prisma migrate deploy && pnpm prisma generate && pnpm prisma db seed && node index.js"
},
"dependencies": {
"@breejs/ts-worker": "2.0.0",
"@fastify/autoload": "5.5.0",
"@fastify/cookie": "8.3.0",
"@fastify/cors": "8.2.0",
"@fastify/env": "4.1.0",
"@fastify/jwt": "6.3.3",
"@fastify/multipart": "7.3.0",
"@fastify/static": "6.5.1",
"@iarna/toml": "2.2.5",
"@ladjs/graceful": "3.0.2",
"@prisma/client": "4.6.1",
"@sentry/node": "7.21.1",
"@sentry/tracing": "7.21.1",
"axe": "11.0.0",
"bcryptjs": "2.4.3",
"bree": "9.1.2",
"cabin": "11.0.1",
"compare-versions": "5.0.1",
"csv-parse": "5.3.2",
"csvtojson": "2.0.10",
"cuid": "2.1.8",
"dayjs": "1.11.6",
"dockerode": "3.3.4",
"dotenv-extended": "2.9.0",
"escape-string-regexp": "5.0.0",
"execa": "6.1.0",
"fastify": "4.10.2",
"fastify-plugin": "4.3.0",
"fastify-socket.io": "4.0.0",
"generate-password": "1.7.0",
"got": "12.5.3",
"is-ip": "5.0.0",
"is-port-reachable": "4.0.0",
"js-yaml": "4.1.0",
"jsonwebtoken": "8.5.1",
"minimist": "^1.2.7",
"node-forge": "1.3.1",
"node-os-utils": "1.3.7",
"p-all": "4.0.0",
"p-throttle": "5.0.0",
"prisma": "4.6.1",
"public-ip": "6.0.1",
"pump": "3.0.0",
"shell-quote": "^1.7.4",
"socket.io": "4.5.3",
"ssh-config": "4.1.6",
"strip-ansi": "7.0.1",
"unique-names-generator": "4.7.1"
},
"devDependencies": {
"@types/node": "18.11.9",
"@types/node-os-utils": "1.3.0",
"@typescript-eslint/eslint-plugin": "5.44.0",
"@typescript-eslint/parser": "5.44.0",
"esbuild": "0.15.15",
"eslint": "8.28.0",
"eslint-config-prettier": "8.5.0",
"eslint-plugin-prettier": "4.2.1",
"nodemon": "2.0.20",
"prettier": "2.7.1",
"rimraf": "3.0.2",
"tsconfig-paths": "4.1.0",
"types-fastify-socket.io": "0.0.1",
"typescript": "4.9.3"
},
"prisma": {
"seed": "node prisma/seed.js"
}
}

View File

@ -1,6 +1,17 @@
import { base64Encode, decrypt, encrypt, executeCommand, generateTimestamp, getDomain, isARM, isDev, prisma, version } from "../common";
import {
base64Encode,
decrypt,
encrypt,
executeCommand,
generateTimestamp,
getDomain,
isARM,
isDev,
prisma,
version
} from '../common';
import { promises as fs } from 'fs';
import { day } from "../dayjs";
import { day } from '../dayjs';
const staticApps = ['static', 'react', 'vuejs', 'svelte', 'gatsby', 'astro', 'eleventy'];
const nodeBased = [
@ -17,7 +28,10 @@ const nodeBased = [
'nextjs'
];
export function setDefaultBaseImage(buildPack: string | null, deploymentType: string | null = null) {
export function setDefaultBaseImage(
buildPack: string | null,
deploymentType: string | null = null
) {
const nodeVersions = [
{
value: 'node:lts',
@ -316,8 +330,8 @@ export function setDefaultBaseImage(buildPack: string | null, deploymentType: st
{
value: 'heroku/builder-classic:22',
label: 'heroku/builder-classic:22'
},
]
}
];
let payload: any = {
baseImage: null,
baseBuildImage: null,
@ -327,7 +341,9 @@ export function setDefaultBaseImage(buildPack: string | null, deploymentType: st
if (nodeBased.includes(buildPack)) {
if (deploymentType === 'static') {
payload.baseImage = isARM(process.arch) ? 'nginx:alpine' : 'webdevops/nginx:alpine';
payload.baseImages = isARM(process.arch) ? staticVersions.filter((version) => !version.value.includes('webdevops')) : staticVersions;
payload.baseImages = isARM(process.arch)
? staticVersions.filter((version) => !version.value.includes('webdevops'))
: staticVersions;
payload.baseBuildImage = 'node:lts';
payload.baseBuildImages = nodeVersions;
} else {
@ -339,7 +355,9 @@ export function setDefaultBaseImage(buildPack: string | null, deploymentType: st
}
if (staticApps.includes(buildPack)) {
payload.baseImage = isARM(process.arch) ? 'nginx:alpine' : 'webdevops/nginx:alpine';
payload.baseImages = isARM(process.arch) ? staticVersions.filter((version) => !version.value.includes('webdevops')) : staticVersions;
payload.baseImages = isARM(process.arch)
? staticVersions.filter((version) => !version.value.includes('webdevops'))
: staticVersions;
payload.baseBuildImage = 'node:lts';
payload.baseBuildImages = nodeVersions;
}
@ -357,12 +375,20 @@ export function setDefaultBaseImage(buildPack: string | null, deploymentType: st
payload.baseImage = 'denoland/deno:latest';
}
if (buildPack === 'php') {
payload.baseImage = isARM(process.arch) ? 'php:8.1-fpm-alpine' : 'webdevops/php-apache:8.2-alpine';
payload.baseImages = isARM(process.arch) ? phpVersions.filter((version) => !version.value.includes('webdevops')) : phpVersions
payload.baseImage = isARM(process.arch)
? 'php:8.1-fpm-alpine'
: 'webdevops/php-apache:8.2-alpine';
payload.baseImages = isARM(process.arch)
? phpVersions.filter((version) => !version.value.includes('webdevops'))
: phpVersions;
}
if (buildPack === 'laravel') {
payload.baseImage = isARM(process.arch) ? 'php:8.1-fpm-alpine' : 'webdevops/php-apache:8.2-alpine';
payload.baseImages = isARM(process.arch) ? phpVersions.filter((version) => !version.value.includes('webdevops')) : phpVersions
payload.baseImage = isARM(process.arch)
? 'php:8.1-fpm-alpine'
: 'webdevops/php-apache:8.2-alpine';
payload.baseImages = isARM(process.arch)
? phpVersions.filter((version) => !version.value.includes('webdevops'))
: phpVersions;
payload.baseBuildImage = 'node:18';
payload.baseBuildImages = nodeVersions;
}
@ -405,7 +431,8 @@ export const setDefaultConfiguration = async (data: any) => {
if (!publishDirectory) publishDirectory = template?.publishDirectory || null;
if (baseDirectory) {
if (!baseDirectory.startsWith('/')) baseDirectory = `/${baseDirectory}`;
if (baseDirectory.endsWith('/') && baseDirectory !== '/') baseDirectory = baseDirectory.slice(0, -1);
if (baseDirectory.endsWith('/') && baseDirectory !== '/')
baseDirectory = baseDirectory.slice(0, -1);
}
if (dockerFileLocation) {
if (!dockerFileLocation.startsWith('/')) dockerFileLocation = `/${dockerFileLocation}`;
@ -414,8 +441,10 @@ export const setDefaultConfiguration = async (data: any) => {
dockerFileLocation = '/Dockerfile';
}
if (dockerComposeFileLocation) {
if (!dockerComposeFileLocation.startsWith('/')) dockerComposeFileLocation = `/${dockerComposeFileLocation}`;
if (dockerComposeFileLocation.endsWith('/')) dockerComposeFileLocation = dockerComposeFileLocation.slice(0, -1);
if (!dockerComposeFileLocation.startsWith('/'))
dockerComposeFileLocation = `/${dockerComposeFileLocation}`;
if (dockerComposeFileLocation.endsWith('/'))
dockerComposeFileLocation = dockerComposeFileLocation.slice(0, -1);
} else {
dockerComposeFileLocation = '/Dockerfile';
}
@ -479,7 +508,6 @@ export const scanningTemplates = {
}
};
export const saveBuildLog = async ({
line,
buildId,
@ -491,7 +519,7 @@ export const saveBuildLog = async ({
}): Promise<any> => {
if (buildId === 'undefined' || buildId === 'null' || !buildId) return;
if (applicationId === 'undefined' || applicationId === 'null' || !applicationId) return;
const { default: got } = await import('got')
const { default: got } = await import('got');
if (typeof line === 'object' && line) {
if (line.shortMessage) {
line = line.shortMessage + '\n' + line.stderr;
@ -504,7 +532,11 @@ export const saveBuildLog = async ({
line = line.replace(regex, '<SENSITIVE_DATA_DELETED>@');
}
const addTimestamp = `[${generateTimestamp()}] ${line}`;
const fluentBitUrl = isDev ? process.env.COOLIFY_CONTAINER_DEV === 'true' ? 'http://coolify-fluentbit:24224' : 'http://localhost:24224' : 'http://coolify-fluentbit:24224';
const fluentBitUrl = isDev
? process.env.COOLIFY_CONTAINER_DEV === 'true'
? 'http://coolify-fluentbit:24224'
: 'http://localhost:24224'
: 'http://coolify-fluentbit:24224';
if (isDev && !process.env.COOLIFY_CONTAINER_DEV) {
console.debug(`[${applicationId}] ${addTimestamp}`);
@ -514,15 +546,17 @@ export const saveBuildLog = async ({
json: {
line: encrypt(line)
}
})
});
} catch (error) {
return await prisma.buildLog.create({
data: {
line: addTimestamp, buildId, time: Number(day().valueOf()), applicationId
line: addTimestamp,
buildId,
time: Number(day().valueOf()),
applicationId
}
});
}
};
export async function copyBaseConfigurationFiles(
@ -610,7 +644,7 @@ export function checkPnpm(installCommand = null, buildCommand = null, startComma
export async function saveDockerRegistryCredentials({ url, username, password, workdir }) {
if (!username || !password) {
return null
return null;
}
let decryptedPassword = decrypt(password);
@ -622,14 +656,14 @@ export async function saveDockerRegistryCredentials({ url, username, password, w
console.log(error);
}
const payload = JSON.stringify({
"auths": {
auths: {
[url]: {
"auth": Buffer.from(`${username}:${decryptedPassword}`).toString('base64')
auth: Buffer.from(`${username}:${decryptedPassword}`).toString('base64')
}
}
})
await fs.writeFile(`${location}/config.json`, payload)
return location
});
await fs.writeFile(`${location}/config.json`, payload);
return location;
}
export async function buildImage({
applicationId,
@ -647,22 +681,34 @@ export async function buildImage({
} else {
await saveBuildLog({ line: `Building production image...`, buildId, applicationId });
}
const dockerFile = isCache ? `${dockerFileLocation}-cache` : `${dockerFileLocation}`
const cache = `${applicationId}:${tag}${isCache ? '-cache' : ''}`
const dockerFile = isCache ? `${dockerFileLocation}-cache` : `${dockerFileLocation}`;
const cache = `${applicationId}:${tag}${isCache ? '-cache' : ''}`;
let location = null
let location = null;
const { dockerRegistry } = await prisma.application.findUnique({ where: { id: applicationId }, select: { dockerRegistry: true } })
const { dockerRegistry } = await prisma.application.findUnique({
where: { id: applicationId },
select: { dockerRegistry: true }
});
if (dockerRegistry) {
const { url, username, password } = dockerRegistry
location = await saveDockerRegistryCredentials({ url, username, password, workdir })
const { url, username, password } = dockerRegistry;
location = await saveDockerRegistryCredentials({ url, username, password, workdir });
}
await executeCommand({ stream: true, debug, buildId, applicationId, dockerId, command: `docker ${location ? `--config ${location}` : ''} build --progress plain -f ${workdir}/${dockerFile} -t ${cache} --build-arg SOURCE_COMMIT=${commit} ${workdir}` })
await executeCommand({
stream: true,
debug,
buildId,
applicationId,
dockerId,
command: `docker ${
location ? `--config ${location}` : ''
} build --progress plain -f ${workdir}/${dockerFile} -t ${cache} --build-arg SOURCE_COMMIT=${commit} ${workdir}`
});
const { status } = await prisma.build.findUnique({ where: { id: buildId } })
const { status } = await prisma.build.findUnique({ where: { id: buildId } });
if (status === 'canceled') {
throw new Error('Canceled.')
throw new Error('Canceled.');
}
}
export function makeLabelForSimpleDockerfile({ applicationId, port, type }) {
@ -726,6 +772,7 @@ export function makeLabelForStandaloneApplication({
}
export async function buildCacheImageWithNode(data, imageForBuild) {
const { default: escapeStringRegexp } = await import('escape-string-regexp');
const {
workdir,
buildId,
@ -744,15 +791,15 @@ export async function buildCacheImageWithNode(data, imageForBuild) {
secrets.forEach((secret) => {
if (secret.isBuildSecret) {
if (pullmergeRequestId) {
const isSecretFound = secrets.filter(s => s.name === secret.name && s.isPRMRSecret)
const isSecretFound = secrets.filter((s) => s.name === secret.name && s.isPRMRSecret);
if (isSecretFound.length > 0) {
Dockerfile.push(`ARG ${secret.name}=${isSecretFound[0].value}`);
Dockerfile.push(`ARG ${secret.name}=${escapeStringRegexp(isSecretFound[0].value)}`);
} else {
Dockerfile.push(`ARG ${secret.name}=${secret.value}`);
Dockerfile.push(`ARG ${secret.name}=${escapeStringRegexp(secret.value)}`);
}
} else {
if (!secret.isPRMRSecret) {
Dockerfile.push(`ARG ${secret.name}=${secret.value}`);
Dockerfile.push(`ARG ${secret.name}=${escapeStringRegexp(secret.value)}`);
}
}
}
@ -772,6 +819,7 @@ export async function buildCacheImageWithNode(data, imageForBuild) {
}
export async function buildCacheImageForLaravel(data, imageForBuild) {
const { default: escapeStringRegexp } = await import('escape-string-regexp');
const { workdir, buildId, secrets, pullmergeRequestId } = data;
const Dockerfile: Array<string> = [];
@ -782,15 +830,15 @@ export async function buildCacheImageForLaravel(data, imageForBuild) {
secrets.forEach((secret) => {
if (secret.isBuildSecret) {
if (pullmergeRequestId) {
const isSecretFound = secrets.filter(s => s.name === secret.name && s.isPRMRSecret)
const isSecretFound = secrets.filter((s) => s.name === secret.name && s.isPRMRSecret);
if (isSecretFound.length > 0) {
Dockerfile.push(`ARG ${secret.name}=${isSecretFound[0].value}`);
Dockerfile.push(`ARG ${secret.name}=${escapeStringRegexp(isSecretFound[0].value)}`);
} else {
Dockerfile.push(`ARG ${secret.name}=${secret.value}`);
Dockerfile.push(`ARG ${secret.name}=${escapeStringRegexp(secret.value)}`);
}
} else {
if (!secret.isPRMRSecret) {
Dockerfile.push(`ARG ${secret.name}=${secret.value}`);
Dockerfile.push(`ARG ${secret.name}=${escapeStringRegexp(secret.value)}`);
}
}
}
@ -804,11 +852,7 @@ export async function buildCacheImageForLaravel(data, imageForBuild) {
}
export async function buildCacheImageWithCargo(data, imageForBuild) {
const {
applicationId,
workdir,
buildId,
} = data;
const { applicationId, workdir, buildId } = data;
const Dockerfile: Array<string> = [];
Dockerfile.push(`FROM ${imageForBuild} as planner-${applicationId}`);

View File

@ -2,6 +2,7 @@ import { promises as fs } from 'fs';
import { buildImage, checkPnpm } from './common';
const createDockerfile = async (data, image): Promise<void> => {
const { default: escapeStringRegexp } = await import('escape-string-regexp');
const {
workdir,
port,
@ -23,15 +24,15 @@ const createDockerfile = async (data, image): Promise<void> => {
secrets.forEach((secret) => {
if (secret.isBuildSecret) {
if (pullmergeRequestId) {
const isSecretFound = secrets.filter(s => s.name === secret.name && s.isPRMRSecret)
const isSecretFound = secrets.filter((s) => s.name === secret.name && s.isPRMRSecret);
if (isSecretFound.length > 0) {
Dockerfile.push(`ARG ${secret.name}=${isSecretFound[0].value}`);
Dockerfile.push(`ARG ${secret.name}=${escapeStringRegexp(isSecretFound[0].value)}`);
} else {
Dockerfile.push(`ARG ${secret.name}=${secret.value}`);
Dockerfile.push(`ARG ${secret.name}=${escapeStringRegexp(secret.value)}`);
}
} else {
if (!secret.isPRMRSecret) {
Dockerfile.push(`ARG ${secret.name}=${secret.value}`);
Dockerfile.push(`ARG ${secret.name}=${escapeStringRegexp(secret.value)}`);
}
}
}

View File

@ -43,6 +43,7 @@ importers:
dockerode: 3.3.4
dotenv-extended: 2.9.0
esbuild: 0.15.15
escape-string-regexp: 5.0.0
eslint: 8.28.0
eslint-config-prettier: 8.5.0
eslint-plugin-prettier: 4.2.1
@ -100,6 +101,7 @@ importers:
dayjs: 1.11.6
dockerode: 3.3.4
dotenv-extended: 2.9.0
escape-string-regexp: 5.0.0
execa: 6.1.0
fastify: 4.10.2
fastify-plugin: 4.3.0