From 9c4e581d8b1a8efdc6be4baf8348ac126ce593de Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Fri, 25 Nov 2022 14:29:01 +0100 Subject: [PATCH] feat: use registry for building --- apps/api/src/index.ts | 17 ++-- apps/api/src/jobs/deployApplication.ts | 104 +++++++++++++------------ apps/api/src/lib/buildPacks/common.ts | 31 +++++++- 3 files changed, 95 insertions(+), 57 deletions(-) diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 133db9152..b5fab46db 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -202,14 +202,14 @@ async function getIPAddress() { try { const settings = await listSettings(); if (!settings.ipv4) { - console.log(`Getting public IPv4 address...`); const ipv4 = await publicIpv4({ timeout: 2000 }) + console.log(`Getting public IPv4 address...`); await prisma.setting.update({ where: { id: settings.id }, data: { ipv4 } }) } if (!settings.ipv6) { - console.log(`Getting public IPv6 address...`); const ipv6 = await publicIpv6({ timeout: 2000 }) + console.log(`Getting public IPv6 address...`); await prisma.setting.update({ where: { id: settings.id }, data: { ipv6 } }) } @@ -223,13 +223,13 @@ async function getTagsTemplates() { const tags = await fs.readFile('./devTags.json', 'utf8') await fs.writeFile('./templates.json', JSON.stringify(yaml.load(templates))) await fs.writeFile('./tags.json', tags) - console.log('Tags and templates loaded in dev mode...') + console.log('[004] Tags and templates loaded in dev mode...') } else { const tags = await got.get('https://get.coollabs.io/coolify/service-tags.json').text() const response = await got.get('https://get.coollabs.io/coolify/service-templates.yaml').text() await fs.writeFile('/app/templates.json', JSON.stringify(yaml.load(response))) await fs.writeFile('/app/tags.json', tags) - console.log('Tags and templates loaded...') + console.log('[004] Tags and templates loaded...') } } catch (error) { @@ -239,15 +239,22 @@ async function getTagsTemplates() { } async function initServer() { try { - console.log(`Initializing server...`); + console.log(`[001] Initializing server...`); await asyncExecShell(`docker network create --attachable coolify`); } catch (error) { } try { + console.log(`[002] Set stuck builds to failed...`); const isOlder = compareVersions('3.8.1', version); if (isOlder === 1) { await prisma.build.updateMany({ where: { status: { in: ['running', 'queued'] } }, data: { status: 'failed' } }); } } catch (error) { } + try { + console.log('[003] Cleaning up old build sources under /tmp/build-sources/...'); + await fs.rm('/tmp/build-sources', { recursive: true, force: true }) + } catch (error) { + console.log(error) + } } async function getArch() { diff --git a/apps/api/src/jobs/deployApplication.ts b/apps/api/src/jobs/deployApplication.ts index edc5c4864..706ab1c46 100644 --- a/apps/api/src/jobs/deployApplication.ts +++ b/apps/api/src/jobs/deployApplication.ts @@ -38,56 +38,70 @@ import * as buildpacks from '../lib/buildPacks'; for (const queueBuild of queuedBuilds) { actions.push(async () => { let application = await prisma.application.findUnique({ where: { id: queueBuild.applicationId }, include: { destinationDocker: true, gitSource: { include: { githubApp: true, gitlabApp: true } }, persistentStorage: true, secrets: true, settings: true, teams: true } }) + let { id: buildId, type, sourceBranch = null, pullmergeRequestId = null, previewApplicationId = null, forceRebuild, sourceRepository = null } = queueBuild + application = decryptApplication(application) + const originalApplicationId = application.id + const { + id: applicationId, + name, + destinationDocker, + destinationDockerId, + gitSource, + configHash, + fqdn, + projectId, + secrets, + phpModules, + settings, + persistentStorage, + pythonWSGI, + pythonModule, + pythonVariable, + denoOptions, + exposePort, + baseImage, + baseBuildImage, + deploymentType, + } = application + + let { + branch, + repository, + buildPack, + port, + installCommand, + buildCommand, + startCommand, + baseDirectory, + publishDirectory, + dockerFileLocation, + dockerComposeConfiguration, + denoMainFile + } = application + + let imageId = applicationId; + let domain = getDomain(fqdn); + if (pullmergeRequestId) { const previewApplications = await prisma.previewApplication.findMany({ where: { applicationId: originalApplicationId, pullmergeRequestId } }) if (previewApplications.length > 0) { previewApplicationId = previewApplications[0].id } + // Previews, we need to get the source branch and set subdomain + branch = sourceBranch; + domain = `${pullmergeRequestId}.${domain}`; + imageId = `${applicationId}-${pullmergeRequestId}`; + repository = sourceRepository || repository; } - const usableApplicationId = previewApplicationId || originalApplicationId + const { workdir, repodir } = await createDirectories({ repository, buildId }); try { if (queueBuild.status === 'running') { await saveBuildLog({ line: 'Building halted, restarting...', buildId, applicationId: application.id }); } - const { - id: applicationId, - name, - destinationDocker, - destinationDockerId, - gitSource, - configHash, - fqdn, - projectId, - secrets, - phpModules, - settings, - persistentStorage, - pythonWSGI, - pythonModule, - pythonVariable, - denoOptions, - exposePort, - baseImage, - baseBuildImage, - deploymentType, - } = application - let { - branch, - repository, - buildPack, - port, - installCommand, - buildCommand, - startCommand, - baseDirectory, - publishDirectory, - dockerFileLocation, - dockerComposeConfiguration, - denoMainFile - } = application + const currentHash = crypto .createHash('sha256') .update( @@ -113,22 +127,14 @@ import * as buildpacks from '../lib/buildPacks'; ) .digest('hex'); const { debug } = settings; - let imageId = applicationId; - let domain = getDomain(fqdn); const volumes = persistentStorage?.map((storage) => { if (storage.oldPath) { - return `${applicationId}${storage.path.replace(/\//gi, '-').replace('-app','')}:${storage.path}`; + return `${applicationId}${storage.path.replace(/\//gi, '-').replace('-app', '')}:${storage.path}`; } return `${applicationId}${storage.path.replace(/\//gi, '-')}:${storage.path}`; }) || []; - // Previews, we need to get the source branch and set subdomain - if (pullmergeRequestId) { - branch = sourceBranch; - domain = `${pullmergeRequestId}.${domain}`; - imageId = `${applicationId}-${pullmergeRequestId}`; - repository = sourceRepository || repository; - } + try { dockerComposeConfiguration = JSON.parse(dockerComposeConfiguration) @@ -141,7 +147,7 @@ import * as buildpacks from '../lib/buildPacks'; } if (destinationType === 'docker') { await prisma.build.update({ where: { id: buildId }, data: { status: 'running' } }); - const { workdir, repodir } = await createDirectories({ repository, buildId }); + const configuration = await setDefaultConfiguration(application); buildPack = configuration.buildPack; @@ -419,6 +425,8 @@ import * as buildpacks from '../lib/buildPacks'; if (error !== 1) { await saveBuildLog({ line: error, buildId, applicationId: application.id }); } + } finally { + await fs.rm(workdir, { recursive: true, force: true }); } }); } diff --git a/apps/api/src/lib/buildPacks/common.ts b/apps/api/src/lib/buildPacks/common.ts index b2afafa1d..3f2350ec7 100644 --- a/apps/api/src/lib/buildPacks/common.ts +++ b/apps/api/src/lib/buildPacks/common.ts @@ -1,4 +1,4 @@ -import { base64Encode, encrypt, executeDockerCmd, generateTimestamp, getDomain, isDev, prisma, version } from "../common"; +import { base64Encode, decrypt, encrypt, executeDockerCmd, generateTimestamp, getDomain, isDev, prisma, version } from "../common"; import { promises as fs } from 'fs'; import { day } from "../dayjs"; @@ -464,7 +464,7 @@ export const saveBuildLog = async ({ const { default: got } = await import('got') if (typeof line === 'object' && line) { if (line.shortMessage) { - line = line.shortMessage + '\n' + line.stderr; + line = line.shortMessage + '\n' + line.stderr; } else { line = JSON.stringify(line); } @@ -577,6 +577,28 @@ export function checkPnpm(installCommand = null, buildCommand = null, startComma ); } +export async function saveDockerRegistryCredentials({ url, username, password, workdir }) { + let decryptedPassword = decrypt(password); + const location = `${workdir}/.docker`; + + if (!username || !password) { + return null + } + + try { + await fs.mkdir(`${workdir}/.docker`); + } catch (error) { + console.log(error); + } + await fs.writeFile(`${location}/config.json`, JSON.stringify({ + "auths": { + [url]: { + "auth": Buffer.from(`${username}:${decryptedPassword}`).toString('base64') + } + } + })) + return location +} export async function buildImage({ applicationId, tag, @@ -602,8 +624,9 @@ export async function buildImage({ } const dockerFile = isCache ? `${dockerFileLocation}-cache` : `${dockerFileLocation}` const cache = `${applicationId}:${tag}${isCache ? '-cache' : ''}` - - await executeDockerCmd({ debug, buildId, applicationId, dockerId, command: `docker build --progress plain -f ${workdir}/${dockerFile} -t ${cache} --build-arg SOURCE_COMMIT=${commit} ${workdir}` }) + const { dockerRegistry: { url, username, password } } = await prisma.application.findUnique({ where: { id: applicationId }, select: { dockerRegistry: true } }) + const location = await saveDockerRegistryCredentials({ url, username, password, workdir }) + await executeDockerCmd({ debug, buildId, applicationId, dockerId, command: `docker ${location ? `--config ${location}` : ''} build --progress plain -f ${workdir}/${dockerFile} -t ${cache} --build-arg SOURCE_COMMIT=${commit} ${workdir}` }) const { status } = await prisma.build.findUnique({ where: { id: buildId } }) if (status === 'canceled') {