From 79b0187b5802214bd70bc902e190846d5a55d5c9 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 23 Aug 2022 11:29:25 +0200 Subject: [PATCH] fix: stream build logs --- apps/api/package.json | 1 + apps/api/src/lib/buildPacks/common.ts | 52 +---------------------- apps/api/src/lib/common.ts | 53 +++++++++++++++++++++-- pnpm-lock.yaml | 61 ++++++++++++++++++++++++++- 4 files changed, 112 insertions(+), 55 deletions(-) diff --git a/apps/api/package.json b/apps/api/package.json index 1daabfa33..b5dcb5823 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -32,6 +32,7 @@ "dayjs": "1.11.5", "dockerode": "3.3.4", "dotenv-extended": "2.9.0", + "execa": "6.1.0", "fastify": "4.5.2", "fastify-plugin": "4.2.0", "generate-password": "1.7.0", diff --git a/apps/api/src/lib/buildPacks/common.ts b/apps/api/src/lib/buildPacks/common.ts index 6684664d6..be907ee7e 100644 --- a/apps/api/src/lib/buildPacks/common.ts +++ b/apps/api/src/lib/buildPacks/common.ts @@ -541,9 +541,6 @@ export async function buildImage({ } else { await saveBuildLog({ line: `Building image started.`, buildId, applicationId }); } - if (debug) { - await saveBuildLog({ line: `\n###############\nIMPORTANT: Due to some issues during implementing Remote Docker Engine, the builds logs are not streamed at the moment - but will be soon! You will see the full build log when the build is finished!\n###############`, buildId, applicationId }); - } if (!debug && isCache) { await saveBuildLog({ line: `Debug turned off. To see more details, allow it in the configuration.`, @@ -553,54 +550,7 @@ export async function buildImage({ } const dockerFile = isCache ? `${dockerFileLocation}-cache` : `${dockerFileLocation}` const cache = `${applicationId}:${tag}${isCache ? '-cache' : ''}` - const { stderr } = await executeDockerCmd({ dockerId, command: `docker build --progress plain -f ${workdir}/${dockerFile} -t ${cache} ${workdir}` }) - if (debug) { - const array = stderr.split('\n') - for (const line of array) { - if (line !== '\n') { - await saveBuildLog({ - line: `${line.replace('\n', '')}`, - buildId, - applicationId - }); - } - } - } - - - // await new Promise((resolve, reject) => { - // const command = spawn(`docker`, ['build', '-f', `${workdir}${dockerFile}`, '-t', `${cache}`,`${workdir}`], { - // env: { - // DOCKER_HOST: 'ssh://root@95.217.178.202', - // DOCKER_BUILDKIT: '1' - // } - // }); - // command.stdout.on('data', function (data) { - // console.log('stdout: ' + data); - // }); - // command.stderr.on('data', function (data) { - // console.log('stderr: ' + data); - // }); - // command.on('error', function (error) { - // console.log(error) - // reject(error) - // }) - // command.on('exit', function (code) { - // console.log('exit code: ' + code); - // resolve(code) - // }); - // }) - - - // console.log({ stdout, stderr }) - // const stream = await docker.engine.buildImage( - // { src: ['.'], context: workdir }, - // { - // dockerfile: isCache ? `${dockerFileLocation}-cache` : dockerFileLocation, - // t: `${applicationId}:${tag}${isCache ? '-cache' : ''}` - // } - // ); - // await streamEvents({ stream, docker, buildId, applicationId, debug }); + await executeDockerCmd({ debug, buildId, applicationId, dockerId, command: `docker build --progress plain -f ${workdir}/${dockerFile} -t ${cache} ${workdir}` }) if (isCache) { await saveBuildLog({ line: `Building cache image successful.`, buildId, applicationId }); } else { diff --git a/apps/api/src/lib/common.ts b/apps/api/src/lib/common.ts index 473979a31..07c422f58 100644 --- a/apps/api/src/lib/common.ts +++ b/apps/api/src/lib/common.ts @@ -1,4 +1,4 @@ -import child from 'child_process'; +import { exec } from 'node:child_process' import util from 'util'; import fs from 'fs/promises'; import yaml from 'js-yaml'; @@ -16,6 +16,7 @@ import sshConfig from 'ssh-config' import { checkContainer, removeContainer } from './docker'; import { day } from './dayjs'; import * as serviceFields from './serviceFields' +import { saveBuildLog } from './buildPacks/common'; export const version = '3.8.0'; export const isDev = process.env.NODE_ENV === 'development'; @@ -85,7 +86,50 @@ export const include: any = { }; export const uniqueName = (): string => uniqueNamesGenerator(customConfig); -export const asyncExecShell = util.promisify(child.exec); +export const asyncExecShell = util.promisify(exec); +export const asyncExecShellStream = async ({ debug, buildId, applicationId, command, engine }: { debug: boolean, buildId: string, applicationId: string, command: string, engine: string }) => { + return await new Promise(async (resolve, reject) => { + const { execaCommand } = await import('execa') + const subprocess = execaCommand(command, { env: { DOCKER_BUILDKIT: "1", DOCKER_HOST: engine } }) + if (debug) { + subprocess.stdout.on('data', async (data) => { + const stdout = data.toString(); + const array = stdout.split('\n') + for (const line of array) { + if (line !== '\n' && line !== '') { + await saveBuildLog({ + line: `${line.replace('\n', '')}`, + buildId, + applicationId + }); + } + } + }) + subprocess.stderr.on('data', async (data) => { + const stderr = data.toString(); + const array = stderr.split('\n') + for (const line of array) { + if (line !== '\n' && line !== '') { + await saveBuildLog({ + line: `${line.replace('\n', '')}`, + buildId, + applicationId + }); + } + } + }) + } + subprocess.on('exit', async (code) => { + await asyncSleep(1000); + if (code === 0) { + resolve(code) + } else { + reject(code) + } + }) + }) +} + export const asyncSleep = (delay: number): Promise => new Promise((resolve) => setTimeout(resolve, delay)); export const prisma = new PrismaClient({ @@ -633,7 +677,7 @@ export async function createRemoteEngineConfiguration(id: string) { } return await fs.writeFile(`${homedir}/.ssh/config`, sshConfig.stringify(config)) } -export async function executeDockerCmd({ dockerId, command }: { dockerId: string, command: string }) { +export async function executeDockerCmd({ debug, buildId, applicationId, dockerId, command }: { debug?: boolean, buildId?: string, applicationId?: string, dockerId: string, command: string }): Promise { let { remoteEngine, remoteIpAddress, engine } = await prisma.destinationDocker.findUnique({ where: { id: dockerId } }) if (remoteEngine) { await createRemoteEngineConfiguration(dockerId) @@ -646,6 +690,9 @@ export async function executeDockerCmd({ dockerId, command }: { dockerId: string command = command.replace(/docker compose/gi, 'docker-compose') } } + if (command.startsWith(`docker build --progress plain`)) { + return await asyncExecShellStream({ debug, buildId, applicationId, command, engine }); + } return await asyncExecShell( `DOCKER_BUILDKIT=1 DOCKER_HOST="${engine}" ${command}` ); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 718357f2b..45e590081 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -40,6 +40,7 @@ importers: eslint: 8.22.0 eslint-config-prettier: 8.5.0 eslint-plugin-prettier: 4.2.1 + execa: 6.1.0 fastify: 4.5.2 fastify-plugin: 4.2.0 generate-password: 1.7.0 @@ -80,6 +81,7 @@ importers: dayjs: 1.11.5 dockerode: 3.3.4 dotenv-extended: 2.9.0 + execa: 6.1.0 fastify: 4.5.2 fastify-plugin: 4.2.0 generate-password: 1.7.0 @@ -3225,6 +3227,21 @@ packages: resolution: {integrity: sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==} dev: false + /execa/6.1.0: + resolution: {integrity: sha512-QVWlX2e50heYJcCPG0iWtf8r0xjEYfz/OYLGDYH+IyjWezzPNxz63qNFOu0l4YftGWuizFVZHHs8PrLU5p2IDA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + cross-spawn: 7.0.3 + get-stream: 6.0.1 + human-signals: 3.0.1 + is-stream: 3.0.0 + merge-stream: 2.0.0 + npm-run-path: 5.1.0 + onetime: 6.0.0 + signal-exit: 3.0.7 + strip-final-newline: 3.0.0 + dev: false + /exit/0.1.2: resolution: {integrity: sha512-Zk/eNKV2zbjpKzrsQ+n1G6poVbErQxJ0LBOJXaKZ1EViLzH+hrLu9cdXI4zw9dBQJslwBEpbQ2P1oS7nDxs6jQ==} engines: {node: '>= 0.8.0'} @@ -3728,6 +3745,11 @@ packages: numbered: 1.1.0 dev: false + /human-signals/3.0.1: + resolution: {integrity: sha512-rQLskxnM/5OCldHo+wNXbpVgDn5A17CUoKX+7Sokwaknlq7CdSnphy0W39GU8dw59XiCXmFXDg4fRuckQRKewQ==} + engines: {node: '>=12.20.0'} + dev: false + /iconv-lite/0.4.24: resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==} engines: {node: '>=0.10.0'} @@ -3975,6 +3997,11 @@ packages: engines: {node: '>=8'} dev: false + /is-stream/3.0.0: + resolution: {integrity: sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dev: false + /is-string-and-not-blank/0.0.2: resolution: {integrity: sha512-FyPGAbNVyZpTeDCTXnzuwbu9/WpNXbCfbHXLpCRpN4GANhS00eEIP5Ef+k5HYSNIzIhdN9zRDoBj6unscECvtQ==} engines: {node: '>=6.4.0'} @@ -4329,6 +4356,10 @@ packages: engines: {node: '>= 0.10.0'} dev: true + /merge-stream/2.0.0: + resolution: {integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==} + dev: false + /merge2/1.4.1: resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} engines: {node: '>= 8'} @@ -4374,6 +4405,11 @@ packages: engines: {node: '>=6'} dev: true + /mimic-fn/4.0.0: + resolution: {integrity: sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==} + engines: {node: '>=12'} + dev: false + /mimic-response/1.0.1: resolution: {integrity: sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==} engines: {node: '>=4'} @@ -4563,6 +4599,13 @@ packages: string.prototype.padend: 3.1.3 dev: true + /npm-run-path/5.1.0: + resolution: {integrity: sha512-sJOdmRGrY2sjNTRMbSvluQqg+8X7ZK61yvzBEIDhz4f8z1TZFYABsqjjCBd/0PUNE9M6QDgHJXQkGUEm7Q+l9Q==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + path-key: 4.0.0 + dev: false + /numbered/1.1.0: resolution: {integrity: sha512-pv/ue2Odr7IfYOO0byC1KgBI10wo5YDauLhxY6/saNzAdAs0r1SotGCPzzCLNPL0xtrAwWRialLu23AAu9xO1g==} dev: false @@ -4616,6 +4659,13 @@ packages: mimic-fn: 2.1.0 dev: true + /onetime/6.0.0: + resolution: {integrity: sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ==} + engines: {node: '>=12'} + dependencies: + mimic-fn: 4.0.0 + dev: false + /opencollective-setup/1.4.1: resolution: {integrity: sha512-XdYWl35SewxvjJCeuFQSkuL1elwOsGYXpzT0/vOaszC16KNmHi1NoLfiN/YJiw/VeP4E+DYVI906oGsXp0b77g==} engines: {node: 11.8.0, npm: 6.5.0} @@ -4827,6 +4877,11 @@ packages: resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} engines: {node: '>=8'} + /path-key/4.0.0: + resolution: {integrity: sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==} + engines: {node: '>=12'} + dev: false + /path-parse/1.0.7: resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} @@ -5440,7 +5495,6 @@ packages: /signal-exit/3.0.7: resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==} - dev: true /simple-swizzle/0.2.2: resolution: {integrity: sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==} @@ -5630,6 +5684,11 @@ packages: resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==} engines: {node: '>=4'} + /strip-final-newline/3.0.0: + resolution: {integrity: sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw==} + engines: {node: '>=12'} + dev: false + /strip-indent/3.0.0: resolution: {integrity: sha512-laJTa3Jb+VQpaC6DseHhF7dXVqHTfJPCRDaEbid/drOhgitgYku/letMUqOXFoWV0zIIUbjpdH2t+tYj4bQMRQ==} engines: {node: '>=8'}