fix: stream build logs

This commit is contained in:
Andras Bacsai 2022-08-23 11:29:25 +02:00
parent bf5659d0e2
commit 79b0187b58
4 changed files with 112 additions and 55 deletions

View File

@ -32,6 +32,7 @@
"dayjs": "1.11.5",
"dockerode": "3.3.4",
"dotenv-extended": "2.9.0",
"execa": "6.1.0",
"fastify": "4.5.2",
"fastify-plugin": "4.2.0",
"generate-password": "1.7.0",

View File

@ -541,9 +541,6 @@ export async function buildImage({
} else {
await saveBuildLog({ line: `Building image started.`, buildId, applicationId });
}
if (debug) {
await saveBuildLog({ line: `\n###############\nIMPORTANT: Due to some issues during implementing Remote Docker Engine, the builds logs are not streamed at the moment - but will be soon! You will see the full build log when the build is finished!\n###############`, buildId, applicationId });
}
if (!debug && isCache) {
await saveBuildLog({
line: `Debug turned off. To see more details, allow it in the configuration.`,
@ -553,54 +550,7 @@ export async function buildImage({
}
const dockerFile = isCache ? `${dockerFileLocation}-cache` : `${dockerFileLocation}`
const cache = `${applicationId}:${tag}${isCache ? '-cache' : ''}`
const { stderr } = await executeDockerCmd({ dockerId, command: `docker build --progress plain -f ${workdir}/${dockerFile} -t ${cache} ${workdir}` })
if (debug) {
const array = stderr.split('\n')
for (const line of array) {
if (line !== '\n') {
await saveBuildLog({
line: `${line.replace('\n', '')}`,
buildId,
applicationId
});
}
}
}
// await new Promise((resolve, reject) => {
// const command = spawn(`docker`, ['build', '-f', `${workdir}${dockerFile}`, '-t', `${cache}`,`${workdir}`], {
// env: {
// DOCKER_HOST: 'ssh://root@95.217.178.202',
// DOCKER_BUILDKIT: '1'
// }
// });
// command.stdout.on('data', function (data) {
// console.log('stdout: ' + data);
// });
// command.stderr.on('data', function (data) {
// console.log('stderr: ' + data);
// });
// command.on('error', function (error) {
// console.log(error)
// reject(error)
// })
// command.on('exit', function (code) {
// console.log('exit code: ' + code);
// resolve(code)
// });
// })
// console.log({ stdout, stderr })
// const stream = await docker.engine.buildImage(
// { src: ['.'], context: workdir },
// {
// dockerfile: isCache ? `${dockerFileLocation}-cache` : dockerFileLocation,
// t: `${applicationId}:${tag}${isCache ? '-cache' : ''}`
// }
// );
// await streamEvents({ stream, docker, buildId, applicationId, debug });
await executeDockerCmd({ debug, buildId, applicationId, dockerId, command: `docker build --progress plain -f ${workdir}/${dockerFile} -t ${cache} ${workdir}` })
if (isCache) {
await saveBuildLog({ line: `Building cache image successful.`, buildId, applicationId });
} else {

View File

@ -1,4 +1,4 @@
import child from 'child_process';
import { exec } from 'node:child_process'
import util from 'util';
import fs from 'fs/promises';
import yaml from 'js-yaml';
@ -16,6 +16,7 @@ import sshConfig from 'ssh-config'
import { checkContainer, removeContainer } from './docker';
import { day } from './dayjs';
import * as serviceFields from './serviceFields'
import { saveBuildLog } from './buildPacks/common';
export const version = '3.8.0';
export const isDev = process.env.NODE_ENV === 'development';
@ -85,7 +86,50 @@ export const include: any = {
};
export const uniqueName = (): string => uniqueNamesGenerator(customConfig);
export const asyncExecShell = util.promisify(child.exec);
export const asyncExecShell = util.promisify(exec);
export const asyncExecShellStream = async ({ debug, buildId, applicationId, command, engine }: { debug: boolean, buildId: string, applicationId: string, command: string, engine: string }) => {
return await new Promise(async (resolve, reject) => {
const { execaCommand } = await import('execa')
const subprocess = execaCommand(command, { env: { DOCKER_BUILDKIT: "1", DOCKER_HOST: engine } })
if (debug) {
subprocess.stdout.on('data', async (data) => {
const stdout = data.toString();
const array = stdout.split('\n')
for (const line of array) {
if (line !== '\n' && line !== '') {
await saveBuildLog({
line: `${line.replace('\n', '')}`,
buildId,
applicationId
});
}
}
})
subprocess.stderr.on('data', async (data) => {
const stderr = data.toString();
const array = stderr.split('\n')
for (const line of array) {
if (line !== '\n' && line !== '') {
await saveBuildLog({
line: `${line.replace('\n', '')}`,
buildId,
applicationId
});
}
}
})
}
subprocess.on('exit', async (code) => {
await asyncSleep(1000);
if (code === 0) {
resolve(code)
} else {
reject(code)
}
})
})
}
export const asyncSleep = (delay: number): Promise<unknown> =>
new Promise((resolve) => setTimeout(resolve, delay));
export const prisma = new PrismaClient({
@ -633,7 +677,7 @@ export async function createRemoteEngineConfiguration(id: string) {
}
return await fs.writeFile(`${homedir}/.ssh/config`, sshConfig.stringify(config))
}
export async function executeDockerCmd({ dockerId, command }: { dockerId: string, command: string }) {
export async function executeDockerCmd({ debug, buildId, applicationId, dockerId, command }: { debug?: boolean, buildId?: string, applicationId?: string, dockerId: string, command: string }): Promise<any> {
let { remoteEngine, remoteIpAddress, engine } = await prisma.destinationDocker.findUnique({ where: { id: dockerId } })
if (remoteEngine) {
await createRemoteEngineConfiguration(dockerId)
@ -646,6 +690,9 @@ export async function executeDockerCmd({ dockerId, command }: { dockerId: string
command = command.replace(/docker compose/gi, 'docker-compose')
}
}
if (command.startsWith(`docker build --progress plain`)) {
return await asyncExecShellStream({ debug, buildId, applicationId, command, engine });
}
return await asyncExecShell(
`DOCKER_BUILDKIT=1 DOCKER_HOST="${engine}" ${command}`
);

61
pnpm-lock.yaml generated
View File

@ -40,6 +40,7 @@ importers:
eslint: 8.22.0
eslint-config-prettier: 8.5.0
eslint-plugin-prettier: 4.2.1
execa: 6.1.0
fastify: 4.5.2
fastify-plugin: 4.2.0
generate-password: 1.7.0
@ -80,6 +81,7 @@ importers:
dayjs: 1.11.5
dockerode: 3.3.4
dotenv-extended: 2.9.0
execa: 6.1.0
fastify: 4.5.2
fastify-plugin: 4.2.0
generate-password: 1.7.0
@ -3225,6 +3227,21 @@ packages:
resolution: {integrity: sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==}
dev: false
/execa/6.1.0:
resolution: {integrity: sha512-QVWlX2e50heYJcCPG0iWtf8r0xjEYfz/OYLGDYH+IyjWezzPNxz63qNFOu0l4YftGWuizFVZHHs8PrLU5p2IDA==}
engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
dependencies:
cross-spawn: 7.0.3
get-stream: 6.0.1
human-signals: 3.0.1
is-stream: 3.0.0
merge-stream: 2.0.0
npm-run-path: 5.1.0
onetime: 6.0.0
signal-exit: 3.0.7
strip-final-newline: 3.0.0
dev: false
/exit/0.1.2:
resolution: {integrity: sha512-Zk/eNKV2zbjpKzrsQ+n1G6poVbErQxJ0LBOJXaKZ1EViLzH+hrLu9cdXI4zw9dBQJslwBEpbQ2P1oS7nDxs6jQ==}
engines: {node: '>= 0.8.0'}
@ -3728,6 +3745,11 @@ packages:
numbered: 1.1.0
dev: false
/human-signals/3.0.1:
resolution: {integrity: sha512-rQLskxnM/5OCldHo+wNXbpVgDn5A17CUoKX+7Sokwaknlq7CdSnphy0W39GU8dw59XiCXmFXDg4fRuckQRKewQ==}
engines: {node: '>=12.20.0'}
dev: false
/iconv-lite/0.4.24:
resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==}
engines: {node: '>=0.10.0'}
@ -3975,6 +3997,11 @@ packages:
engines: {node: '>=8'}
dev: false
/is-stream/3.0.0:
resolution: {integrity: sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA==}
engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
dev: false
/is-string-and-not-blank/0.0.2:
resolution: {integrity: sha512-FyPGAbNVyZpTeDCTXnzuwbu9/WpNXbCfbHXLpCRpN4GANhS00eEIP5Ef+k5HYSNIzIhdN9zRDoBj6unscECvtQ==}
engines: {node: '>=6.4.0'}
@ -4329,6 +4356,10 @@ packages:
engines: {node: '>= 0.10.0'}
dev: true
/merge-stream/2.0.0:
resolution: {integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==}
dev: false
/merge2/1.4.1:
resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==}
engines: {node: '>= 8'}
@ -4374,6 +4405,11 @@ packages:
engines: {node: '>=6'}
dev: true
/mimic-fn/4.0.0:
resolution: {integrity: sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==}
engines: {node: '>=12'}
dev: false
/mimic-response/1.0.1:
resolution: {integrity: sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==}
engines: {node: '>=4'}
@ -4563,6 +4599,13 @@ packages:
string.prototype.padend: 3.1.3
dev: true
/npm-run-path/5.1.0:
resolution: {integrity: sha512-sJOdmRGrY2sjNTRMbSvluQqg+8X7ZK61yvzBEIDhz4f8z1TZFYABsqjjCBd/0PUNE9M6QDgHJXQkGUEm7Q+l9Q==}
engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
dependencies:
path-key: 4.0.0
dev: false
/numbered/1.1.0:
resolution: {integrity: sha512-pv/ue2Odr7IfYOO0byC1KgBI10wo5YDauLhxY6/saNzAdAs0r1SotGCPzzCLNPL0xtrAwWRialLu23AAu9xO1g==}
dev: false
@ -4616,6 +4659,13 @@ packages:
mimic-fn: 2.1.0
dev: true
/onetime/6.0.0:
resolution: {integrity: sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ==}
engines: {node: '>=12'}
dependencies:
mimic-fn: 4.0.0
dev: false
/opencollective-setup/1.4.1:
resolution: {integrity: sha512-XdYWl35SewxvjJCeuFQSkuL1elwOsGYXpzT0/vOaszC16KNmHi1NoLfiN/YJiw/VeP4E+DYVI906oGsXp0b77g==}
engines: {node: 11.8.0, npm: 6.5.0}
@ -4827,6 +4877,11 @@ packages:
resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==}
engines: {node: '>=8'}
/path-key/4.0.0:
resolution: {integrity: sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==}
engines: {node: '>=12'}
dev: false
/path-parse/1.0.7:
resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==}
@ -5440,7 +5495,6 @@ packages:
/signal-exit/3.0.7:
resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==}
dev: true
/simple-swizzle/0.2.2:
resolution: {integrity: sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==}
@ -5630,6 +5684,11 @@ packages:
resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==}
engines: {node: '>=4'}
/strip-final-newline/3.0.0:
resolution: {integrity: sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw==}
engines: {node: '>=12'}
dev: false
/strip-indent/3.0.0:
resolution: {integrity: sha512-laJTa3Jb+VQpaC6DseHhF7dXVqHTfJPCRDaEbid/drOhgitgYku/letMUqOXFoWV0zIIUbjpdH2t+tYj4bQMRQ==}
engines: {node: '>=8'}