From 32020fd33653824fb5893b13749eb297c19fa1f2 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 5 Apr 2022 10:21:40 +0200 Subject: [PATCH 01/10] fix: Add git lfs while deploying --- Dockerfile | 2 +- src/lib/common.ts | 19 +++++++++++ src/lib/importers/github.ts | 65 +++++++++++++++++-------------------- src/lib/importers/gitlab.ts | 2 +- src/lib/queues/index.ts | 19 +++++++++-- 5 files changed, 68 insertions(+), 39 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5d0a8912b..9494ff702 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ WORKDIR /app LABEL coolify.managed true -RUN apk add --no-cache git openssh-client curl jq cmake sqlite +RUN apk add --no-cache git git-lfs openssh-client curl jq cmake sqlite RUN curl -f https://get.pnpm.io/v6.16.js | node - add --global pnpm@6 RUN pnpm add -g pnpm diff --git a/src/lib/common.ts b/src/lib/common.ts index 49a637065..d6b36b209 100644 --- a/src/lib/common.ts +++ b/src/lib/common.ts @@ -46,11 +46,30 @@ const customConfig: Config = { export const version = currentVersion; export const asyncExecShell = util.promisify(child.exec); export const asyncSleep = (delay) => new Promise((resolve) => setTimeout(resolve, delay)); +export const asyncUntil = (condition, delay, maxIterations) => { + return new Promise(async (resolve, reject) => { + let iterations = 0; + while (!condition()) { + if (maxIterations && iterations >= maxIterations) { + reject(new Error('Max iterations reached.')); + return; + } + await asyncSleep(delay); + iterations++; + } + resolve(); + }); +}; + export const sentry = Sentry; export const uniqueName = () => uniqueNamesGenerator(customConfig); export const saveBuildLog = async ({ line, buildId, applicationId }) => { + if (line.includes('ghs_')) { + const regex = /ghs_.*@/g; + line = line.replace(regex, '@'); + } const addTimestamp = `${generateTimestamp()} ${line}`; return await buildLogQueue.add(buildId, { buildId, line: addTimestamp, applicationId }); }; diff --git a/src/lib/importers/github.ts b/src/lib/importers/github.ts index 1ab3ca90a..53754205e 100644 --- a/src/lib/importers/github.ts +++ b/src/lib/importers/github.ts @@ -15,40 +15,35 @@ export default async function ({ branch, buildId }): Promise { - try { - const url = htmlUrl.replace('https://', '').replace('http://', ''); - await saveBuildLog({ line: 'GitHub importer started.', buildId, applicationId }); - const { privateKey, appId, installationId } = await db.getUniqueGithubApp({ githubAppId }); - const githubPrivateKey = privateKey.replace(/\\n/g, '\n').replace(/"/g, ''); + const url = htmlUrl.replace('https://', '').replace('http://', ''); + await saveBuildLog({ line: 'GitHub importer started.', buildId, applicationId }); + const { privateKey, appId, installationId } = await db.getUniqueGithubApp({ githubAppId }); + const githubPrivateKey = privateKey.replace(/\\n/g, '\n').replace(/"/g, ''); - const payload = { - iat: Math.round(new Date().getTime() / 1000), - exp: Math.round(new Date().getTime() / 1000 + 60), - iss: appId - }; - const jwtToken = jsonwebtoken.sign(payload, githubPrivateKey, { - algorithm: 'RS256' - }); - const { token } = await got - .post(`${apiUrl}/app/installations/${installationId}/access_tokens`, { - headers: { - Authorization: `Bearer ${jwtToken}`, - Accept: 'application/vnd.github.machine-man-preview+json' - } - }) - .json(); - await saveBuildLog({ - line: `Cloning ${repository}:${branch} branch.`, - buildId, - applicationId - }); - await asyncExecShell( - `git clone -q -b ${branch} https://x-access-token:${token}@${url}/${repository}.git ${workdir}/ && cd ${workdir} && git submodule update --init --recursive && cd ..` - ); - const { stdout: commit } = await asyncExecShell(`cd ${workdir}/ && git rev-parse HEAD`); - return commit.replace('\n', ''); - } catch (error) { - console.log({ error }); - return ErrorHandler(error); - } + const payload = { + iat: Math.round(new Date().getTime() / 1000), + exp: Math.round(new Date().getTime() / 1000 + 60), + iss: appId + }; + const jwtToken = jsonwebtoken.sign(payload, githubPrivateKey, { + algorithm: 'RS256' + }); + const { token } = await got + .post(`${apiUrl}/app/installations/${installationId}/access_tokens`, { + headers: { + Authorization: `Bearer ${jwtToken}`, + Accept: 'application/vnd.github.machine-man-preview+json' + } + }) + .json(); + await saveBuildLog({ + line: `Cloning ${repository}:${branch} branch.`, + buildId, + applicationId + }); + await asyncExecShell( + `git clone -q -b ${branch} https://x-access-token:${token}@${url}/${repository}.git ${workdir}/ && cd ${workdir} && git submodule update --init --recursive && git lfs pull && cd .. ` + ); + const { stdout: commit } = await asyncExecShell(`cd ${workdir}/ && git rev-parse HEAD`); + return commit.replace('\n', ''); } diff --git a/src/lib/importers/gitlab.ts b/src/lib/importers/gitlab.ts index 07bbf7954..81ef54c10 100644 --- a/src/lib/importers/gitlab.ts +++ b/src/lib/importers/gitlab.ts @@ -22,7 +22,7 @@ export default async function ({ }); await asyncExecShell( - `git clone -q -b ${branch} git@${url}:${repository}.git --config core.sshCommand="ssh -q -i ${repodir}id.rsa -o StrictHostKeyChecking=no" ${workdir}/ && cd ${workdir}/ && git submodule update --init --recursive && cd ..` + `git clone -q -b ${branch} git@${url}:${repository}.git --config core.sshCommand="ssh -q -i ${repodir}id.rsa -o StrictHostKeyChecking=no" ${workdir}/ && cd ${workdir}/ && git submodule update --init --recursive && git lfs pull && cd .. ` ); const { stdout: commit } = await asyncExecShell(`cd ${workdir}/ && git rev-parse HEAD`); return commit.replace('\n', ''); diff --git a/src/lib/queues/index.ts b/src/lib/queues/index.ts index 0f7dd5815..58b9e583d 100644 --- a/src/lib/queues/index.ts +++ b/src/lib/queues/index.ts @@ -11,7 +11,7 @@ import proxy from './proxy'; import ssl from './ssl'; import sslrenewal from './sslrenewal'; -import { asyncExecShell, saveBuildLog } from '$lib/common'; +import { asyncExecShell, asyncUntil, saveBuildLog } from '$lib/common'; let { Queue, Worker } = Bullmq; let redisHost = 'localhost'; @@ -128,7 +128,22 @@ buildWorker.on('completed', async (job: Bullmq.Job) => { buildWorker.on('failed', async (job: Bullmq.Job, failedReason) => { try { - await prisma.build.update({ where: { id: job.data.build_id }, data: { status: 'failed' } }); + await asyncUntil( + async () => { + const found = await prisma.build.findFirst({ + where: { id: job.data.build_id, status: 'failed' } + }); + if (!found) { + return await prisma.build.update({ + where: { id: job.data.build_id }, + data: { status: 'failed' } + }); + } + return true; + }, + 200, + 5 + ); } catch (error) { console.log(error); } finally { From a3ee57995c804ddd079d600e1e2ac57f8d717163 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 5 Apr 2022 10:21:46 +0200 Subject: [PATCH 02/10] chore: Version++ --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1d9ccbdb7..2aa11b114 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "coolify", "description": "An open-source & self-hostable Heroku / Netlify alternative.", - "version": "2.3.2", + "version": "2.3.3", "license": "AGPL-3.0", "scripts": { "dev": "docker-compose -f docker-compose-dev.yaml up -d && cross-env NODE_ENV=development & svelte-kit dev", From 7ab5a4bfcf42166cffffcadc03eaf5311537bcb9 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 5 Apr 2022 10:30:18 +0200 Subject: [PATCH 03/10] fix: Try to update build status several times --- src/lib/queues/index.ts | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/lib/queues/index.ts b/src/lib/queues/index.ts index 58b9e583d..6c1cc29c2 100644 --- a/src/lib/queues/index.ts +++ b/src/lib/queues/index.ts @@ -116,7 +116,22 @@ const buildWorker = new Worker(buildQueueName, async (job) => await builder(job) buildWorker.on('completed', async (job: Bullmq.Job) => { try { - await prisma.build.update({ where: { id: job.data.build_id }, data: { status: 'success' } }); + await asyncUntil( + async () => { + const found = await prisma.build.findFirst({ + where: { id: job.data.build_id, status: 'success' } + }); + if (!found) { + return await prisma.build.update({ + where: { id: job.data.build_id }, + data: { status: 'success' } + }); + } + return true; + }, + 100, + 5 + ); } catch (error) { console.log(error); } finally { @@ -141,7 +156,7 @@ buildWorker.on('failed', async (job: Bullmq.Job, failedReason) => { } return true; }, - 200, + 100, 5 ); } catch (error) { From b1c25e98d70a0ab9419158472d5b0b8f10cd3a21 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 5 Apr 2022 10:34:31 +0200 Subject: [PATCH 04/10] fix: Update stucked builds on startup --- prisma/seed.cjs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/prisma/seed.cjs b/prisma/seed.cjs index 61f3928f6..cc5225fa4 100644 --- a/prisma/seed.cjs +++ b/prisma/seed.cjs @@ -50,6 +50,11 @@ async function main() { } }); } + // Update stucked builds + await prisma.build.updateMany({ + where: { status: 'running' }, + data: { status: 'success' } + }); } main() .catch((e) => { From c8c23c53ef0bfac7fc6d45b13a75ea3b1fd43124 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 5 Apr 2022 10:36:09 +0200 Subject: [PATCH 05/10] fix: Update stucked builds --- src/lib/queues/builder.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/queues/builder.ts b/src/lib/queues/builder.ts index 0ef207f2a..f6d57862e 100644 --- a/src/lib/queues/builder.ts +++ b/src/lib/queues/builder.ts @@ -61,7 +61,7 @@ export default async function (job) { await asyncSleep(500); await db.prisma.build.updateMany({ where: { - status: 'queued', + status: { in: ['queued', 'running'] }, id: { not: buildId }, applicationId, createdAt: { lt: new Date(new Date().getTime() - 60 * 60 * 1000) } From 5684674bd7962f93e950a9194dce9ec73d3524f4 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 5 Apr 2022 10:36:28 +0200 Subject: [PATCH 06/10] fix: revert seed --- prisma/seed.cjs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/prisma/seed.cjs b/prisma/seed.cjs index cc5225fa4..61f3928f6 100644 --- a/prisma/seed.cjs +++ b/prisma/seed.cjs @@ -50,11 +50,6 @@ async function main() { } }); } - // Update stucked builds - await prisma.build.updateMany({ - where: { status: 'running' }, - data: { status: 'success' } - }); } main() .catch((e) => { From 03cde08d67b2e1bafdf48065f71cd4cdcffb007b Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 5 Apr 2022 10:58:47 +0200 Subject: [PATCH 07/10] fix: Lame fixing --- src/lib/queues/index.ts | 42 ++++++++++------------------------------- 1 file changed, 10 insertions(+), 32 deletions(-) diff --git a/src/lib/queues/index.ts b/src/lib/queues/index.ts index 6c1cc29c2..c8646c3a1 100644 --- a/src/lib/queues/index.ts +++ b/src/lib/queues/index.ts @@ -116,54 +116,32 @@ const buildWorker = new Worker(buildQueueName, async (job) => await builder(job) buildWorker.on('completed', async (job: Bullmq.Job) => { try { - await asyncUntil( - async () => { - const found = await prisma.build.findFirst({ - where: { id: job.data.build_id, status: 'success' } - }); - if (!found) { - return await prisma.build.update({ - where: { id: job.data.build_id }, - data: { status: 'success' } - }); - } - return true; - }, - 100, - 5 - ); + await prisma.build.update({ where: { id: job.data.build_id }, data: { status: 'success' } }); } catch (error) { + setTimeout(async () => { + await prisma.build.update({ where: { id: job.data.build_id }, data: { status: 'success' } }); + }, 1234); console.log(error); } finally { const workdir = `/tmp/build-sources/${job.data.repository}/${job.data.build_id}`; if (!dev) await asyncExecShell(`rm -fr ${workdir}`); + await prisma.build.update({ where: { id: job.data.build_id }, data: { status: 'success' } }); } return; }); buildWorker.on('failed', async (job: Bullmq.Job, failedReason) => { try { - await asyncUntil( - async () => { - const found = await prisma.build.findFirst({ - where: { id: job.data.build_id, status: 'failed' } - }); - if (!found) { - return await prisma.build.update({ - where: { id: job.data.build_id }, - data: { status: 'failed' } - }); - } - return true; - }, - 100, - 5 - ); + await prisma.build.update({ where: { id: job.data.build_id }, data: { status: 'failed' } }); } catch (error) { + setTimeout(async () => { + await prisma.build.update({ where: { id: job.data.build_id }, data: { status: 'failed' } }); + }, 1234); console.log(error); } finally { const workdir = `/tmp/build-sources/${job.data.repository}`; if (!dev) await asyncExecShell(`rm -fr ${workdir}`); + await prisma.build.update({ where: { id: job.data.build_id }, data: { status: 'failed' } }); } await saveBuildLog({ line: 'Failed to deploy!', From 41ca265e5a4dd65295c431fdf2f8dacb85be045b Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 5 Apr 2022 11:00:36 +0200 Subject: [PATCH 08/10] Try to not restart redis? --- src/routes/update.json.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/update.json.ts b/src/routes/update.json.ts index 42f4dd6ee..80e6482f0 100644 --- a/src/routes/update.json.ts +++ b/src/routes/update.json.ts @@ -39,7 +39,7 @@ export const post: RequestHandler = async (event) => { await asyncExecShell(`docker pull coollabsio/coolify:${latestVersion}`); await asyncExecShell(`env | grep COOLIFY > .env`); await asyncExecShell( - `docker run --rm -tid --env-file .env -v /var/run/docker.sock:/var/run/docker.sock -v coolify-db coollabsio/coolify:${latestVersion} /bin/sh -c "env | grep COOLIFY > .env && echo 'TAG=${latestVersion}' >> .env && docker stop -t 0 coolify coolify-redis && docker rm coolify coolify-redis && docker compose up -d --force-recreate"` + `docker run --rm -tid --env-file .env -v /var/run/docker.sock:/var/run/docker.sock -v coolify-db coollabsio/coolify:${latestVersion} /bin/sh -c "env | grep COOLIFY > .env && echo 'TAG=${latestVersion}' >> .env && docker stop -t 0 coolify && docker rm coolify && docker compose up -d --force-recreate"` ); return { status: 200, From d7d570393fcd2b5fa52bb1521f7d0b861eb1caef Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 5 Apr 2022 11:09:17 +0200 Subject: [PATCH 09/10] Revert try --- src/routes/update.json.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/update.json.ts b/src/routes/update.json.ts index 80e6482f0..42f4dd6ee 100644 --- a/src/routes/update.json.ts +++ b/src/routes/update.json.ts @@ -39,7 +39,7 @@ export const post: RequestHandler = async (event) => { await asyncExecShell(`docker pull coollabsio/coolify:${latestVersion}`); await asyncExecShell(`env | grep COOLIFY > .env`); await asyncExecShell( - `docker run --rm -tid --env-file .env -v /var/run/docker.sock:/var/run/docker.sock -v coolify-db coollabsio/coolify:${latestVersion} /bin/sh -c "env | grep COOLIFY > .env && echo 'TAG=${latestVersion}' >> .env && docker stop -t 0 coolify && docker rm coolify && docker compose up -d --force-recreate"` + `docker run --rm -tid --env-file .env -v /var/run/docker.sock:/var/run/docker.sock -v coolify-db coollabsio/coolify:${latestVersion} /bin/sh -c "env | grep COOLIFY > .env && echo 'TAG=${latestVersion}' >> .env && docker stop -t 0 coolify coolify-redis && docker rm coolify coolify-redis && docker compose up -d --force-recreate"` ); return { status: 200, From b70fe09d17104e7ba931bf388a1d964aaceea27d Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 5 Apr 2022 11:13:13 +0200 Subject: [PATCH 10/10] fix: Remove asyncUntil --- src/lib/common.ts | 14 -------------- src/lib/queues/index.ts | 2 +- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/src/lib/common.ts b/src/lib/common.ts index d6b36b209..680e35208 100644 --- a/src/lib/common.ts +++ b/src/lib/common.ts @@ -46,20 +46,6 @@ const customConfig: Config = { export const version = currentVersion; export const asyncExecShell = util.promisify(child.exec); export const asyncSleep = (delay) => new Promise((resolve) => setTimeout(resolve, delay)); -export const asyncUntil = (condition, delay, maxIterations) => { - return new Promise(async (resolve, reject) => { - let iterations = 0; - while (!condition()) { - if (maxIterations && iterations >= maxIterations) { - reject(new Error('Max iterations reached.')); - return; - } - await asyncSleep(delay); - iterations++; - } - resolve(); - }); -}; export const sentry = Sentry; diff --git a/src/lib/queues/index.ts b/src/lib/queues/index.ts index c8646c3a1..736d76789 100644 --- a/src/lib/queues/index.ts +++ b/src/lib/queues/index.ts @@ -11,7 +11,7 @@ import proxy from './proxy'; import ssl from './ssl'; import sslrenewal from './sslrenewal'; -import { asyncExecShell, asyncUntil, saveBuildLog } from '$lib/common'; +import { asyncExecShell, saveBuildLog } from '$lib/common'; let { Queue, Worker } = Bullmq; let redisHost = 'localhost';