From 0a448672402e4393dc2c5d5d86ddc547c328995e Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Fri, 12 Aug 2022 09:38:02 +0200 Subject: [PATCH 1/5] chore: version++ --- apps/api/src/lib/common.ts | 12 ++++++------ package.json | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/api/src/lib/common.ts b/apps/api/src/lib/common.ts index ca1caa697..ccbd969c9 100644 --- a/apps/api/src/lib/common.ts +++ b/apps/api/src/lib/common.ts @@ -17,7 +17,7 @@ import { checkContainer, removeContainer } from './docker'; import { day } from './dayjs'; import * as serviceFields from './serviceFields' -export const version = '3.2.2'; +export const version = '3.2.3'; export const isDev = process.env.NODE_ENV === 'development'; const algorithm = 'aes-256-ctr'; @@ -38,8 +38,8 @@ export function getAPIUrl() { const newURL = href.replace('https://', 'https://3001-').replace(/\/$/, '') return newURL } - if (process.env.CODESANDBOX_HOST) { - return `https://${process.env.CODESANDBOX_HOST.replace(/\$PORT/,'3001')}` + if (process.env.CODESANDBOX_HOST) { + return `https://${process.env.CODESANDBOX_HOST.replace(/\$PORT/, '3001')}` } return isDev ? 'http://localhost:3001' : 'http://localhost:3000'; } @@ -50,8 +50,8 @@ export function getUIUrl() { const newURL = href.replace('https://', 'https://3000-').replace(/\/$/, '') return newURL } - if (process.env.CODESANDBOX_HOST) { - return `https://${process.env.CODESANDBOX_HOST.replace(/\$PORT/,'3000')}` + if (process.env.CODESANDBOX_HOST) { + return `https://${process.env.CODESANDBOX_HOST.replace(/\$PORT/, '3000')}` } return 'http://localhost:3000'; } @@ -1637,7 +1637,7 @@ export function persistentVolumes(id, persistentStorage, config) { return `${id}${storage.path.replace(/\//gi, '-')}:${storage.path}`; }) || []; - let volumes = [ ...persistentVolume] + let volumes = [...persistentVolume] if (config.volume) volumes = [config.volume, ...volumes] const composeVolumes = volumes.length > 0 && volumes.map((volume) => { diff --git a/package.json b/package.json index 6127427d4..751b76f3b 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "coolify", "description": "An open-source & self-hostable Heroku / Netlify alternative.", - "version": "3.2.2", + "version": "3.2.3", "license": "Apache-2.0", "repository": "github:coollabsio/coolify", "scripts": { From e92d0914c284801366f951da160c083beb661acb Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Fri, 12 Aug 2022 09:38:11 +0200 Subject: [PATCH 2/5] fix: cleanup stucked prisma-engines --- Dockerfile | 2 +- apps/api/src/index.ts | 5 +- apps/api/src/jobs/checkProxies.ts | 147 +++++++++++----------- apps/api/src/jobs/cleanupPrismaEngines.ts | 16 +++ apps/api/src/lib/scheduler.ts | 3 + 5 files changed, 99 insertions(+), 74 deletions(-) create mode 100644 apps/api/src/jobs/cleanupPrismaEngines.ts diff --git a/Dockerfile b/Dockerfile index 239399c98..778d46764 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ ENV PRISMA_QUERY_ENGINE_BINARY=/app/prisma-engines/query-engine \ COPY --from=coollabsio/prisma-engine:3.15 /prisma-engines/query-engine /prisma-engines/migration-engine /prisma-engines/introspection-engine /prisma-engines/prisma-fmt /app/prisma-engines/ -RUN apk add --no-cache git git-lfs openssh-client curl jq cmake sqlite openssl +RUN apk add --no-cache git git-lfs openssh-client curl jq cmake sqlite openssl psmisc RUN curl -sL https://unpkg.com/@pnpm/self-installer | node RUN mkdir -p ~/.docker/cli-plugins/ diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 4b71f70d8..ed862a59e 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -104,6 +104,7 @@ fastify.listen({ port, host }, async (err: any, address: any) => { await initServer(); await scheduler.start('deployApplication'); await scheduler.start('cleanupStorage'); + await scheduler.start('cleanupPrismaEngines'); await scheduler.start('checkProxies'); // Check if no build is running @@ -116,14 +117,14 @@ fastify.listen({ port, host }, async (err: any, address: any) => { scheduler.workers.get('deployApplication').postMessage("status:autoUpdater"); } } - }, 60000 * 15) + }, isDev ? 5000 : 60000 * 15) // Cleanup storage setInterval(async () => { if (scheduler.workers.has('deployApplication')) { scheduler.workers.get('deployApplication').postMessage("status:cleanupStorage"); } - }, 60000 * 10) + }, isDev ? 5000 : 60000 * 10) scheduler.on('worker deleted', async (name) => { if (name === 'autoUpdater' || name === 'cleanupStorage') { diff --git a/apps/api/src/jobs/checkProxies.ts b/apps/api/src/jobs/checkProxies.ts index 761554061..e307ef858 100644 --- a/apps/api/src/jobs/checkProxies.ts +++ b/apps/api/src/jobs/checkProxies.ts @@ -4,87 +4,92 @@ import { checkContainer } from '../lib/docker'; (async () => { if (parentPort) { - // Coolify Proxy local - const engine = '/var/run/docker.sock'; - const localDocker = await prisma.destinationDocker.findFirst({ - where: { engine, network: 'coolify' } - }); - if (localDocker && localDocker.isCoolifyProxyUsed) { - // Remove HAProxy - const found = await checkContainer({ dockerId: localDocker.id, container: 'coolify-haproxy' }); - if (found) { - await executeDockerCmd({ - dockerId: localDocker.id, - command: `docker stop -t 0 coolify-haproxy && docker rm coolify-haproxy` - }) - } - await startTraefikProxy(localDocker.id); - - } - - // TCP Proxies - const databasesWithPublicPort = await prisma.database.findMany({ - where: { publicPort: { not: null } }, - include: { settings: true, destinationDocker: true } - }); - for (const database of databasesWithPublicPort) { - const { destinationDockerId, destinationDocker, publicPort, id } = database; - if (destinationDockerId && destinationDocker.isCoolifyProxyUsed) { - const { privatePort } = generateDatabaseConfiguration(database); + try { + // Coolify Proxy local + const engine = '/var/run/docker.sock'; + const localDocker = await prisma.destinationDocker.findFirst({ + where: { engine, network: 'coolify' } + }); + if (localDocker && localDocker.isCoolifyProxyUsed) { // Remove HAProxy - const found = await checkContainer({ - dockerId: localDocker.id, container: `haproxy-for-${publicPort}` - }); + const found = await checkContainer({ dockerId: localDocker.id, container: 'coolify-haproxy' }); if (found) { await executeDockerCmd({ dockerId: localDocker.id, - command: `docker stop -t 0 haproxy-for-${publicPort} && docker rm haproxy-for-${publicPort}` + command: `docker stop -t 0 coolify-haproxy && docker rm coolify-haproxy` }) } - await startTraefikTCPProxy(destinationDocker, id, publicPort, privatePort); + await startTraefikProxy(localDocker.id); + } - } - } - const wordpressWithFtp = await prisma.wordpress.findMany({ - where: { ftpPublicPort: { not: null } }, - include: { service: { include: { destinationDocker: true } } } - }); - for (const ftp of wordpressWithFtp) { - const { service, ftpPublicPort } = ftp; - const { destinationDockerId, destinationDocker, id } = service; - if (destinationDockerId && destinationDocker.isCoolifyProxyUsed) { - // Remove HAProxy - const found = await checkContainer({ dockerId: localDocker.id, container: `haproxy-for-${ftpPublicPort}` }); - if (found) { - await executeDockerCmd({ - dockerId: localDocker.id, - command: `docker stop -t 0 haproxy -for-${ftpPublicPort} && docker rm haproxy-for-${ftpPublicPort}` - }) + // TCP Proxies + const databasesWithPublicPort = await prisma.database.findMany({ + where: { publicPort: { not: null } }, + include: { settings: true, destinationDocker: true } + }); + for (const database of databasesWithPublicPort) { + const { destinationDockerId, destinationDocker, publicPort, id } = database; + if (destinationDockerId && destinationDocker.isCoolifyProxyUsed) { + const { privatePort } = generateDatabaseConfiguration(database); + // Remove HAProxy + const found = await checkContainer({ + dockerId: localDocker.id, container: `haproxy-for-${publicPort}` + }); + if (found) { + await executeDockerCmd({ + dockerId: localDocker.id, + command: `docker stop -t 0 haproxy-for-${publicPort} && docker rm haproxy-for-${publicPort}` + }) + } + await startTraefikTCPProxy(destinationDocker, id, publicPort, privatePort); } - await startTraefikTCPProxy(destinationDocker, id, ftpPublicPort, 22, 'wordpressftp'); } + const wordpressWithFtp = await prisma.wordpress.findMany({ + where: { ftpPublicPort: { not: null } }, + include: { service: { include: { destinationDocker: true } } } + }); + for (const ftp of wordpressWithFtp) { + const { service, ftpPublicPort } = ftp; + const { destinationDockerId, destinationDocker, id } = service; + if (destinationDockerId && destinationDocker.isCoolifyProxyUsed) { + // Remove HAProxy + const found = await checkContainer({ dockerId: localDocker.id, container: `haproxy-for-${ftpPublicPort}` }); + if (found) { + await executeDockerCmd({ + dockerId: localDocker.id, + command: `docker stop -t 0 haproxy -for-${ftpPublicPort} && docker rm haproxy-for-${ftpPublicPort}` + }) + } + await startTraefikTCPProxy(destinationDocker, id, ftpPublicPort, 22, 'wordpressftp'); + } + } + + // HTTP Proxies + const minioInstances = await prisma.minio.findMany({ + where: { publicPort: { not: null } }, + include: { service: { include: { destinationDocker: true } } } + }); + for (const minio of minioInstances) { + const { service, publicPort } = minio; + const { destinationDockerId, destinationDocker, id } = service; + if (destinationDockerId && destinationDocker.isCoolifyProxyUsed) { + // Remove HAProxy + const found = await checkContainer({ dockerId: localDocker.id, container: `${id}-${publicPort}` }); + if (found) { + await executeDockerCmd({ + dockerId: localDocker.id, + command: `docker stop -t 0 ${id}-${publicPort} && docker rm ${id}-${publicPort} ` + }) + } + await startTraefikTCPProxy(destinationDocker, id, publicPort, 9000); + } + } + + } catch (error) { + + } finally { + await prisma.$disconnect(); } - // HTTP Proxies - const minioInstances = await prisma.minio.findMany({ - where: { publicPort: { not: null } }, - include: { service: { include: { destinationDocker: true } } } - }); - for (const minio of minioInstances) { - const { service, publicPort } = minio; - const { destinationDockerId, destinationDocker, id } = service; - if (destinationDockerId && destinationDocker.isCoolifyProxyUsed) { - // Remove HAProxy - const found = await checkContainer({ dockerId: localDocker.id, container: `${id}-${publicPort}` }); - if (found) { - await executeDockerCmd({ - dockerId: localDocker.id, - command: `docker stop -t 0 ${id}-${publicPort} && docker rm ${id}-${publicPort} ` - }) - } - await startTraefikTCPProxy(destinationDocker, id, publicPort, 9000); - } - } - await prisma.$disconnect(); } else process.exit(0); })(); diff --git a/apps/api/src/jobs/cleanupPrismaEngines.ts b/apps/api/src/jobs/cleanupPrismaEngines.ts new file mode 100644 index 000000000..b1c912edc --- /dev/null +++ b/apps/api/src/jobs/cleanupPrismaEngines.ts @@ -0,0 +1,16 @@ +import { parentPort } from 'node:worker_threads'; +import { asyncExecShell, isDev, prisma } from '../lib/common'; + +(async () => { + if (parentPort) { + if (!isDev) { + try { + await asyncExecShell(`killall -q -e /app/prisma-engines/query-engine -o 10m`) + } catch (error) { + console.log(error); + } finally { + await prisma.$disconnect(); + } + } + } else process.exit(0); +})(); diff --git a/apps/api/src/lib/scheduler.ts b/apps/api/src/lib/scheduler.ts index c32226bc9..d48ed1f11 100644 --- a/apps/api/src/lib/scheduler.ts +++ b/apps/api/src/lib/scheduler.ts @@ -35,6 +35,9 @@ const options: any = { { name: 'cleanupStorage', }, + { + name: 'cleanupPrismaEngines', + }, { name: 'checkProxies', interval: '10s' From d6cfc2624f8a1df0f2871bd11f8a674253e61078 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Fri, 12 Aug 2022 10:08:48 +0200 Subject: [PATCH 3/5] fix: toast --- apps/ui/src/routes/applications/[id]/_Secret.svelte | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/apps/ui/src/routes/applications/[id]/_Secret.svelte b/apps/ui/src/routes/applications/[id]/_Secret.svelte index 3ba71062c..12d69db19 100644 --- a/apps/ui/src/routes/applications/[id]/_Secret.svelte +++ b/apps/ui/src/routes/applications/[id]/_Secret.svelte @@ -39,7 +39,7 @@ async function createSecret(isNew: any) { try { - if (!name || !value) return + if (!name || !value) return; await saveSecret({ isNew, name, @@ -53,12 +53,16 @@ name = ''; value = ''; isBuildSecret = false; + addToast({ + message: 'Secret added.', + type: 'success' + }); } - dispatch('refresh'); addToast({ - message: 'Secret removed.', + message: 'Secret updated.', type: 'success' }); + dispatch('refresh'); } catch (error) { console.log(error); return errorNotification(error); @@ -79,7 +83,7 @@ applicationId: id }); addToast({ - message: 'Secret removed.', + message: 'Secret updated.', type: 'success' }); } From ddd09412cd33bf7c429e0c2c9c04ca9e8d04eaca Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Fri, 12 Aug 2022 10:18:05 +0200 Subject: [PATCH 4/5] fix: Secrets --- apps/api/src/jobs/deployApplication.ts | 6 +++--- apps/api/src/lib/buildPacks/common.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/api/src/jobs/deployApplication.ts b/apps/api/src/jobs/deployApplication.ts index c780595f9..9782067d1 100644 --- a/apps/api/src/jobs/deployApplication.ts +++ b/apps/api/src/jobs/deployApplication.ts @@ -192,9 +192,9 @@ import * as buildpacks from '../lib/buildPacks'; } catch (error) { // } + await copyBaseConfigurationFiles(buildPack, workdir, buildId, applicationId, baseImage); if (!imageFound || deployNeeded) { // if (true) { - await copyBaseConfigurationFiles(buildPack, workdir, buildId, applicationId, baseImage); if (buildpacks[buildPack]) await buildpacks[buildPack]({ dockerId: destinationDocker.id, @@ -248,11 +248,11 @@ import * as buildpacks from '../lib/buildPacks'; secrets.forEach((secret) => { if (pullmergeRequestId) { if (secret.isPRMRSecret) { - envs.push(`${secret.name}=${secret.value}`); + envs.push(`${secret.name}='${secret.value}'`); } } else { if (!secret.isPRMRSecret) { - envs.push(`${secret.name}=${secret.value}`); + envs.push(`${secret.name}='${secret.value}'`); } } }); diff --git a/apps/api/src/lib/buildPacks/common.ts b/apps/api/src/lib/buildPacks/common.ts index 3c34c5f31..62aa271c1 100644 --- a/apps/api/src/lib/buildPacks/common.ts +++ b/apps/api/src/lib/buildPacks/common.ts @@ -523,7 +523,7 @@ export async function buildImage({ await saveBuildLog({ line: `Building image started.`, buildId, applicationId }); } if (debug) { - await saveBuildLog({ line: `\n###############\nIMPORTANT: Due to some issues during implementing Remote Docker Engine, the builds logs are not streamed at the moment. You will see the full build log when the build is finished!\n###############`, buildId, applicationId }); + await saveBuildLog({ line: `\n###############\nIMPORTANT: Due to some issues during implementing Remote Docker Engine, the builds logs are not streamed at the moment - but will be soon! You will see the full build log when the build is finished!\n###############`, buildId, applicationId }); } if (!debug && isCache) { await saveBuildLog({ From 5d5a478cd15810c6031ece69ea0cb0dec3dcca5e Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Fri, 12 Aug 2022 10:36:51 +0200 Subject: [PATCH 5/5] fix: cleanup prisma engine if there is more than 1 --- apps/api/src/jobs/cleanupPrismaEngines.ts | 5 ++++- apps/api/src/lib/scheduler.ts | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/api/src/jobs/cleanupPrismaEngines.ts b/apps/api/src/jobs/cleanupPrismaEngines.ts index b1c912edc..335bdce7d 100644 --- a/apps/api/src/jobs/cleanupPrismaEngines.ts +++ b/apps/api/src/jobs/cleanupPrismaEngines.ts @@ -5,7 +5,10 @@ import { asyncExecShell, isDev, prisma } from '../lib/common'; if (parentPort) { if (!isDev) { try { - await asyncExecShell(`killall -q -e /app/prisma-engines/query-engine -o 10m`) + const { stdout } = await asyncExecShell(`ps -ef | grep /app/prisma-engines/query-engine | grep -v grep | wc -l | xargs`) + if (stdout.trim() != null && stdout.trim() != '' && Number(stdout.trim()) > 1) { + await asyncExecShell(`killall -q -e /app/prisma-engines/query-engine -o 10m`) + } } catch (error) { console.log(error); } finally { diff --git a/apps/api/src/lib/scheduler.ts b/apps/api/src/lib/scheduler.ts index d48ed1f11..148764351 100644 --- a/apps/api/src/lib/scheduler.ts +++ b/apps/api/src/lib/scheduler.ts @@ -37,6 +37,7 @@ const options: any = { }, { name: 'cleanupPrismaEngines', + interval: '1m' }, { name: 'checkProxies',