From 0f8f33e9fe557cbbcbabb4a7506c2deddfe3920c Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Fri, 26 Aug 2022 09:49:17 +0200 Subject: [PATCH] remove unnecessary things --- apps/api/src/jobs/autoUpdater.ts | 43 -- apps/api/src/jobs/checkProxies.ts | 81 ---- apps/api/src/jobs/cleanupPrismaEngines.ts | 19 - apps/api/src/jobs/cleanupStorage.ts | 60 --- apps/api/src/jobs/deployApplication-old.ts | 366 ------------------ apps/api/src/jobs/infrastructure.ts | 6 +- apps/api/src/lib/common.ts | 40 +- apps/api/src/lib/scheduler.ts | 4 +- .../applications/[id]/logs/build.svelte | 1 - 9 files changed, 23 insertions(+), 597 deletions(-) delete mode 100644 apps/api/src/jobs/autoUpdater.ts delete mode 100644 apps/api/src/jobs/checkProxies.ts delete mode 100644 apps/api/src/jobs/cleanupPrismaEngines.ts delete mode 100644 apps/api/src/jobs/cleanupStorage.ts delete mode 100644 apps/api/src/jobs/deployApplication-old.ts diff --git a/apps/api/src/jobs/autoUpdater.ts b/apps/api/src/jobs/autoUpdater.ts deleted file mode 100644 index 566ffea29..000000000 --- a/apps/api/src/jobs/autoUpdater.ts +++ /dev/null @@ -1,43 +0,0 @@ -import axios from 'axios'; -import compareVersions from 'compare-versions'; -import { parentPort } from 'node:worker_threads'; -import { asyncExecShell, asyncSleep, isDev, prisma, version } from '../lib/common'; - -(async () => { - if (parentPort) { - try { - const currentVersion = version; - const { data: versions } = await axios - .get( - `https://get.coollabs.io/versions.json` - , { - params: { - appId: process.env['COOLIFY_APP_ID'] || undefined, - version: currentVersion - } - }) - const latestVersion = versions['coolify'].main.version; - const isUpdateAvailable = compareVersions(latestVersion, currentVersion); - if (isUpdateAvailable === 1) { - const activeCount = 0 - if (activeCount === 0) { - if (!isDev) { - console.log(`Updating Coolify to ${latestVersion}.`); - await asyncExecShell(`docker pull coollabsio/coolify:${latestVersion}`); - await asyncExecShell(`env | grep COOLIFY > .env`); - await asyncExecShell( - `docker run --rm -tid --env-file .env -v /var/run/docker.sock:/var/run/docker.sock -v coolify-db coollabsio/coolify:${latestVersion} /bin/sh -c "env | grep COOLIFY > .env && echo 'TAG=${latestVersion}' >> .env && docker stop -t 0 coolify && docker rm coolify && docker compose up -d --force-recreate"` - ); - } else { - console.log('Updating (not really in dev mode).'); - } - } - } - } catch (error) { - console.log(error); - } finally { - await prisma.$disconnect(); - } - - } else process.exit(0); -})(); diff --git a/apps/api/src/jobs/checkProxies.ts b/apps/api/src/jobs/checkProxies.ts deleted file mode 100644 index 7ba6726ec..000000000 --- a/apps/api/src/jobs/checkProxies.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { parentPort } from 'node:worker_threads'; -import { prisma, startTraefikTCPProxy, generateDatabaseConfiguration, startTraefikProxy, executeDockerCmd, listSettings } from '../lib/common'; -import { checkContainer } from '../lib/docker'; - -(async () => { - if (parentPort) { - try { - const { default: isReachable } = await import('is-port-reachable'); - let portReachable; - - const { arch, ipv4, ipv6 } = await listSettings(); - // Coolify Proxy local - const engine = '/var/run/docker.sock'; - const localDocker = await prisma.destinationDocker.findFirst({ - where: { engine, network: 'coolify' } - }); - if (localDocker && localDocker.isCoolifyProxyUsed) { - portReachable = await isReachable(80, { host: ipv4 || ipv6 }) - console.log({ port: 80, portReachable }); - if (!portReachable) { - await startTraefikProxy(localDocker.id); - } - } - - // TCP Proxies - const databasesWithPublicPort = await prisma.database.findMany({ - where: { publicPort: { not: null } }, - include: { settings: true, destinationDocker: true } - }); - for (const database of databasesWithPublicPort) { - const { destinationDockerId, destinationDocker, publicPort, id } = database; - if (destinationDockerId && destinationDocker.isCoolifyProxyUsed) { - const { privatePort } = generateDatabaseConfiguration(database, arch); - portReachable = await isReachable(publicPort, { host: destinationDocker.remoteIpAddress || ipv4 || ipv6 }) - console.log({ publicPort, portReachable }); - if (!portReachable) { - await startTraefikTCPProxy(destinationDocker, id, publicPort, privatePort); - } - } - } - const wordpressWithFtp = await prisma.wordpress.findMany({ - where: { ftpPublicPort: { not: null } }, - include: { service: { include: { destinationDocker: true } } } - }); - for (const ftp of wordpressWithFtp) { - const { service, ftpPublicPort } = ftp; - const { destinationDockerId, destinationDocker, id } = service; - if (destinationDockerId && destinationDocker.isCoolifyProxyUsed) { - portReachable = await isReachable(ftpPublicPort, { host: destinationDocker.remoteIpAddress || ipv4 || ipv6 }) - console.log({ ftpPublicPort, portReachable }); - if (!portReachable) { - await startTraefikTCPProxy(destinationDocker, id, ftpPublicPort, 22, 'wordpressftp'); - } - } - } - - // HTTP Proxies - const minioInstances = await prisma.minio.findMany({ - where: { publicPort: { not: null } }, - include: { service: { include: { destinationDocker: true } } } - }); - for (const minio of minioInstances) { - const { service, publicPort } = minio; - const { destinationDockerId, destinationDocker, id } = service; - if (destinationDockerId && destinationDocker.isCoolifyProxyUsed) { - portReachable = await isReachable(publicPort, { host: destinationDocker.remoteIpAddress || ipv4 || ipv6 }) - console.log({ publicPort, portReachable }); - if (!portReachable) { - await startTraefikTCPProxy(destinationDocker, id, publicPort, 9000); - } - } - } - - } catch (error) { - - } finally { - await prisma.$disconnect(); - } - - } else process.exit(0); -})(); diff --git a/apps/api/src/jobs/cleanupPrismaEngines.ts b/apps/api/src/jobs/cleanupPrismaEngines.ts deleted file mode 100644 index 335bdce7d..000000000 --- a/apps/api/src/jobs/cleanupPrismaEngines.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { parentPort } from 'node:worker_threads'; -import { asyncExecShell, isDev, prisma } from '../lib/common'; - -(async () => { - if (parentPort) { - if (!isDev) { - try { - const { stdout } = await asyncExecShell(`ps -ef | grep /app/prisma-engines/query-engine | grep -v grep | wc -l | xargs`) - if (stdout.trim() != null && stdout.trim() != '' && Number(stdout.trim()) > 1) { - await asyncExecShell(`killall -q -e /app/prisma-engines/query-engine -o 10m`) - } - } catch (error) { - console.log(error); - } finally { - await prisma.$disconnect(); - } - } - } else process.exit(0); -})(); diff --git a/apps/api/src/jobs/cleanupStorage.ts b/apps/api/src/jobs/cleanupStorage.ts deleted file mode 100644 index 4740c1df8..000000000 --- a/apps/api/src/jobs/cleanupStorage.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { parentPort } from 'node:worker_threads'; -import { asyncExecShell, cleanupDockerStorage, executeDockerCmd, isDev, prisma } from '../lib/common'; - -(async () => { - if (parentPort) { - const destinationDockers = await prisma.destinationDocker.findMany(); - let enginesDone = new Set() - for (const destination of destinationDockers) { - if (enginesDone.has(destination.engine) || enginesDone.has(destination.remoteIpAddress)) return - if (destination.engine) enginesDone.add(destination.engine) - if (destination.remoteIpAddress) enginesDone.add(destination.remoteIpAddress) - - let lowDiskSpace = false; - try { - let stdout = null - if (!isDev) { - const output = await executeDockerCmd({ dockerId: destination.id, command: `CONTAINER=$(docker ps -lq | head -1) && docker exec $CONTAINER sh -c 'df -kPT /'` }) - stdout = output.stdout; - } else { - const output = await asyncExecShell( - `df -kPT /` - ); - stdout = output.stdout; - } - let lines = stdout.trim().split('\n'); - let header = lines[0]; - let regex = - /^Filesystem\s+|Type\s+|1024-blocks|\s+Used|\s+Available|\s+Capacity|\s+Mounted on\s*$/g; - const boundaries = []; - let match; - - while ((match = regex.exec(header))) { - boundaries.push(match[0].length); - } - - boundaries[boundaries.length - 1] = -1; - const data = lines.slice(1).map((line) => { - const cl = boundaries.map((boundary) => { - const column = boundary > 0 ? line.slice(0, boundary) : line; - line = line.slice(boundary); - return column.trim(); - }); - return { - capacity: Number.parseInt(cl[5], 10) / 100 - }; - }); - if (data.length > 0) { - const { capacity } = data[0]; - if (capacity > 0.8) { - lowDiskSpace = true; - } - } - } catch (error) { - console.log(error); - } - await cleanupDockerStorage(destination.id, lowDiskSpace, false) - } - await prisma.$disconnect(); - } else process.exit(0); -})(); diff --git a/apps/api/src/jobs/deployApplication-old.ts b/apps/api/src/jobs/deployApplication-old.ts deleted file mode 100644 index 93d358fe3..000000000 --- a/apps/api/src/jobs/deployApplication-old.ts +++ /dev/null @@ -1,366 +0,0 @@ -import { parentPort } from 'node:worker_threads'; -import crypto from 'crypto'; -import fs from 'fs/promises'; -import yaml from 'js-yaml'; - -import { copyBaseConfigurationFiles, makeLabelForStandaloneApplication, saveBuildLog, setDefaultConfiguration } from '../lib/buildPacks/common'; -import { createDirectories, decrypt, defaultComposeConfiguration, executeDockerCmd, getDomain, prisma } from '../lib/common'; -import * as importers from '../lib/importers'; -import * as buildpacks from '../lib/buildPacks'; - -(async () => { - if (parentPort) { - const concurrency = 1 - const PQueue = await import('p-queue'); - const queue = new PQueue.default({ concurrency }); - parentPort.on('message', async (message) => { - if (parentPort) { - if (message === 'error') throw new Error('oops'); - if (message === 'cancel') { - parentPort.postMessage('cancelled'); - return; - } - if (message === 'status:autoUpdater') { - parentPort.postMessage({ size: queue.size, pending: queue.pending, caller: 'autoUpdater' }); - return; - } - if (message === 'status:cleanupStorage') { - parentPort.postMessage({ size: queue.size, pending: queue.pending, caller: 'cleanupStorage' }); - return; - } - if (message === 'action:flushQueue') { - queue.clear() - return; - } - - await queue.add(async () => { - const { - id: applicationId, - repository, - name, - destinationDocker, - destinationDockerId, - gitSource, - build_id: buildId, - configHash, - fqdn, - projectId, - secrets, - phpModules, - type, - pullmergeRequestId = null, - sourceBranch = null, - settings, - persistentStorage, - pythonWSGI, - pythonModule, - pythonVariable, - denoOptions, - exposePort, - baseImage, - baseBuildImage, - deploymentType, - forceRebuild - } = message - let { - branch, - buildPack, - port, - installCommand, - buildCommand, - startCommand, - baseDirectory, - publishDirectory, - dockerFileLocation, - denoMainFile - } = message - const currentHash = crypto - .createHash('sha256') - .update( - JSON.stringify({ - pythonWSGI, - pythonModule, - pythonVariable, - deploymentType, - denoOptions, - baseImage, - baseBuildImage, - buildPack, - port, - exposePort, - installCommand, - buildCommand, - startCommand, - secrets, - branch, - repository, - fqdn - }) - ) - .digest('hex'); - try { - const { debug } = settings; - if (concurrency === 1) { - await prisma.build.updateMany({ - where: { - status: { in: ['queued', 'running'] }, - id: { not: buildId }, - applicationId, - createdAt: { lt: new Date(new Date().getTime() - 10 * 1000) } - }, - data: { status: 'failed' } - }); - } - let imageId = applicationId; - let domain = getDomain(fqdn); - const volumes = - persistentStorage?.map((storage) => { - return `${applicationId}${storage.path.replace(/\//gi, '-')}:${buildPack !== 'docker' ? '/app' : '' - }${storage.path}`; - }) || []; - // Previews, we need to get the source branch and set subdomain - if (pullmergeRequestId) { - branch = sourceBranch; - domain = `${pullmergeRequestId}.${domain}`; - imageId = `${applicationId}-${pullmergeRequestId}`; - } - - let deployNeeded = true; - let destinationType; - - if (destinationDockerId) { - destinationType = 'docker'; - } - if (destinationType === 'docker') { - await prisma.build.update({ where: { id: buildId }, data: { status: 'running' } }); - const { workdir, repodir } = await createDirectories({ repository, buildId }); - const configuration = await setDefaultConfiguration(message); - - buildPack = configuration.buildPack; - port = configuration.port; - installCommand = configuration.installCommand; - startCommand = configuration.startCommand; - buildCommand = configuration.buildCommand; - publishDirectory = configuration.publishDirectory; - baseDirectory = configuration.baseDirectory; - dockerFileLocation = configuration.dockerFileLocation; - denoMainFile = configuration.denoMainFile; - const commit = await importers[gitSource.type]({ - applicationId, - debug, - workdir, - repodir, - githubAppId: gitSource.githubApp?.id, - gitlabAppId: gitSource.gitlabApp?.id, - customPort: gitSource.customPort, - repository, - branch, - buildId, - apiUrl: gitSource.apiUrl, - htmlUrl: gitSource.htmlUrl, - projectId, - deployKeyId: gitSource.gitlabApp?.deployKeyId || null, - privateSshKey: decrypt(gitSource.gitlabApp?.privateSshKey) || null, - forPublic: gitSource.forPublic - }); - if (!commit) { - throw new Error('No commit found?'); - } - let tag = commit.slice(0, 7); - if (pullmergeRequestId) { - tag = `${commit.slice(0, 7)}-${pullmergeRequestId}`; - } - - try { - await prisma.build.update({ where: { id: buildId }, data: { commit } }); - } catch (err) { - console.log(err); - } - - if (!pullmergeRequestId) { - - if (configHash !== currentHash) { - deployNeeded = true; - if (configHash) { - await saveBuildLog({ line: 'Configuration changed.', buildId, applicationId }); - } - } else { - deployNeeded = false; - } - } else { - deployNeeded = true; - } - - let imageFound = false; - try { - await executeDockerCmd({ - dockerId: destinationDocker.id, - command: `docker image inspect ${applicationId}:${tag}` - }) - imageFound = true; - } catch (error) { - // - } - await copyBaseConfigurationFiles(buildPack, workdir, buildId, applicationId, baseImage); - - if (forceRebuild) deployNeeded = true - if (!imageFound || deployNeeded) { - // if (true) { - if (buildpacks[buildPack]) - await buildpacks[buildPack]({ - dockerId: destinationDocker.id, - buildId, - applicationId, - domain, - name, - type, - pullmergeRequestId, - buildPack, - repository, - branch, - projectId, - publishDirectory, - debug, - commit, - tag, - workdir, - port: exposePort ? `${exposePort}:${port}` : port, - installCommand, - buildCommand, - startCommand, - baseDirectory, - secrets, - phpModules, - pythonWSGI, - pythonModule, - pythonVariable, - dockerFileLocation, - denoMainFile, - denoOptions, - baseImage, - baseBuildImage, - deploymentType - }); - else { - await saveBuildLog({ line: `Build pack ${buildPack} not found`, buildId, applicationId }); - throw new Error(`Build pack ${buildPack} not found.`); - } - } else { - await saveBuildLog({ line: 'Build image already available - no rebuild required.', buildId, applicationId }); - } - try { - await executeDockerCmd({ dockerId: destinationDocker.id, command: `docker stop -t 0 ${imageId}` }) - await executeDockerCmd({ dockerId: destinationDocker.id, command: `docker rm ${imageId}` }) - } catch (error) { - // - } - const envs = [ - `PORT=${port}` - ]; - if (secrets.length > 0) { - secrets.forEach((secret) => { - if (pullmergeRequestId) { - if (secret.isPRMRSecret) { - envs.push(`${secret.name}=${secret.value}`); - } - } else { - if (!secret.isPRMRSecret) { - envs.push(`${secret.name}=${secret.value}`); - } - } - }); - } - await fs.writeFile(`${workdir}/.env`, envs.join('\n')); - const labels = makeLabelForStandaloneApplication({ - applicationId, - fqdn, - name, - type, - pullmergeRequestId, - buildPack, - repository, - branch, - projectId, - port: exposePort ? `${exposePort}:${port}` : port, - commit, - installCommand, - buildCommand, - startCommand, - baseDirectory, - publishDirectory - }); - let envFound = false; - try { - envFound = !!(await fs.stat(`${workdir}/.env`)); - } catch (error) { - // - } - try { - await saveBuildLog({ line: 'Deployment started.', buildId, applicationId }); - const composeVolumes = volumes.map((volume) => { - return { - [`${volume.split(':')[0]}`]: { - name: volume.split(':')[0] - } - }; - }); - const composeFile = { - version: '3.8', - services: { - [imageId]: { - image: `${applicationId}:${tag}`, - container_name: imageId, - volumes, - env_file: envFound ? [`${workdir}/.env`] : [], - labels, - depends_on: [], - expose: [port], - ...(exposePort ? { ports: [`${exposePort}:${port}`] } : {}), - // logging: { - // driver: 'fluentd', - // }, - ...defaultComposeConfiguration(destinationDocker.network), - } - }, - networks: { - [destinationDocker.network]: { - external: true - } - }, - volumes: Object.assign({}, ...composeVolumes) - }; - await fs.writeFile(`${workdir}/docker-compose.yml`, yaml.dump(composeFile)); - await executeDockerCmd({ dockerId: destinationDocker.id, command: `docker compose --project-directory ${workdir} up -d` }) - await saveBuildLog({ line: 'Deployment successful!', buildId, applicationId }); - } catch (error) { - await saveBuildLog({ line: error, buildId, applicationId }); - await prisma.build.updateMany({ - where: { id: message.build_id, status: { in: ['queued', 'running'] } }, - data: { status: 'failed' } - }); - throw new Error(error); - } - await saveBuildLog({ line: 'Proxy will be updated shortly.', buildId, applicationId }); - await prisma.build.update({ where: { id: message.build_id }, data: { status: 'success' } }); - if (!pullmergeRequestId) await prisma.application.update({ - where: { id: applicationId }, - data: { configHash: currentHash } - }); - } - - } - catch (error) { - await prisma.build.updateMany({ - where: { id: message.build_id, status: { in: ['queued', 'running'] } }, - data: { status: 'failed' } - }); - await saveBuildLog({ line: error, buildId, applicationId }); - } finally { - await prisma.$disconnect(); - } - }); - await prisma.$disconnect(); - } - }); - } else process.exit(0); -})(); diff --git a/apps/api/src/jobs/infrastructure.ts b/apps/api/src/jobs/infrastructure.ts index 740d011d2..d19163686 100644 --- a/apps/api/src/jobs/infrastructure.ts +++ b/apps/api/src/jobs/infrastructure.ts @@ -52,7 +52,6 @@ async function checkProxies() { }); if (localDocker && localDocker.isCoolifyProxyUsed) { portReachable = await isReachable(80, { host: ipv4 || ipv6 }) - console.log({ port: 80, portReachable }); if (!portReachable) { await startTraefikProxy(localDocker.id); } @@ -68,7 +67,6 @@ async function checkProxies() { if (destinationDockerId && destinationDocker.isCoolifyProxyUsed) { const { privatePort } = generateDatabaseConfiguration(database, arch); portReachable = await isReachable(publicPort, { host: destinationDocker.remoteIpAddress || ipv4 || ipv6 }) - console.log({ publicPort, portReachable }); if (!portReachable) { await startTraefikTCPProxy(destinationDocker, id, publicPort, privatePort); } @@ -83,7 +81,6 @@ async function checkProxies() { const { destinationDockerId, destinationDocker, id } = service; if (destinationDockerId && destinationDocker.isCoolifyProxyUsed) { portReachable = await isReachable(ftpPublicPort, { host: destinationDocker.remoteIpAddress || ipv4 || ipv6 }) - console.log({ ftpPublicPort, portReachable }); if (!portReachable) { await startTraefikTCPProxy(destinationDocker, id, ftpPublicPort, 22, 'wordpressftp'); } @@ -100,14 +97,13 @@ async function checkProxies() { const { destinationDockerId, destinationDocker, id } = service; if (destinationDockerId && destinationDocker.isCoolifyProxyUsed) { portReachable = await isReachable(publicPort, { host: destinationDocker.remoteIpAddress || ipv4 || ipv6 }) - console.log({ publicPort, portReachable }); if (!portReachable) { await startTraefikTCPProxy(destinationDocker, id, publicPort, 9000); } } } } catch (error) { - + } } async function cleanupPrismaEngines() { diff --git a/apps/api/src/lib/common.ts b/apps/api/src/lib/common.ts index da4cd3ebd..148bacfff 100644 --- a/apps/api/src/lib/common.ts +++ b/apps/api/src/lib/common.ts @@ -135,32 +135,32 @@ export const asyncSleep = (delay: number): Promise => new Promise((resolve) => setTimeout(resolve, delay)); export const prisma = new PrismaClient({ errorFormat: 'minimal', - log: [ - { - emit: 'event', - level: 'query', - }, - { - emit: 'stdout', - level: 'error', - }, - { - emit: 'stdout', - level: 'info', - }, - { - emit: 'stdout', - level: 'warn', - }, - ], + // log: [ + // { + // emit: 'event', + // level: 'query', + // }, + // { + // emit: 'stdout', + // level: 'error', + // }, + // { + // emit: 'stdout', + // level: 'info', + // }, + // { + // emit: 'stdout', + // level: 'warn', + // }, + // ], }); -prisma.$on('query', (e) => { +// prisma.$on('query', (e) => { // console.log({e}) // console.log('Query: ' + e.query) // console.log('Params: ' + e.params) // console.log('Duration: ' + e.duration + 'ms') - }) +// }) export const base64Encode = (text: string): string => { return Buffer.from(text).toString('base64'); }; diff --git a/apps/api/src/lib/scheduler.ts b/apps/api/src/lib/scheduler.ts index 2e7db85cb..ebff53e12 100644 --- a/apps/api/src/lib/scheduler.ts +++ b/apps/api/src/lib/scheduler.ts @@ -9,7 +9,8 @@ Bree.extend(TSBree); const options: any = { defaultExtension: 'js', - logger: new Cabin(), + // logger: new Cabin(), + logger: false, workerMessageHandler: async ({ name, message }) => { if (name === 'deployApplication' && message?.deploying) { if (scheduler.workers.has('autoUpdater') || scheduler.workers.has('cleanupStorage')) { @@ -24,7 +25,6 @@ const options: any = { }; if (isDev) options.root = path.join(__dirname, '../jobs'); - export const scheduler = new Bree(options); diff --git a/apps/ui/src/routes/applications/[id]/logs/build.svelte b/apps/ui/src/routes/applications/[id]/logs/build.svelte index 56faaf7d5..d1373e71e 100644 --- a/apps/ui/src/routes/applications/[id]/logs/build.svelte +++ b/apps/ui/src/routes/applications/[id]/logs/build.svelte @@ -158,7 +158,6 @@ {build.type} -
{#if build.status === 'running'}