diff --git a/apps/api/package.json b/apps/api/package.json index 3fe675596..6b48e4b78 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -22,6 +22,7 @@ "@fastify/jwt": "6.3.2", "@fastify/static": "6.5.0", "@iarna/toml": "2.2.5", + "@ladjs/graceful": "3.0.2", "@prisma/client": "3.15.2", "axios": "0.27.2", "bcryptjs": "2.4.3", diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 3f4876c94..dac8d34ef 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -7,9 +7,8 @@ import path, { join } from 'path'; import autoLoad from '@fastify/autoload'; import { asyncExecShell, asyncSleep, isDev, listSettings, prisma, version } from './lib/common'; import { scheduler } from './lib/scheduler'; -import axios from 'axios'; import compareVersions from 'compare-versions'; - +import Graceful from '@ladjs/graceful' declare module 'fastify' { interface FastifyInstance { config: { @@ -104,45 +103,38 @@ fastify.listen({ port, host }, async (err: any, address: any) => { } console.log(`Coolify's API is listening on ${host}:${port}`); await initServer(); - await scheduler.start('cleanupPrismaEngines'); - await scheduler.start('checkProxies'); + const graceful = new Graceful({ brees: [scheduler] }); + graceful.listen(); + setInterval(async () => { if (!scheduler.workers.has('deployApplication')) { scheduler.run('deployApplication'); } + if (!scheduler.workers.has('infrastructure')) { + scheduler.run('infrastructure'); + } }, 2000) - // Check for update & if no build is running + // autoUpdater setInterval(async () => { - const { isAutoUpdateEnabled } = await prisma.setting.findFirst(); - if (isAutoUpdateEnabled) { - const currentVersion = version; - const { data: versions } = await axios - .get( - `https://get.coollabs.io/versions.json` - , { - params: { - appId: process.env['COOLIFY_APP_ID'] || undefined, - version: currentVersion - } - }) - const latestVersion = versions['coolify'].main.version; - const isUpdateAvailable = compareVersions(latestVersion, currentVersion); - if (isUpdateAvailable === 1) { - if (!scheduler.workers.has('deployApplication')) { - await scheduler.run('autoUpdater') - } - } - } + scheduler.workers.has('infrastructure') && scheduler.workers.get('infrastructure').postMessage("action:autoUpdater") }, isDev ? 5000 : 60000 * 15) - // Cleanup storage + // cleanupStorage setInterval(async () => { - if (!scheduler.workers.has('deployApplication') && !scheduler.workers.has('cleanupStorage')) { - await scheduler.run('cleanupStorage') - } - }, isDev ? 5000 : 60000 * 10) + scheduler.workers.has('infrastructure') && scheduler.workers.get('infrastructure').postMessage("action:cleanupStorage") + }, isDev ? 6000 : 60000 * 10) + + // checkProxies + setInterval(async () => { + scheduler.workers.has('infrastructure') && scheduler.workers.get('infrastructure').postMessage("action:checkProxies") + }, 10000) + + // cleanupPrismaEngines + // setInterval(async () => { + // scheduler.workers.has('infrastructure') && scheduler.workers.get('infrastructure').postMessage("action:cleanupPrismaEngines") + // }, 60000) await getArch(); await getIPAddress(); diff --git a/apps/api/src/jobs/autoUpdater.ts b/apps/api/src/jobs/autoUpdater.ts deleted file mode 100644 index 566ffea29..000000000 --- a/apps/api/src/jobs/autoUpdater.ts +++ /dev/null @@ -1,43 +0,0 @@ -import axios from 'axios'; -import compareVersions from 'compare-versions'; -import { parentPort } from 'node:worker_threads'; -import { asyncExecShell, asyncSleep, isDev, prisma, version } from '../lib/common'; - -(async () => { - if (parentPort) { - try { - const currentVersion = version; - const { data: versions } = await axios - .get( - `https://get.coollabs.io/versions.json` - , { - params: { - appId: process.env['COOLIFY_APP_ID'] || undefined, - version: currentVersion - } - }) - const latestVersion = versions['coolify'].main.version; - const isUpdateAvailable = compareVersions(latestVersion, currentVersion); - if (isUpdateAvailable === 1) { - const activeCount = 0 - if (activeCount === 0) { - if (!isDev) { - console.log(`Updating Coolify to ${latestVersion}.`); - await asyncExecShell(`docker pull coollabsio/coolify:${latestVersion}`); - await asyncExecShell(`env | grep COOLIFY > .env`); - await asyncExecShell( - `docker run --rm -tid --env-file .env -v /var/run/docker.sock:/var/run/docker.sock -v coolify-db coollabsio/coolify:${latestVersion} /bin/sh -c "env | grep COOLIFY > .env && echo 'TAG=${latestVersion}' >> .env && docker stop -t 0 coolify && docker rm coolify && docker compose up -d --force-recreate"` - ); - } else { - console.log('Updating (not really in dev mode).'); - } - } - } - } catch (error) { - console.log(error); - } finally { - await prisma.$disconnect(); - } - - } else process.exit(0); -})(); diff --git a/apps/api/src/jobs/checkProxies.ts b/apps/api/src/jobs/checkProxies.ts deleted file mode 100644 index 7ba6726ec..000000000 --- a/apps/api/src/jobs/checkProxies.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { parentPort } from 'node:worker_threads'; -import { prisma, startTraefikTCPProxy, generateDatabaseConfiguration, startTraefikProxy, executeDockerCmd, listSettings } from '../lib/common'; -import { checkContainer } from '../lib/docker'; - -(async () => { - if (parentPort) { - try { - const { default: isReachable } = await import('is-port-reachable'); - let portReachable; - - const { arch, ipv4, ipv6 } = await listSettings(); - // Coolify Proxy local - const engine = '/var/run/docker.sock'; - const localDocker = await prisma.destinationDocker.findFirst({ - where: { engine, network: 'coolify' } - }); - if (localDocker && localDocker.isCoolifyProxyUsed) { - portReachable = await isReachable(80, { host: ipv4 || ipv6 }) - console.log({ port: 80, portReachable }); - if (!portReachable) { - await startTraefikProxy(localDocker.id); - } - } - - // TCP Proxies - const databasesWithPublicPort = await prisma.database.findMany({ - where: { publicPort: { not: null } }, - include: { settings: true, destinationDocker: true } - }); - for (const database of databasesWithPublicPort) { - const { destinationDockerId, destinationDocker, publicPort, id } = database; - if (destinationDockerId && destinationDocker.isCoolifyProxyUsed) { - const { privatePort } = generateDatabaseConfiguration(database, arch); - portReachable = await isReachable(publicPort, { host: destinationDocker.remoteIpAddress || ipv4 || ipv6 }) - console.log({ publicPort, portReachable }); - if (!portReachable) { - await startTraefikTCPProxy(destinationDocker, id, publicPort, privatePort); - } - } - } - const wordpressWithFtp = await prisma.wordpress.findMany({ - where: { ftpPublicPort: { not: null } }, - include: { service: { include: { destinationDocker: true } } } - }); - for (const ftp of wordpressWithFtp) { - const { service, ftpPublicPort } = ftp; - const { destinationDockerId, destinationDocker, id } = service; - if (destinationDockerId && destinationDocker.isCoolifyProxyUsed) { - portReachable = await isReachable(ftpPublicPort, { host: destinationDocker.remoteIpAddress || ipv4 || ipv6 }) - console.log({ ftpPublicPort, portReachable }); - if (!portReachable) { - await startTraefikTCPProxy(destinationDocker, id, ftpPublicPort, 22, 'wordpressftp'); - } - } - } - - // HTTP Proxies - const minioInstances = await prisma.minio.findMany({ - where: { publicPort: { not: null } }, - include: { service: { include: { destinationDocker: true } } } - }); - for (const minio of minioInstances) { - const { service, publicPort } = minio; - const { destinationDockerId, destinationDocker, id } = service; - if (destinationDockerId && destinationDocker.isCoolifyProxyUsed) { - portReachable = await isReachable(publicPort, { host: destinationDocker.remoteIpAddress || ipv4 || ipv6 }) - console.log({ publicPort, portReachable }); - if (!portReachable) { - await startTraefikTCPProxy(destinationDocker, id, publicPort, 9000); - } - } - } - - } catch (error) { - - } finally { - await prisma.$disconnect(); - } - - } else process.exit(0); -})(); diff --git a/apps/api/src/jobs/cleanupPrismaEngines.ts b/apps/api/src/jobs/cleanupPrismaEngines.ts deleted file mode 100644 index 335bdce7d..000000000 --- a/apps/api/src/jobs/cleanupPrismaEngines.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { parentPort } from 'node:worker_threads'; -import { asyncExecShell, isDev, prisma } from '../lib/common'; - -(async () => { - if (parentPort) { - if (!isDev) { - try { - const { stdout } = await asyncExecShell(`ps -ef | grep /app/prisma-engines/query-engine | grep -v grep | wc -l | xargs`) - if (stdout.trim() != null && stdout.trim() != '' && Number(stdout.trim()) > 1) { - await asyncExecShell(`killall -q -e /app/prisma-engines/query-engine -o 10m`) - } - } catch (error) { - console.log(error); - } finally { - await prisma.$disconnect(); - } - } - } else process.exit(0); -})(); diff --git a/apps/api/src/jobs/cleanupStorage.ts b/apps/api/src/jobs/cleanupStorage.ts deleted file mode 100644 index 4740c1df8..000000000 --- a/apps/api/src/jobs/cleanupStorage.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { parentPort } from 'node:worker_threads'; -import { asyncExecShell, cleanupDockerStorage, executeDockerCmd, isDev, prisma } from '../lib/common'; - -(async () => { - if (parentPort) { - const destinationDockers = await prisma.destinationDocker.findMany(); - let enginesDone = new Set() - for (const destination of destinationDockers) { - if (enginesDone.has(destination.engine) || enginesDone.has(destination.remoteIpAddress)) return - if (destination.engine) enginesDone.add(destination.engine) - if (destination.remoteIpAddress) enginesDone.add(destination.remoteIpAddress) - - let lowDiskSpace = false; - try { - let stdout = null - if (!isDev) { - const output = await executeDockerCmd({ dockerId: destination.id, command: `CONTAINER=$(docker ps -lq | head -1) && docker exec $CONTAINER sh -c 'df -kPT /'` }) - stdout = output.stdout; - } else { - const output = await asyncExecShell( - `df -kPT /` - ); - stdout = output.stdout; - } - let lines = stdout.trim().split('\n'); - let header = lines[0]; - let regex = - /^Filesystem\s+|Type\s+|1024-blocks|\s+Used|\s+Available|\s+Capacity|\s+Mounted on\s*$/g; - const boundaries = []; - let match; - - while ((match = regex.exec(header))) { - boundaries.push(match[0].length); - } - - boundaries[boundaries.length - 1] = -1; - const data = lines.slice(1).map((line) => { - const cl = boundaries.map((boundary) => { - const column = boundary > 0 ? line.slice(0, boundary) : line; - line = line.slice(boundary); - return column.trim(); - }); - return { - capacity: Number.parseInt(cl[5], 10) / 100 - }; - }); - if (data.length > 0) { - const { capacity } = data[0]; - if (capacity > 0.8) { - lowDiskSpace = true; - } - } - } catch (error) { - console.log(error); - } - await cleanupDockerStorage(destination.id, lowDiskSpace, false) - } - await prisma.$disconnect(); - } else process.exit(0); -})(); diff --git a/apps/api/src/jobs/deployApplication-old.ts b/apps/api/src/jobs/deployApplication-old.ts deleted file mode 100644 index 93d358fe3..000000000 --- a/apps/api/src/jobs/deployApplication-old.ts +++ /dev/null @@ -1,366 +0,0 @@ -import { parentPort } from 'node:worker_threads'; -import crypto from 'crypto'; -import fs from 'fs/promises'; -import yaml from 'js-yaml'; - -import { copyBaseConfigurationFiles, makeLabelForStandaloneApplication, saveBuildLog, setDefaultConfiguration } from '../lib/buildPacks/common'; -import { createDirectories, decrypt, defaultComposeConfiguration, executeDockerCmd, getDomain, prisma } from '../lib/common'; -import * as importers from '../lib/importers'; -import * as buildpacks from '../lib/buildPacks'; - -(async () => { - if (parentPort) { - const concurrency = 1 - const PQueue = await import('p-queue'); - const queue = new PQueue.default({ concurrency }); - parentPort.on('message', async (message) => { - if (parentPort) { - if (message === 'error') throw new Error('oops'); - if (message === 'cancel') { - parentPort.postMessage('cancelled'); - return; - } - if (message === 'status:autoUpdater') { - parentPort.postMessage({ size: queue.size, pending: queue.pending, caller: 'autoUpdater' }); - return; - } - if (message === 'status:cleanupStorage') { - parentPort.postMessage({ size: queue.size, pending: queue.pending, caller: 'cleanupStorage' }); - return; - } - if (message === 'action:flushQueue') { - queue.clear() - return; - } - - await queue.add(async () => { - const { - id: applicationId, - repository, - name, - destinationDocker, - destinationDockerId, - gitSource, - build_id: buildId, - configHash, - fqdn, - projectId, - secrets, - phpModules, - type, - pullmergeRequestId = null, - sourceBranch = null, - settings, - persistentStorage, - pythonWSGI, - pythonModule, - pythonVariable, - denoOptions, - exposePort, - baseImage, - baseBuildImage, - deploymentType, - forceRebuild - } = message - let { - branch, - buildPack, - port, - installCommand, - buildCommand, - startCommand, - baseDirectory, - publishDirectory, - dockerFileLocation, - denoMainFile - } = message - const currentHash = crypto - .createHash('sha256') - .update( - JSON.stringify({ - pythonWSGI, - pythonModule, - pythonVariable, - deploymentType, - denoOptions, - baseImage, - baseBuildImage, - buildPack, - port, - exposePort, - installCommand, - buildCommand, - startCommand, - secrets, - branch, - repository, - fqdn - }) - ) - .digest('hex'); - try { - const { debug } = settings; - if (concurrency === 1) { - await prisma.build.updateMany({ - where: { - status: { in: ['queued', 'running'] }, - id: { not: buildId }, - applicationId, - createdAt: { lt: new Date(new Date().getTime() - 10 * 1000) } - }, - data: { status: 'failed' } - }); - } - let imageId = applicationId; - let domain = getDomain(fqdn); - const volumes = - persistentStorage?.map((storage) => { - return `${applicationId}${storage.path.replace(/\//gi, '-')}:${buildPack !== 'docker' ? '/app' : '' - }${storage.path}`; - }) || []; - // Previews, we need to get the source branch and set subdomain - if (pullmergeRequestId) { - branch = sourceBranch; - domain = `${pullmergeRequestId}.${domain}`; - imageId = `${applicationId}-${pullmergeRequestId}`; - } - - let deployNeeded = true; - let destinationType; - - if (destinationDockerId) { - destinationType = 'docker'; - } - if (destinationType === 'docker') { - await prisma.build.update({ where: { id: buildId }, data: { status: 'running' } }); - const { workdir, repodir } = await createDirectories({ repository, buildId }); - const configuration = await setDefaultConfiguration(message); - - buildPack = configuration.buildPack; - port = configuration.port; - installCommand = configuration.installCommand; - startCommand = configuration.startCommand; - buildCommand = configuration.buildCommand; - publishDirectory = configuration.publishDirectory; - baseDirectory = configuration.baseDirectory; - dockerFileLocation = configuration.dockerFileLocation; - denoMainFile = configuration.denoMainFile; - const commit = await importers[gitSource.type]({ - applicationId, - debug, - workdir, - repodir, - githubAppId: gitSource.githubApp?.id, - gitlabAppId: gitSource.gitlabApp?.id, - customPort: gitSource.customPort, - repository, - branch, - buildId, - apiUrl: gitSource.apiUrl, - htmlUrl: gitSource.htmlUrl, - projectId, - deployKeyId: gitSource.gitlabApp?.deployKeyId || null, - privateSshKey: decrypt(gitSource.gitlabApp?.privateSshKey) || null, - forPublic: gitSource.forPublic - }); - if (!commit) { - throw new Error('No commit found?'); - } - let tag = commit.slice(0, 7); - if (pullmergeRequestId) { - tag = `${commit.slice(0, 7)}-${pullmergeRequestId}`; - } - - try { - await prisma.build.update({ where: { id: buildId }, data: { commit } }); - } catch (err) { - console.log(err); - } - - if (!pullmergeRequestId) { - - if (configHash !== currentHash) { - deployNeeded = true; - if (configHash) { - await saveBuildLog({ line: 'Configuration changed.', buildId, applicationId }); - } - } else { - deployNeeded = false; - } - } else { - deployNeeded = true; - } - - let imageFound = false; - try { - await executeDockerCmd({ - dockerId: destinationDocker.id, - command: `docker image inspect ${applicationId}:${tag}` - }) - imageFound = true; - } catch (error) { - // - } - await copyBaseConfigurationFiles(buildPack, workdir, buildId, applicationId, baseImage); - - if (forceRebuild) deployNeeded = true - if (!imageFound || deployNeeded) { - // if (true) { - if (buildpacks[buildPack]) - await buildpacks[buildPack]({ - dockerId: destinationDocker.id, - buildId, - applicationId, - domain, - name, - type, - pullmergeRequestId, - buildPack, - repository, - branch, - projectId, - publishDirectory, - debug, - commit, - tag, - workdir, - port: exposePort ? `${exposePort}:${port}` : port, - installCommand, - buildCommand, - startCommand, - baseDirectory, - secrets, - phpModules, - pythonWSGI, - pythonModule, - pythonVariable, - dockerFileLocation, - denoMainFile, - denoOptions, - baseImage, - baseBuildImage, - deploymentType - }); - else { - await saveBuildLog({ line: `Build pack ${buildPack} not found`, buildId, applicationId }); - throw new Error(`Build pack ${buildPack} not found.`); - } - } else { - await saveBuildLog({ line: 'Build image already available - no rebuild required.', buildId, applicationId }); - } - try { - await executeDockerCmd({ dockerId: destinationDocker.id, command: `docker stop -t 0 ${imageId}` }) - await executeDockerCmd({ dockerId: destinationDocker.id, command: `docker rm ${imageId}` }) - } catch (error) { - // - } - const envs = [ - `PORT=${port}` - ]; - if (secrets.length > 0) { - secrets.forEach((secret) => { - if (pullmergeRequestId) { - if (secret.isPRMRSecret) { - envs.push(`${secret.name}=${secret.value}`); - } - } else { - if (!secret.isPRMRSecret) { - envs.push(`${secret.name}=${secret.value}`); - } - } - }); - } - await fs.writeFile(`${workdir}/.env`, envs.join('\n')); - const labels = makeLabelForStandaloneApplication({ - applicationId, - fqdn, - name, - type, - pullmergeRequestId, - buildPack, - repository, - branch, - projectId, - port: exposePort ? `${exposePort}:${port}` : port, - commit, - installCommand, - buildCommand, - startCommand, - baseDirectory, - publishDirectory - }); - let envFound = false; - try { - envFound = !!(await fs.stat(`${workdir}/.env`)); - } catch (error) { - // - } - try { - await saveBuildLog({ line: 'Deployment started.', buildId, applicationId }); - const composeVolumes = volumes.map((volume) => { - return { - [`${volume.split(':')[0]}`]: { - name: volume.split(':')[0] - } - }; - }); - const composeFile = { - version: '3.8', - services: { - [imageId]: { - image: `${applicationId}:${tag}`, - container_name: imageId, - volumes, - env_file: envFound ? [`${workdir}/.env`] : [], - labels, - depends_on: [], - expose: [port], - ...(exposePort ? { ports: [`${exposePort}:${port}`] } : {}), - // logging: { - // driver: 'fluentd', - // }, - ...defaultComposeConfiguration(destinationDocker.network), - } - }, - networks: { - [destinationDocker.network]: { - external: true - } - }, - volumes: Object.assign({}, ...composeVolumes) - }; - await fs.writeFile(`${workdir}/docker-compose.yml`, yaml.dump(composeFile)); - await executeDockerCmd({ dockerId: destinationDocker.id, command: `docker compose --project-directory ${workdir} up -d` }) - await saveBuildLog({ line: 'Deployment successful!', buildId, applicationId }); - } catch (error) { - await saveBuildLog({ line: error, buildId, applicationId }); - await prisma.build.updateMany({ - where: { id: message.build_id, status: { in: ['queued', 'running'] } }, - data: { status: 'failed' } - }); - throw new Error(error); - } - await saveBuildLog({ line: 'Proxy will be updated shortly.', buildId, applicationId }); - await prisma.build.update({ where: { id: message.build_id }, data: { status: 'success' } }); - if (!pullmergeRequestId) await prisma.application.update({ - where: { id: applicationId }, - data: { configHash: currentHash } - }); - } - - } - catch (error) { - await prisma.build.updateMany({ - where: { id: message.build_id, status: { in: ['queued', 'running'] } }, - data: { status: 'failed' } - }); - await saveBuildLog({ line: error, buildId, applicationId }); - } finally { - await prisma.$disconnect(); - } - }); - await prisma.$disconnect(); - } - }); - } else process.exit(0); -})(); diff --git a/apps/api/src/jobs/deployApplication.ts b/apps/api/src/jobs/deployApplication.ts index d5d4a6d2f..ca5a5e230 100644 --- a/apps/api/src/jobs/deployApplication.ts +++ b/apps/api/src/jobs/deployApplication.ts @@ -14,17 +14,19 @@ import * as buildpacks from '../lib/buildPacks'; if (message === 'error') throw new Error('oops'); if (message === 'cancel') { parentPort.postMessage('cancelled'); + await prisma.$disconnect() process.exit(0); } }); - try { - const pThrottle = await import('p-throttle') - const throttle = pThrottle.default({ - limit: 1, - interval: 2000 - }); + const pThrottle = await import('p-throttle') + const throttle = pThrottle.default({ + limit: 1, + interval: 2000 + }); - const th = throttle(async () => { + + const th = throttle(async () => { + try { const queuedBuilds = await prisma.build.findMany({ where: { status: 'queued' }, orderBy: { createdAt: 'asc' } }); const { concurrentBuilds } = await prisma.setting.findFirst({}) if (queuedBuilds.length > 0) { @@ -356,14 +358,13 @@ import * as buildpacks from '../lib/buildPacks'; } await pAll.default(actions, { concurrency }) } - }) - while (true) { - await th() + } catch (error) { + } finally { } + }) - } catch (error) { - } finally { - await prisma.$disconnect() + while (true) { + await th() } diff --git a/apps/api/src/jobs/infrastructure.ts b/apps/api/src/jobs/infrastructure.ts new file mode 100644 index 000000000..d19163686 --- /dev/null +++ b/apps/api/src/jobs/infrastructure.ts @@ -0,0 +1,216 @@ +import { parentPort } from 'node:worker_threads'; +import axios from 'axios'; +import compareVersions from 'compare-versions'; +import { asyncExecShell, cleanupDockerStorage, executeDockerCmd, isDev, prisma, startTraefikTCPProxy, generateDatabaseConfiguration, startTraefikProxy, listSettings, version } from '../lib/common'; + +async function disconnect() { + await prisma.$disconnect(); +} +async function autoUpdater() { + try { + const currentVersion = version; + const { data: versions } = await axios + .get( + `https://get.coollabs.io/versions.json` + , { + params: { + appId: process.env['COOLIFY_APP_ID'] || undefined, + version: currentVersion + } + }) + const latestVersion = versions['coolify'].main.version; + const isUpdateAvailable = compareVersions(latestVersion, currentVersion); + if (isUpdateAvailable === 1) { + const activeCount = 0 + if (activeCount === 0) { + if (!isDev) { + console.log(`Updating Coolify to ${latestVersion}.`); + await asyncExecShell(`docker pull coollabsio/coolify:${latestVersion}`); + await asyncExecShell(`env | grep COOLIFY > .env`); + await asyncExecShell( + `docker run --rm -tid --env-file .env -v /var/run/docker.sock:/var/run/docker.sock -v coolify-db coollabsio/coolify:${latestVersion} /bin/sh -c "env | grep COOLIFY > .env && echo 'TAG=${latestVersion}' >> .env && docker stop -t 0 coolify && docker rm coolify && docker compose up -d --force-recreate"` + ); + } else { + console.log('Updating (not really in dev mode).'); + } + } + } + } catch (error) { + console.log(error); + } +} +async function checkProxies() { + try { + const { default: isReachable } = await import('is-port-reachable'); + let portReachable; + + const { arch, ipv4, ipv6 } = await listSettings(); + // Coolify Proxy local + const engine = '/var/run/docker.sock'; + const localDocker = await prisma.destinationDocker.findFirst({ + where: { engine, network: 'coolify' } + }); + if (localDocker && localDocker.isCoolifyProxyUsed) { + portReachable = await isReachable(80, { host: ipv4 || ipv6 }) + if (!portReachable) { + await startTraefikProxy(localDocker.id); + } + } + + // TCP Proxies + const databasesWithPublicPort = await prisma.database.findMany({ + where: { publicPort: { not: null } }, + include: { settings: true, destinationDocker: true } + }); + for (const database of databasesWithPublicPort) { + const { destinationDockerId, destinationDocker, publicPort, id } = database; + if (destinationDockerId && destinationDocker.isCoolifyProxyUsed) { + const { privatePort } = generateDatabaseConfiguration(database, arch); + portReachable = await isReachable(publicPort, { host: destinationDocker.remoteIpAddress || ipv4 || ipv6 }) + if (!portReachable) { + await startTraefikTCPProxy(destinationDocker, id, publicPort, privatePort); + } + } + } + const wordpressWithFtp = await prisma.wordpress.findMany({ + where: { ftpPublicPort: { not: null } }, + include: { service: { include: { destinationDocker: true } } } + }); + for (const ftp of wordpressWithFtp) { + const { service, ftpPublicPort } = ftp; + const { destinationDockerId, destinationDocker, id } = service; + if (destinationDockerId && destinationDocker.isCoolifyProxyUsed) { + portReachable = await isReachable(ftpPublicPort, { host: destinationDocker.remoteIpAddress || ipv4 || ipv6 }) + if (!portReachable) { + await startTraefikTCPProxy(destinationDocker, id, ftpPublicPort, 22, 'wordpressftp'); + } + } + } + + // HTTP Proxies + const minioInstances = await prisma.minio.findMany({ + where: { publicPort: { not: null } }, + include: { service: { include: { destinationDocker: true } } } + }); + for (const minio of minioInstances) { + const { service, publicPort } = minio; + const { destinationDockerId, destinationDocker, id } = service; + if (destinationDockerId && destinationDocker.isCoolifyProxyUsed) { + portReachable = await isReachable(publicPort, { host: destinationDocker.remoteIpAddress || ipv4 || ipv6 }) + if (!portReachable) { + await startTraefikTCPProxy(destinationDocker, id, publicPort, 9000); + } + } + } + } catch (error) { + + } +} +async function cleanupPrismaEngines() { + if (!isDev) { + try { + const { stdout } = await asyncExecShell(`ps -ef | grep /app/prisma-engines/query-engine | grep -v grep | wc -l | xargs`) + if (stdout.trim() != null && stdout.trim() != '' && Number(stdout.trim()) > 1) { + await asyncExecShell(`killall -q -e /app/prisma-engines/query-engine -o 1m`) + } + } catch (error) { + console.log(error); + } + } +} +async function cleanupStorage() { + const destinationDockers = await prisma.destinationDocker.findMany(); + let enginesDone = new Set() + for (const destination of destinationDockers) { + if (enginesDone.has(destination.engine) || enginesDone.has(destination.remoteIpAddress)) return + if (destination.engine) enginesDone.add(destination.engine) + if (destination.remoteIpAddress) enginesDone.add(destination.remoteIpAddress) + + let lowDiskSpace = false; + try { + let stdout = null + if (!isDev) { + const output = await executeDockerCmd({ dockerId: destination.id, command: `CONTAINER=$(docker ps -lq | head -1) && docker exec $CONTAINER sh -c 'df -kPT /'` }) + stdout = output.stdout; + } else { + const output = await asyncExecShell( + `df -kPT /` + ); + stdout = output.stdout; + } + let lines = stdout.trim().split('\n'); + let header = lines[0]; + let regex = + /^Filesystem\s+|Type\s+|1024-blocks|\s+Used|\s+Available|\s+Capacity|\s+Mounted on\s*$/g; + const boundaries = []; + let match; + + while ((match = regex.exec(header))) { + boundaries.push(match[0].length); + } + + boundaries[boundaries.length - 1] = -1; + const data = lines.slice(1).map((line) => { + const cl = boundaries.map((boundary) => { + const column = boundary > 0 ? line.slice(0, boundary) : line; + line = line.slice(boundary); + return column.trim(); + }); + return { + capacity: Number.parseInt(cl[5], 10) / 100 + }; + }); + if (data.length > 0) { + const { capacity } = data[0]; + if (capacity > 0.8) { + lowDiskSpace = true; + } + } + } catch (error) { + console.log(error); + } + await cleanupDockerStorage(destination.id, lowDiskSpace, false) + } +} + +(async () => { + let status = { + cleanupStorage: false, + autoUpdater: false + } + if (parentPort) { + parentPort.on('message', async (message) => { + if (parentPort) { + if (message === 'error') throw new Error('oops'); + if (message === 'cancel') { + parentPort.postMessage('cancelled'); + process.exit(1); + } + if (message === 'action:cleanupStorage') { + if (!status.autoUpdater) { + status.cleanupStorage = true + await cleanupStorage(); + status.cleanupStorage = false + } + return; + } + if (message === 'action:cleanupPrismaEngines') { + await cleanupPrismaEngines(); + return; + } + if (message === 'action:checkProxies') { + await checkProxies(); + return; + } + if (message === 'action:autoUpdater') { + if (!status.cleanupStorage) { + status.autoUpdater = true + await autoUpdater(); + status.autoUpdater = false + } + return; + } + } + }); + } else process.exit(0); +})(); diff --git a/apps/api/src/lib/common.ts b/apps/api/src/lib/common.ts index cceca5ba7..148bacfff 100644 --- a/apps/api/src/lib/common.ts +++ b/apps/api/src/lib/common.ts @@ -135,30 +135,31 @@ export const asyncSleep = (delay: number): Promise => new Promise((resolve) => setTimeout(resolve, delay)); export const prisma = new PrismaClient({ errorFormat: 'minimal', - log: [ - { - emit: 'event', - level: 'query', - }, - { - emit: 'stdout', - level: 'error', - }, - { - emit: 'stdout', - level: 'info', - }, - { - emit: 'stdout', - level: 'warn', - }, - ], + // log: [ + // { + // emit: 'event', + // level: 'query', + // }, + // { + // emit: 'stdout', + // level: 'error', + // }, + // { + // emit: 'stdout', + // level: 'info', + // }, + // { + // emit: 'stdout', + // level: 'warn', + // }, + // ], }); // prisma.$on('query', (e) => { -// console.log('Query: ' + e.query) -// console.log('Params: ' + e.params) -// console.log('Duration: ' + e.duration + 'ms') + // console.log({e}) + // console.log('Query: ' + e.query) + // console.log('Params: ' + e.params) + // console.log('Duration: ' + e.duration + 'ms') // }) export const base64Encode = (text: string): string => { return Buffer.from(text).toString('base64'); diff --git a/apps/api/src/lib/scheduler.ts b/apps/api/src/lib/scheduler.ts index 6c233d050..ebff53e12 100644 --- a/apps/api/src/lib/scheduler.ts +++ b/apps/api/src/lib/scheduler.ts @@ -9,7 +9,8 @@ Bree.extend(TSBree); const options: any = { defaultExtension: 'js', - logger: new Cabin(), + // logger: new Cabin(), + logger: false, workerMessageHandler: async ({ name, message }) => { if (name === 'deployApplication' && message?.deploying) { if (scheduler.workers.has('autoUpdater') || scheduler.workers.has('cleanupStorage')) { @@ -18,28 +19,12 @@ const options: any = { } }, jobs: [ - { - name: 'deployApplication', - }, - { - name: 'cleanupStorage', - }, - { - name: 'cleanupPrismaEngines', - interval: '1m' - }, - { - name: 'checkProxies', - interval: '10s' - }, - { - name: 'autoUpdater', - } + { name: 'infrastructure' }, + { name: 'deployApplication' }, ], }; if (isDev) options.root = path.join(__dirname, '../jobs'); - export const scheduler = new Bree(options); diff --git a/apps/ui/src/routes/applications/[id]/logs/build.svelte b/apps/ui/src/routes/applications/[id]/logs/build.svelte index 56faaf7d5..d1373e71e 100644 --- a/apps/ui/src/routes/applications/[id]/logs/build.svelte +++ b/apps/ui/src/routes/applications/[id]/logs/build.svelte @@ -158,7 +158,6 @@ {build.type} -
{#if build.status === 'running'} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b440d0264..34c431e77 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -22,6 +22,7 @@ importers: '@fastify/jwt': 6.3.2 '@fastify/static': 6.5.0 '@iarna/toml': 2.2.5 + '@ladjs/graceful': 3.0.2 '@prisma/client': 3.15.2 '@types/node': 18.7.13 '@types/node-os-utils': 1.3.0 @@ -52,7 +53,8 @@ importers: node-forge: 1.3.1 node-os-utils: 1.3.7 nodemon: 2.0.19 - p-queue: 7.3.0 + p-all: 4.0.0 + p-throttle: 5.0.0 prettier: 2.7.1 prisma: 3.15.2 public-ip: 6.0.1 @@ -63,7 +65,7 @@ importers: typescript: 4.7.4 unique-names-generator: 4.7.1 dependencies: - '@breejs/ts-worker': 2.0.0_yjs2yukaec33oijlee4f5n7fqa + '@breejs/ts-worker': 2.0.0_rzqxabipis2a5sxrpk4obdh4zu '@fastify/autoload': 5.2.0 '@fastify/cookie': 8.0.0 '@fastify/cors': 8.1.0 @@ -71,6 +73,7 @@ importers: '@fastify/jwt': 6.3.2 '@fastify/static': 6.5.0 '@iarna/toml': 2.2.5 + '@ladjs/graceful': 3.0.2 '@prisma/client': 3.15.2_prisma@3.15.2 axios: 0.27.2 bcryptjs: 2.4.3 @@ -214,12 +217,11 @@ packages: engines: {node: '>= 10'} dev: false - /@breejs/ts-worker/2.0.0_yjs2yukaec33oijlee4f5n7fqa: + /@breejs/ts-worker/2.0.0_rzqxabipis2a5sxrpk4obdh4zu: resolution: {integrity: sha512-6anHRcmgYlF7mrm/YVRn6rx2cegLuiY3VBxkkimOTWC/dVQeH336imVSuIKEGKTwiuNTPr2hswVdDSneNuXg3A==} engines: {node: '>= 12.11'} peerDependencies: bree: '>=9.0.0' - tsconfig-paths: '>= 4' dependencies: bree: 9.1.2 ts-node: 10.8.2_57uwcby55h6tzvkj3v5sfcgxoe @@ -378,6 +380,14 @@ packages: resolution: {integrity: sha512-hZere0rUga8kTzSTFbHREXpD9E/jwi94+B5RyLAmMIzl/w/EK1z7rFEnMHzPkU4AZkL42JWSsGXoV8LXMihybg==} dev: false + /@ladjs/graceful/3.0.2: + resolution: {integrity: sha512-T4Z+0R0zgZfR32KIs3FEuH7oFSnhj3c+00wVtp07aeIl8PDfQGcXGB3C8SfOZ2EzPMRuIpIul5kHkVBdSTULXw==} + engines: {node: '>=14'} + dependencies: + lil-http-terminator: 1.2.2 + p-is-promise: 3.0.0 + dev: false + /@leichtgewicht/ip-codec/2.0.4: resolution: {integrity: sha512-Hcv+nVC0kZnQ3tD9GVu5xSMR4VVYOteQIr/hwFPVEvPdlXqgGEuRjiheChHgdM+JyqdgNcmzZOX/tnl0JOiI7A==} dev: false @@ -405,8 +415,8 @@ packages: '@nodelib/fs.scandir': 2.1.5 fastq: 1.13.0 - /@playwright/test/1.24.2: - resolution: {integrity: sha512-Q4X224pRHw4Dtkk5PoNJplZCokLNvVbXD9wDQEMrHcEuvWpJWEQDeJ9gEwkZ3iCWSFSWBshIX177B231XW4wOQ==} + /@playwright/test/1.25.1: + resolution: {integrity: sha512-IJ4X0yOakXtwkhbnNzKkaIgXe6df7u3H3FnuhI9Jqh+CdO0e/lYQlDLYiyI9cnXK8E7UAppAWP+VqAv6VX7HQg==} engines: {node: '>=14'} hasBin: true dependencies: @@ -2158,7 +2168,7 @@ packages: css-selector-tokenizer: 0.8.0 postcss: 8.4.16 postcss-js: 4.0.0_postcss@8.4.16 - tailwindcss: 3.1.8_postcss@8.4.16 + tailwindcss: 3.1.8 transitivePeerDependencies: - ts-node dev: false @@ -4031,6 +4041,11 @@ packages: set-cookie-parser: 2.4.8 dev: false + /lil-http-terminator/1.2.2: + resolution: {integrity: sha512-2n6gKJIKgPjy4JfSlwsQnAA7wK4SEA1cegqdsYwr7qObVfIHdELjDGjEcYWJanfF/u/mRzIT2WPqhpzC6R9pZw==} + engines: {node: '>=8'} + dev: false + /lilconfig/2.0.6: resolution: {integrity: sha512-9JROoBW7pobfsx+Sq2JsASvCo6Pfo6WWoUW79HuB1BCoBXD4PLWJPqDF6fNj67pqBYTbAHkE57M1kS/+L1neOg==} engines: {node: '>=10'} @@ -4575,6 +4590,11 @@ packages: engines: {node: '>=4'} dev: false + /p-is-promise/3.0.0: + resolution: {integrity: sha512-Wo8VsW4IRQSKVXsJCn7TomUaVtyfjVDn3nUP7kE967BQk0CwFpdbZs0X0uk5sW9mkBa9eNM7hCMaG93WUAwxYQ==} + engines: {node: '>=8'} + dev: false + /p-limit/2.3.0: resolution: {integrity: sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==} engines: {node: '>=6'} @@ -4616,8 +4636,8 @@ packages: p-limit: 3.1.0 dev: true - /p-queue/7.3.0: - resolution: {integrity: sha512-5fP+yVQ0qp0rEfZoDTlP2c3RYBgxvRsw30qO+VtPPc95lyvSG+x6USSh1TuLB4n96IO6I8/oXQGsTgtna4q2nQ==} + /p-map/5.5.0: + resolution: {integrity: sha512-VFqfGDHlx87K66yZrNdI4YGtD70IRyd+zSvgks6mzHPRNkoKy+9EKP4SFC77/vTTQYmRmti7dvqC+m5jBrBAcg==} engines: {node: '>=12'} dependencies: aggregate-error: 4.0.1 @@ -5724,15 +5744,13 @@ packages: peerDependencies: tailwindcss: '>= 2.x.x' dependencies: - tailwindcss: 3.1.8_postcss@8.4.16 + tailwindcss: 3.1.8 dev: true - /tailwindcss/3.1.8_postcss@8.4.16: + /tailwindcss/3.1.8: resolution: {integrity: sha512-YSneUCZSFDYMwk+TGq8qYFdCA3yfBRdBlS7txSq0LUmzyeqRe3a8fBQzbz9M3WS/iFT4BNf/nmw9mEzrnSaC0g==} engines: {node: '>=12.13.0'} hasBin: true - peerDependencies: - postcss: ^8.0.9 dependencies: arg: 5.0.2 chokidar: 3.5.3