From d5f2d2266331bc5d04af5d6ad9f273c8f1a3d5a4 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 13 Jul 2022 13:40:41 +0000 Subject: [PATCH] fix: Cleanup less often and can do it manually --- apps/api/src/jobs/cleanupStorage.ts | 116 +++++++----------- apps/api/src/lib/common.ts | 46 +++++++ apps/api/src/routes/api/v1/handlers.ts | 11 +- apps/api/src/routes/api/v1/iam/handlers.ts | 2 +- apps/api/src/routes/api/v1/index.ts | 6 +- .../src/routes/api/v1/services/handlers.ts | 66 +++++++++- apps/ui/src/lib/components/Usage.svelte | 9 +- 7 files changed, 177 insertions(+), 79 deletions(-) diff --git a/apps/api/src/jobs/cleanupStorage.ts b/apps/api/src/jobs/cleanupStorage.ts index 5cb04bdab..dd8636175 100644 --- a/apps/api/src/jobs/cleanupStorage.ts +++ b/apps/api/src/jobs/cleanupStorage.ts @@ -1,5 +1,5 @@ import { parentPort } from 'node:worker_threads'; -import { asyncExecShell, isDev, prisma, version } from '../lib/common'; +import { asyncExecShell, cleanupDockerStorage, isDev, prisma, version } from '../lib/common'; import { getEngine } from '../lib/docker'; (async () => { @@ -9,82 +9,52 @@ import { getEngine } from '../lib/docker'; for (const engine of engines) { let lowDiskSpace = false; const host = getEngine(engine); - // try { - // let stdout = null - // if (!isDev) { - // const output = await asyncExecShell( - // `DOCKER_HOST=${host} docker exec coolify sh -c 'df -kPT /'` - // ); - // stdout = output.stdout; - // } else { - // const output = await asyncExecShell( - // `df -kPT /` - // ); - // stdout = output.stdout; - // } - // let lines = stdout.trim().split('\n'); - // let header = lines[0]; - // let regex = - // /^Filesystem\s+|Type\s+|1024-blocks|\s+Used|\s+Available|\s+Capacity|\s+Mounted on\s*$/g; - // const boundaries = []; - // let match; - - // while ((match = regex.exec(header))) { - // boundaries.push(match[0].length); - // } - - // boundaries[boundaries.length - 1] = -1; - // const data = lines.slice(1).map((line) => { - // const cl = boundaries.map((boundary) => { - // const column = boundary > 0 ? line.slice(0, boundary) : line; - // line = line.slice(boundary); - // return column.trim(); - // }); - // return { - // capacity: Number.parseInt(cl[5], 10) / 100 - // }; - // }); - // if (data.length > 0) { - // const { capacity } = data[0]; - // if (capacity > 0.6) { - // lowDiskSpace = true; - // } - // } - // } catch (error) { - // console.log(error); - // } - if (!isDev) { - // Cleanup old coolify images - try { - let { stdout: images } = await asyncExecShell( - `DOCKER_HOST=${host} docker images coollabsio/coolify --filter before="coollabsio/coolify:${version}" -q | xargs ` + try { + let stdout = null + if (!isDev) { + const output = await asyncExecShell( + `DOCKER_HOST=${host} docker exec coolify sh -c 'df -kPT /'` ); - images = images.trim(); - if (images) { - await asyncExecShell(`DOCKER_HOST=${host} docker rmi -f ${images}`); + stdout = output.stdout; + } else { + const output = await asyncExecShell( + `df -kPT /` + ); + stdout = output.stdout; + } + let lines = stdout.trim().split('\n'); + let header = lines[0]; + let regex = + /^Filesystem\s+|Type\s+|1024-blocks|\s+Used|\s+Available|\s+Capacity|\s+Mounted on\s*$/g; + const boundaries = []; + let match; + + while ((match = regex.exec(header))) { + boundaries.push(match[0].length); + } + + boundaries[boundaries.length - 1] = -1; + const data = lines.slice(1).map((line) => { + const cl = boundaries.map((boundary) => { + const column = boundary > 0 ? line.slice(0, boundary) : line; + line = line.slice(boundary); + return column.trim(); + }); + return { + capacity: Number.parseInt(cl[5], 10) / 100 + }; + }); + if (data.length > 0) { + const { capacity } = data[0]; + if (capacity > 0.8) { + lowDiskSpace = true; } - } catch (error) { - //console.log(error); } - try { - await asyncExecShell(`DOCKER_HOST=${host} docker container prune -f`); - } catch (error) { - //console.log(error); - } - try { - await asyncExecShell(`DOCKER_HOST=${host} docker image prune -f`); - } catch (error) { - //console.log(error); - } - try { - await asyncExecShell(`DOCKER_HOST=${host} docker image prune -a -f`); - } catch (error) { - //console.log(error); - } - } else { - console.log(`[DEV MODE] Low disk space: ${lowDiskSpace}`); + } catch (error) { + console.log(error); } + await cleanupDockerStorage(host, lowDiskSpace, false) } - await prisma.$disconnect(); + await prisma.$disconnect(); } else process.exit(0); })(); diff --git a/apps/api/src/lib/common.ts b/apps/api/src/lib/common.ts index 4f2f1ebcb..cb14fd89f 100644 --- a/apps/api/src/lib/common.ts +++ b/apps/api/src/lib/common.ts @@ -14,6 +14,7 @@ import cuid from 'cuid'; import { checkContainer, getEngine, removeContainer } from './docker'; import { day } from './dayjs'; import * as serviceFields from './serviceFields' +import axios from 'axios'; export const version = '3.1.1'; export const isDev = process.env.NODE_ENV === 'development'; @@ -1494,4 +1495,49 @@ async function cleanupDB(buildId: string) { if (data?.status === 'queued' || data?.status === 'running') { await prisma.build.update({ where: { id: buildId }, data: { status: 'failed' } }); } +} + +export function convertTolOldVolumeNames(type) { + if (type === 'nocodb') { + return 'nc' + } +} +export async function getAvailableServices(): Promise { + const { data } = await axios.get(`https://gist.githubusercontent.com/andrasbacsai/4aac36d8d6214dbfc34fa78110554a50/raw/291a957ee6ac01d480465623e183a30230ad921f/availableServices.json`) + return data +} +export async function cleanupDockerStorage(host, lowDiskSpace, force) { + // Cleanup old coolify images + try { + let { stdout: images } = await asyncExecShell( + `DOCKER_HOST=${host} docker images coollabsio/coolify --filter before="coollabsio/coolify:${version}" -q | xargs ` + ); + images = images.trim(); + if (images) { + await asyncExecShell(`DOCKER_HOST=${host} docker rmi -f ${images}`); + } + } catch (error) { + //console.log(error); + } + if (lowDiskSpace || force) { + if (isDev) { + if (!force) console.log(`[DEV MODE] Low disk space: ${lowDiskSpace}`); + return + } + try { + await asyncExecShell(`DOCKER_HOST=${host} docker container prune -f`); + } catch (error) { + //console.log(error); + } + try { + await asyncExecShell(`DOCKER_HOST=${host} docker image prune -f`); + } catch (error) { + //console.log(error); + } + try { + await asyncExecShell(`DOCKER_HOST=${host} docker image prune -a -f`); + } catch (error) { + //console.log(error); + } + } } \ No newline at end of file diff --git a/apps/api/src/routes/api/v1/handlers.ts b/apps/api/src/routes/api/v1/handlers.ts index 7ba24c461..95988f1f1 100644 --- a/apps/api/src/routes/api/v1/handlers.ts +++ b/apps/api/src/routes/api/v1/handlers.ts @@ -4,7 +4,7 @@ import axios from 'axios'; import compare from 'compare-versions'; import cuid from 'cuid'; import bcrypt from 'bcryptjs'; -import { asyncExecShell, asyncSleep, errorHandler, isDev, prisma, uniqueName, version } from '../../../lib/common'; +import { asyncExecShell, asyncSleep, cleanupDockerStorage, errorHandler, isDev, prisma, uniqueName, version } from '../../../lib/common'; import type { FastifyReply, FastifyRequest } from 'fastify'; import type { Login, Update } from '.'; @@ -15,7 +15,14 @@ export async function hashPassword(password: string): Promise { return bcrypt.hash(password, saltRounds); } - +export async function cleanupManually() { + try { + await cleanupDockerStorage('unix:///var/run/docker.sock', true, true) + return {} + } catch ({ status, message }) { + return errorHandler({ status, message }) + } +} export async function checkUpdate(request: FastifyRequest) { try { const currentVersion = version; diff --git a/apps/api/src/routes/api/v1/iam/handlers.ts b/apps/api/src/routes/api/v1/iam/handlers.ts index 12095c26d..0e3668f64 100644 --- a/apps/api/src/routes/api/v1/iam/handlers.ts +++ b/apps/api/src/routes/api/v1/iam/handlers.ts @@ -445,7 +445,7 @@ export async function setPermission(request: FastifyRequest, reply: FastifyReply export async function changePassword(request: FastifyRequest, reply: FastifyReply) { try { const { id } = request.body; - await prisma.user.update({ where: { id: undefined }, data: { password: 'RESETME' } }); + await prisma.user.update({ where: { id }, data: { password: 'RESETME' } }); return reply.code(201).send() } catch ({ status, message }) { return errorHandler({ status, message }) diff --git a/apps/api/src/routes/api/v1/index.ts b/apps/api/src/routes/api/v1/index.ts index c8438cffe..ef81b5792 100644 --- a/apps/api/src/routes/api/v1/index.ts +++ b/apps/api/src/routes/api/v1/index.ts @@ -1,6 +1,6 @@ import { FastifyPluginAsync } from 'fastify'; import { scheduler } from '../../../lib/scheduler'; -import { checkUpdate, login, showDashboard, update, showUsage, getCurrentUser } from './handlers'; +import { checkUpdate, login, showDashboard, update, showUsage, getCurrentUser, cleanupManually } from './handlers'; export interface Update { Body: { latestVersion: string } @@ -46,6 +46,10 @@ const root: FastifyPluginAsync = async (fastify, opts): Promise => { fastify.get('/usage', { onRequest: [fastify.authenticate] }, async () => await showUsage()); + + fastify.post('/internal/cleanup', { + onRequest: [fastify.authenticate] + }, async () => await cleanupManually()); }; export default root; diff --git a/apps/api/src/routes/api/v1/services/handlers.ts b/apps/api/src/routes/api/v1/services/handlers.ts index 02943b18f..e2fa3404a 100644 --- a/apps/api/src/routes/api/v1/services/handlers.ts +++ b/apps/api/src/routes/api/v1/services/handlers.ts @@ -2,11 +2,75 @@ import type { FastifyReply, FastifyRequest } from 'fastify'; import fs from 'fs/promises'; import yaml from 'js-yaml'; import bcrypt from 'bcryptjs'; -import { prisma, uniqueName, asyncExecShell, getServiceImage, getServiceImages, configureServiceType, getServiceFromDB, getContainerUsage, removeService, isDomainConfigured, saveUpdateableFields, fixType, decrypt, encrypt, getServiceMainPort, createDirectories, ComposeFile, makeLabelForServices, getFreePort, getDomain, errorHandler, supportedServiceTypesAndVersions, generatePassword, isDev, stopTcpHttpProxy } from '../../../../lib/common'; +import { prisma, uniqueName, asyncExecShell, getServiceImage, getServiceImages, configureServiceType, getServiceFromDB, getContainerUsage, removeService, isDomainConfigured, saveUpdateableFields, fixType, decrypt, encrypt, getServiceMainPort, createDirectories, ComposeFile, makeLabelForServices, getFreePort, getDomain, errorHandler, supportedServiceTypesAndVersions, generatePassword, isDev, stopTcpHttpProxy, getAvailableServices } from '../../../../lib/common'; import { day } from '../../../../lib/dayjs'; import { checkContainer, dockerInstance, getEngine, removeContainer } from '../../../../lib/docker'; import cuid from 'cuid'; +async function startServiceNew(request: FastifyRequest) { + try { + const { id } = request.params; + const teamId = request.user.teamId; + const service = await getServiceFromDB({ id, teamId }); + const { type, version, destinationDockerId, destinationDocker, serviceSecret, exposePort } = + service; + const network = destinationDockerId && destinationDocker.network; + const host = getEngine(destinationDocker.engine); + const port = getServiceMainPort(type); + + const { workdir } = await createDirectories({ repository: type, buildId: id }); + const image = getServiceImage(type); + const config = (await getAvailableServices()).find((name) => name.name === type).compose + const environmentVariables = {} + if (serviceSecret.length > 0) { + serviceSecret.forEach((secret) => { + environmentVariables[secret.name] = secret.value; + }); + } + config.services[id] = JSON.parse(JSON.stringify(config.services[type])) + config.services[id].container_name = id + config.services[id].image = `${image}:${version}` + config.services[id].ports = (exposePort ? [`${exposePort}:${port}`] : []), + config.services[id].restart = "always" + config.services[id].networks = [network] + config.services[id].labels = makeLabelForServices(type) + config.services[id].deploy = { + restart_policy: { + condition: 'on-failure', + delay: '5s', + max_attempts: 3, + window: '120s' + } + } + config.networks = { + [network]: { + external: true + } + } + config.volumes = {} + config.services[id].volumes.forEach((volume, index) => { + let oldVolumeName = volume.split(':')[0] + const path = volume.split(':')[1] + oldVolumeName = convertTolOldVolumeNames(type) + const volumeName = `${id}-${oldVolumeName}` + config.volumes[volumeName] = { + name: volumeName + } + config.services[id].volumes[index] = `${volumeName}:${path}` + }) + delete config.services[type] + config.services[id].environment = environmentVariables + const composeFileDestination = `${workdir}/docker-compose.yaml`; + await fs.writeFile(composeFileDestination, yaml.dump(config)); + await asyncExecShell(`DOCKER_HOST=${host} docker compose -f ${composeFileDestination} pull`); + await asyncExecShell(`DOCKER_HOST=${host} docker compose -f ${composeFileDestination} up -d`); + return {} + } catch ({ status, message }) { + return errorHandler({ status, message }) + } +} + + export async function listServices(request: FastifyRequest) { try { const userId = request.user.userId; diff --git a/apps/ui/src/lib/components/Usage.svelte b/apps/ui/src/lib/components/Usage.svelte index cd4105c60..22dcaa4a5 100644 --- a/apps/ui/src/lib/components/Usage.svelte +++ b/apps/ui/src/lib/components/Usage.svelte @@ -23,7 +23,7 @@ }; import { appSession } from '$lib/store'; import { onDestroy, onMount } from 'svelte'; - import { get } from '$lib/api'; + import { get, post } from '$lib/api'; import { errorNotification } from '$lib/common'; import Trend from './Trend.svelte'; async function getStatus() { @@ -59,6 +59,9 @@ cpu: 'stable', disk: 'stable' }; + async function manuallyCleanupStorage() { + return await post('/internal/cleanup', {}); + } {#if $appSession.teamId === '0'} @@ -129,6 +132,9 @@
{usage?.disk.usedGb}GB
+
+
Resources
{/if}