From 907c2414ae6d96ed714bf8a811063122eaa40f93 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 13 Apr 2022 19:52:56 +0200 Subject: [PATCH 01/10] chore:version++ --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b51c1160f..41a593dcf 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "coolify", "description": "An open-source & self-hostable Heroku / Netlify alternative.", - "version": "2.4.8", + "version": "2.4.9", "license": "AGPL-3.0", "scripts": { "dev": "docker-compose -f docker-compose-dev.yaml up -d && cross-env NODE_ENV=development & svelte-kit dev", From 8419cdf604cb84c9e988ae3630b5242d9a0a8cc9 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 13 Apr 2022 19:59:30 +0200 Subject: [PATCH 02/10] fix: Postgres root pw is pw field --- src/routes/databases/[id]/_Databases/_PostgreSQL.svelte | 1 + 1 file changed, 1 insertion(+) diff --git a/src/routes/databases/[id]/_Databases/_PostgreSQL.svelte b/src/routes/databases/[id]/_Databases/_PostgreSQL.svelte index 343ad74d6..ac52233f1 100644 --- a/src/routes/databases/[id]/_Databases/_PostgreSQL.svelte +++ b/src/routes/databases/[id]/_Databases/_PostgreSQL.svelte @@ -29,6 +29,7 @@ disabled={!isRunning} readonly={!isRunning} placeholder="Generated automatically after start" + isPasswordField id="rootUserPassword" name="rootUserPassword" bind:value={database.rootUserPassword} From ce2757f514669f6220e1ddeb37d10c002630948f Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 13 Apr 2022 21:06:22 +0200 Subject: [PATCH 03/10] fix: Teams view --- src/routes/iam/index.json.ts | 15 +++++--- src/routes/iam/index.svelte | 72 ++++++++++++++++-------------------- 2 files changed, 42 insertions(+), 45 deletions(-) diff --git a/src/routes/iam/index.json.ts b/src/routes/iam/index.json.ts index fa44e55ba..6ff142eb3 100644 --- a/src/routes/iam/index.json.ts +++ b/src/routes/iam/index.json.ts @@ -13,20 +13,25 @@ export const get: RequestHandler = async (event) => { select: { id: true, email: true, teams: true } }); let accounts = []; + let allTeams = []; if (teamId === '0') { accounts = await db.prisma.user.findMany({ select: { id: true, email: true, teams: true } }); + allTeams = await db.prisma.team.findMany({ + where: { users: { none: { id: userId } } }, + include: { permissions: true } + }); } - - const teams = await db.prisma.permission.findMany({ - where: { userId: teamId === '0' ? undefined : userId }, - include: { team: { include: { _count: { select: { users: true } } } } } + const ownTeams = await db.prisma.team.findMany({ + where: { users: { some: { id: userId } } }, + include: { permissions: true } }); const invitations = await db.prisma.teamInvitation.findMany({ where: { uid: userId } }); return { status: 200, body: { - teams, + ownTeams, + allTeams, invitations, account, accounts diff --git a/src/routes/iam/index.svelte b/src/routes/iam/index.svelte index 1ebfaac60..7590385cf 100644 --- a/src/routes/iam/index.svelte +++ b/src/routes/iam/index.svelte @@ -36,18 +36,8 @@ if (accounts.length === 0) { accounts.push(account); } - export let teams; - - const ownTeams = teams.filter((team) => { - if (team.team.id === $session.teamId) { - return team; - } - }); - const otherTeams = teams.filter((team) => { - if (team.team.id !== $session.teamId) { - return team; - } - }); + export let ownTeams; + export let allTeams; async function resetPassword(id) { const sure = window.confirm('Are you sure you want to reset the password?'); @@ -167,49 +157,51 @@
Teams
From 2bd3802a6f35e28904cd6e6cd6f65214102e0b92 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 14 Apr 2022 00:04:46 +0200 Subject: [PATCH 04/10] fix: Improved tcp proxy monitoring for databases/ftp --- src/lib/database/common.ts | 27 +++++++++++++++ src/lib/queues/index.ts | 15 ++++++++ src/lib/queues/proxyTcpHttp.ts | 34 +++++++++++++++++++ .../[id]/_Databases/_PostgreSQL.svelte | 1 + src/routes/databases/[id]/delete.json.ts | 4 +-- src/routes/databases/[id]/settings.json.ts | 8 ++--- src/routes/services/[id]/minio/start.json.ts | 9 ++--- .../services/[id]/wordpress/settings.json.ts | 7 ++-- 8 files changed, 86 insertions(+), 19 deletions(-) create mode 100644 src/lib/queues/proxyTcpHttp.ts diff --git a/src/lib/database/common.ts b/src/lib/database/common.ts index 541fb8685..47a8c3f9a 100644 --- a/src/lib/database/common.ts +++ b/src/lib/database/common.ts @@ -9,6 +9,7 @@ import { default as ProdPrisma } from '@prisma/client'; import type { Database, DatabaseSettings } from '@prisma/client'; import generator from 'generate-password'; import forge from 'node-forge'; +import getPort, { portNumbers } from 'get-port'; export function generatePassword(length = 24): string { return generator.generate({ @@ -251,3 +252,29 @@ export function generateDatabaseConfiguration(database: Database & { settings: D }; } } + +export async function getFreePort() { + const data = await prisma.setting.findFirst(); + const { minPort, maxPort } = data; + + const dbUsed = await ( + await prisma.database.findMany({ + where: { publicPort: { not: null } }, + select: { publicPort: true } + }) + ).map((a) => a.publicPort); + const wpFtpUsed = await ( + await prisma.wordpress.findMany({ + where: { ftpPublicPort: { not: null } }, + select: { ftpPublicPort: true } + }) + ).map((a) => a.ftpPublicPort); + const wpUsed = await ( + await prisma.wordpress.findMany({ + where: { mysqlPublicPort: { not: null } }, + select: { mysqlPublicPort: true } + }) + ).map((a) => a.mysqlPublicPort); + const usedPorts = [...dbUsed, ...wpFtpUsed, ...wpUsed]; + return await getPort({ port: portNumbers(minPort, maxPort), exclude: usedPorts }); +} diff --git a/src/lib/queues/index.ts b/src/lib/queues/index.ts index 694ddc9cc..84d5f7cbc 100644 --- a/src/lib/queues/index.ts +++ b/src/lib/queues/index.ts @@ -7,6 +7,7 @@ import builder from './builder'; import logger from './logger'; import cleanup from './cleanup'; import proxy from './proxy'; +import proxyTcpHttp from './proxyTcpHttp'; import ssl from './ssl'; import sslrenewal from './sslrenewal'; @@ -29,17 +30,20 @@ const connectionOptions = { const cron = async (): Promise => { new QueueScheduler('proxy', connectionOptions); + new QueueScheduler('proxyTcpHttp', connectionOptions); new QueueScheduler('cleanup', connectionOptions); new QueueScheduler('ssl', connectionOptions); new QueueScheduler('sslRenew', connectionOptions); const queue = { proxy: new Queue('proxy', { ...connectionOptions }), + proxyTcpHttp: new Queue('proxyTcpHttp', { ...connectionOptions }), cleanup: new Queue('cleanup', { ...connectionOptions }), ssl: new Queue('ssl', { ...connectionOptions }), sslRenew: new Queue('sslRenew', { ...connectionOptions }) }; await queue.proxy.drain(); + await queue.proxyTcpHttp.drain(); await queue.cleanup.drain(); await queue.ssl.drain(); await queue.sslRenew.drain(); @@ -54,6 +58,16 @@ const cron = async (): Promise => { } ); + new Worker( + 'proxyTcpHttp', + async () => { + await proxyTcpHttp(); + }, + { + ...connectionOptions + } + ); + new Worker( 'ssl', async () => { @@ -85,6 +99,7 @@ const cron = async (): Promise => { ); await queue.proxy.add('proxy', {}, { repeat: { every: 10000 } }); + await queue.proxyTcpHttp.add('proxyTcpHttp', {}, { repeat: { every: 10000 } }); await queue.ssl.add('ssl', {}, { repeat: { every: dev ? 10000 : 60000 } }); if (!dev) await queue.cleanup.add('cleanup', {}, { repeat: { every: 300000 } }); await queue.sslRenew.add('sslRenew', {}, { repeat: { every: 1800000 } }); diff --git a/src/lib/queues/proxyTcpHttp.ts b/src/lib/queues/proxyTcpHttp.ts new file mode 100644 index 000000000..00b5d759e --- /dev/null +++ b/src/lib/queues/proxyTcpHttp.ts @@ -0,0 +1,34 @@ +import { ErrorHandler, generateDatabaseConfiguration, prisma } from '$lib/database'; +import { checkContainer, startTcpProxy } from '$lib/haproxy'; + +export default async function (): Promise { + try { + const databasesWithPublicPort = await prisma.database.findMany({ + where: { publicPort: { not: null } }, + include: { settings: true, destinationDocker: true } + }); + for (const database of databasesWithPublicPort) { + const { destinationDockerId, destinationDocker, publicPort, id } = database; + if (destinationDockerId) { + const { privatePort } = generateDatabaseConfiguration(database); + await startTcpProxy(destinationDocker, id, publicPort, privatePort); + } + } + const wordpressWithFtp = await prisma.wordpress.findMany({ + where: { ftpPublicPort: { not: null } }, + include: { service: { include: { destinationDocker: true } } } + }); + for (const ftp of wordpressWithFtp) { + const { service, ftpPublicPort, id } = ftp; + const { destinationDockerId, destinationDocker } = service; + if (destinationDockerId) { + await startTcpProxy(destinationDocker, `${id}-ftp`, ftpPublicPort, 22); + } + } + } catch (error) { + return ErrorHandler(error.response?.body || error); + } +} diff --git a/src/routes/databases/[id]/_Databases/_PostgreSQL.svelte b/src/routes/databases/[id]/_Databases/_PostgreSQL.svelte index ac52233f1..b5e4ba1dd 100644 --- a/src/routes/databases/[id]/_Databases/_PostgreSQL.svelte +++ b/src/routes/databases/[id]/_Databases/_PostgreSQL.svelte @@ -34,6 +34,7 @@ name="rootUserPassword" bind:value={database.rootUserPassword} /> +
diff --git a/src/routes/databases/[id]/delete.json.ts b/src/routes/databases/[id]/delete.json.ts index c81916504..d731492ad 100644 --- a/src/routes/databases/[id]/delete.json.ts +++ b/src/routes/databases/[id]/delete.json.ts @@ -1,7 +1,7 @@ import { getUserDetails } from '$lib/common'; import * as db from '$lib/database'; import { ErrorHandler, stopDatabase } from '$lib/database'; -import { deleteProxy } from '$lib/haproxy'; +import { stopTcpHttpProxy } from '$lib/haproxy'; import type { RequestHandler } from '@sveltejs/kit'; export const del: RequestHandler = async (event) => { @@ -12,7 +12,7 @@ export const del: RequestHandler = async (event) => { const database = await db.getDatabase({ id, teamId }); if (database.destinationDockerId) { const everStarted = await stopDatabase(database); - if (everStarted) await deleteProxy({ id }); + if (everStarted) await stopTcpHttpProxy(database.destinationDocker, database.publicPort); } await db.removeDatabase({ id }); return { status: 200 }; diff --git a/src/routes/databases/[id]/settings.json.ts b/src/routes/databases/[id]/settings.json.ts index 042bb36ef..1f09946c6 100644 --- a/src/routes/databases/[id]/settings.json.ts +++ b/src/routes/databases/[id]/settings.json.ts @@ -1,20 +1,16 @@ import { getUserDetails } from '$lib/common'; import * as db from '$lib/database'; -import { generateDatabaseConfiguration, ErrorHandler } from '$lib/database'; +import { generateDatabaseConfiguration, ErrorHandler, getFreePort } from '$lib/database'; import { startTcpProxy, stopTcpHttpProxy } from '$lib/haproxy'; import type { RequestHandler } from '@sveltejs/kit'; -import getPort, { portNumbers } from 'get-port'; export const post: RequestHandler = async (event) => { const { status, body, teamId } = await getUserDetails(event); if (status === 401) return { status, body }; const { id } = event.params; - const data = await db.prisma.setting.findFirst(); - const { minPort, maxPort } = data; - const { isPublic, appendOnly = true } = await event.request.json(); - const publicPort = await getPort({ port: portNumbers(minPort, maxPort) }); + const publicPort = await getFreePort(); try { await db.setDatabase({ id, isPublic, appendOnly }); diff --git a/src/routes/services/[id]/minio/start.json.ts b/src/routes/services/[id]/minio/start.json.ts index 7c3dc9e75..4fb49f07d 100644 --- a/src/routes/services/[id]/minio/start.json.ts +++ b/src/routes/services/[id]/minio/start.json.ts @@ -4,9 +4,7 @@ import { promises as fs } from 'fs'; import yaml from 'js-yaml'; import type { RequestHandler } from '@sveltejs/kit'; import { startHttpProxy } from '$lib/haproxy'; -import getPort, { portNumbers } from 'get-port'; -import { getDomain } from '$lib/components/common'; -import { ErrorHandler, getServiceImage } from '$lib/database'; +import { ErrorHandler, getFreePort, getServiceImage } from '$lib/database'; import { makeLabelForServices } from '$lib/buildPacks/common'; import type { ComposeFile } from '$lib/types/composeFile'; @@ -28,13 +26,10 @@ export const post: RequestHandler = async (event) => { serviceSecret } = service; - const data = await db.prisma.setting.findFirst(); - const { minPort, maxPort } = data; - const network = destinationDockerId && destinationDocker.network; const host = getEngine(destinationDocker.engine); - const publicPort = await getPort({ port: portNumbers(minPort, maxPort) }); + const publicPort = await getFreePort(); const consolePort = 9001; const apiPort = 9000; diff --git a/src/routes/services/[id]/wordpress/settings.json.ts b/src/routes/services/[id]/wordpress/settings.json.ts index e82b6baba..708dc18af 100644 --- a/src/routes/services/[id]/wordpress/settings.json.ts +++ b/src/routes/services/[id]/wordpress/settings.json.ts @@ -2,7 +2,7 @@ import { dev } from '$app/env'; import { asyncExecShell, getEngine, getUserDetails } from '$lib/common'; import { decrypt, encrypt } from '$lib/crypto'; import * as db from '$lib/database'; -import { generateDatabaseConfiguration, ErrorHandler, generatePassword } from '$lib/database'; +import { ErrorHandler, generatePassword, getFreePort } from '$lib/database'; import { checkContainer, startTcpProxy, stopTcpHttpProxy } from '$lib/haproxy'; import type { ComposeFile } from '$lib/types/composeFile'; import type { RequestHandler } from '@sveltejs/kit'; @@ -16,11 +16,10 @@ export const post: RequestHandler = async (event) => { if (status === 401) return { status, body }; const { id } = event.params; - const data = await db.prisma.setting.findFirst(); - const { minPort, maxPort } = data; const { ftpEnabled } = await event.request.json(); - const publicPort = await getPort({ port: portNumbers(minPort, maxPort) }); + const publicPort = await getFreePort(); + let ftpUser = cuid(); let ftpPassword = generatePassword(); From 3a252509d050c0d7cf1091574050ba9de4544e13 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 14 Apr 2022 15:04:18 +0200 Subject: [PATCH 05/10] fix: Add HTTP proxy checks --- src/lib/haproxy/index.ts | 8 ++++---- src/lib/queues/proxyTcpHttp.ts | 16 +++++++++++++++- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/lib/haproxy/index.ts b/src/lib/haproxy/index.ts index 618d410d5..9aa3797b4 100644 --- a/src/lib/haproxy/index.ts +++ b/src/lib/haproxy/index.ts @@ -127,10 +127,10 @@ export async function startTcpProxy( const containerName = `haproxy-for-${publicPort}`; const found = await checkContainer(engine, containerName); - const foundDB = await checkContainer(engine, id); + const foundDependentContainer = await checkContainer(engine, id); try { - if (foundDB && !found) { + if (foundDependentContainer && !found) { const { stdout: Config } = await asyncExecShell( `DOCKER_HOST="${host}" docker network inspect bridge --format '{{json .IPAM.Config }}'` ); @@ -157,10 +157,10 @@ export async function startHttpProxy( const containerName = `haproxy-for-${publicPort}`; const found = await checkContainer(engine, containerName); - const foundDB = await checkContainer(engine, id); + const foundDependentContainer = await checkContainer(engine, id); try { - if (foundDB && !found) { + if (foundDependentContainer && !found) { const { stdout: Config } = await asyncExecShell( `DOCKER_HOST="${host}" docker network inspect bridge --format '{{json .IPAM.Config }}'` ); diff --git a/src/lib/queues/proxyTcpHttp.ts b/src/lib/queues/proxyTcpHttp.ts index 00b5d759e..f9c509587 100644 --- a/src/lib/queues/proxyTcpHttp.ts +++ b/src/lib/queues/proxyTcpHttp.ts @@ -1,11 +1,12 @@ import { ErrorHandler, generateDatabaseConfiguration, prisma } from '$lib/database'; -import { checkContainer, startTcpProxy } from '$lib/haproxy'; +import { startHttpProxy, startTcpProxy } from '$lib/haproxy'; export default async function (): Promise { try { + // TCP Proxies const databasesWithPublicPort = await prisma.database.findMany({ where: { publicPort: { not: null } }, include: { settings: true, destinationDocker: true } @@ -28,6 +29,19 @@ export default async function (): Promise Date: Thu, 14 Apr 2022 18:34:43 +0200 Subject: [PATCH 06/10] fix: Loading of new destinations --- src/routes/new/destination/_LocalDocker.svelte | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/routes/new/destination/_LocalDocker.svelte b/src/routes/new/destination/_LocalDocker.svelte index ca1c0733e..e02a8163a 100644 --- a/src/routes/new/destination/_LocalDocker.svelte +++ b/src/routes/new/destination/_LocalDocker.svelte @@ -11,7 +11,9 @@ let loading = false; async function handleSubmit() { + if (loading) return; try { + loading = true; await post('/new/destination/check.json', { network: payload.network }); const { id } = await post('/new/destination/docker.json', { ...payload From f5714536967d10b8eecd392cfcc071f7f49d7c8e Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 14 Apr 2022 18:45:42 +0200 Subject: [PATCH 07/10] fix: Better performance for cleanup images --- src/lib/queues/cleanup.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/lib/queues/cleanup.ts b/src/lib/queues/cleanup.ts index 05f795799..d5dc49381 100644 --- a/src/lib/queues/cleanup.ts +++ b/src/lib/queues/cleanup.ts @@ -2,8 +2,9 @@ import { asyncExecShell, getEngine, version } from '$lib/common'; import { prisma } from '$lib/database'; export default async function (): Promise { const destinationDockers = await prisma.destinationDocker.findMany(); - for (const destinationDocker of destinationDockers) { - const host = getEngine(destinationDocker.engine); + const engines = [...new Set(destinationDockers.map(({ engine }) => engine))]; + for (const engine of engines) { + const host = getEngine(engine); // Cleanup old coolify images try { let { stdout: images } = await asyncExecShell( @@ -28,7 +29,7 @@ export default async function (): Promise { } // Cleanup old images older than a day try { - await asyncExecShell(`DOCKER_HOST=${host} docker image prune --filter "until=24h" -a -f`); + await asyncExecShell(`DOCKER_HOST=${host} docker image prune --filter "until=72h" -a -f`); } catch (error) { //console.log(error); } From 25ff63770338810753541bb1bc00a28b7a3a4934 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 14 Apr 2022 21:43:05 +0200 Subject: [PATCH 08/10] fix: Remove proxy container in case of dependent container is down --- src/lib/haproxy/index.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/lib/haproxy/index.ts b/src/lib/haproxy/index.ts index 9aa3797b4..6770d5c93 100644 --- a/src/lib/haproxy/index.ts +++ b/src/lib/haproxy/index.ts @@ -141,6 +141,11 @@ export async function startTcpProxy( } -d coollabsio/${defaultProxyImageTcp}` ); } + if (!foundDependentContainer && found) { + return await asyncExecShell( + `DOCKER_HOST=${host} docker stop -t 0 ${containerName} && docker rm ${containerName}` + ); + } } catch (error) { return error; } @@ -169,6 +174,11 @@ export async function startHttpProxy( `DOCKER_HOST=${host} docker run --restart always -e PORT=${publicPort} -e APP=${id} -e PRIVATE_PORT=${privatePort} --add-host 'host.docker.internal:host-gateway' --add-host 'host.docker.internal:${ip}' --network ${network} -p ${publicPort}:${publicPort} --name ${containerName} -d coollabsio/${defaultProxyImageHttp}` ); } + if (!foundDependentContainer && found) { + return await asyncExecShell( + `DOCKER_HOST=${host} docker stop -t 0 ${containerName} && docker rm ${containerName}` + ); + } } catch (error) { return error; } From fd4bbe17f0caaebd69e4da830512ff748ebe8c77 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 14 Apr 2022 21:43:22 +0200 Subject: [PATCH 09/10] fix: Restart local docker coolify proxy in case of something happens to it --- src/lib/queues/proxyTcpHttp.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/lib/queues/proxyTcpHttp.ts b/src/lib/queues/proxyTcpHttp.ts index f9c509587..5ba2266c2 100644 --- a/src/lib/queues/proxyTcpHttp.ts +++ b/src/lib/queues/proxyTcpHttp.ts @@ -1,11 +1,18 @@ import { ErrorHandler, generateDatabaseConfiguration, prisma } from '$lib/database'; -import { startHttpProxy, startTcpProxy } from '$lib/haproxy'; +import { startCoolifyProxy, startHttpProxy, startTcpProxy } from '$lib/haproxy'; export default async function (): Promise { try { + // Coolify Proxy + const localDocker = await prisma.destinationDocker.findFirst({ + where: { engine: '/var/run/docker.sock' } + }); + if (localDocker && localDocker.isCoolifyProxyUsed) { + await startCoolifyProxy('/var/run/docker.sock'); + } // TCP Proxies const databasesWithPublicPort = await prisma.database.findMany({ where: { publicPort: { not: null } }, From 260a47a366a90b0611ef08c54ac9722ede48736e Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 14 Apr 2022 23:11:24 +0200 Subject: [PATCH 10/10] fix: Id of service container --- src/lib/queues/proxyTcpHttp.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/queues/proxyTcpHttp.ts b/src/lib/queues/proxyTcpHttp.ts index 5ba2266c2..c268d2aa0 100644 --- a/src/lib/queues/proxyTcpHttp.ts +++ b/src/lib/queues/proxyTcpHttp.ts @@ -30,8 +30,8 @@ export default async function (): Promise