From eee201013c0f21eb66278dfc133f7f4ee799ee95 Mon Sep 17 00:00:00 2001 From: David Koch Gregersen Date: Mon, 6 Mar 2023 22:31:01 +0100 Subject: [PATCH 1/7] Fixing multiple remotes breaking the server overview --- apps/api/src/lib/common.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/api/src/lib/common.ts b/apps/api/src/lib/common.ts index 0014f793e..4ef8790f9 100644 --- a/apps/api/src/lib/common.ts +++ b/apps/api/src/lib/common.ts @@ -510,7 +510,9 @@ export async function createRemoteEngineConfiguration(id: string) { remoteUser } = await prisma.destinationDocker.findFirst({ where: { id }, include: { sshKey: true } }); await fs.writeFile(sshKeyFile, decrypt(privateKey) + '\n', { encoding: 'utf8', mode: 400 }); - const config = sshConfig.parse(''); + + const currentConfigFileContent = (await fs.readFile(`${homedir}/.ssh/config`)).toString(); + const config = sshConfig.parse(currentConfigFileContent.toString()); const Host = `${remoteIpAddress}-remote`; try { From 04f7e8e7776417f7a288b62e70ba355ad043153c Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 7 Mar 2023 10:31:10 +0100 Subject: [PATCH 2/7] fix: host volumes --- apps/api/src/lib/buildPacks/compose.ts | 24 +++++++++---- apps/api/src/lib/common.ts | 2 +- .../routes/applications/[id]/storages.svelte | 36 +++++++++++-------- package.json | 2 +- 4 files changed, 40 insertions(+), 24 deletions(-) diff --git a/apps/api/src/lib/buildPacks/compose.ts b/apps/api/src/lib/buildPacks/compose.ts index 439462231..2d16c1f8d 100644 --- a/apps/api/src/lib/buildPacks/compose.ts +++ b/apps/api/src/lib/buildPacks/compose.ts @@ -78,15 +78,25 @@ export default async function (data) { if (value['volumes']?.length > 0) { value['volumes'] = value['volumes'].map((volume) => { let [v, path, permission] = volume.split(':'); - if (!path) { - path = v; - v = `${applicationId}${v.replace(/\//gi, '-').replace(/\./gi, '')}`; + if ( + v.startsWith('.') || + v.startsWith('..') || + v.startsWith('/') || + v.startsWith('~') + ) { + // Nothing to do here, host path } else { - v = `${applicationId}${v.replace(/\//gi, '-').replace(/\./gi, '')}`; + if (!path) { + path = v; + v = `${applicationId}${v.replace(/\//gi, '-').replace(/\./gi, '')}`; + } else { + v = `${applicationId}${v.replace(/\//gi, '-').replace(/\./gi, '')}`; + } + composeVolumes[v] = { + name: v + }; } - composeVolumes[v] = { - name: v - }; + return `${v}:${path}${permission ? ':' + permission : ''}`; }); } diff --git a/apps/api/src/lib/common.ts b/apps/api/src/lib/common.ts index 67d8775e7..8b931653d 100644 --- a/apps/api/src/lib/common.ts +++ b/apps/api/src/lib/common.ts @@ -19,7 +19,7 @@ import { saveBuildLog, saveDockerRegistryCredentials } from './buildPacks/common import { scheduler } from './scheduler'; import type { ExecaChildProcess } from 'execa'; -export const version = '3.12.23'; +export const version = '3.12.26'; export const isDev = process.env.NODE_ENV === 'development'; export const proxyPort = process.env.COOLIFY_PROXY_PORT; export const proxySecurePort = process.env.COOLIFY_PROXY_SECURE_PORT; diff --git a/apps/ui/src/routes/applications/[id]/storages.svelte b/apps/ui/src/routes/applications/[id]/storages.svelte index d67d84a32..7f16645a2 100644 --- a/apps/ui/src/routes/applications/[id]/storages.svelte +++ b/apps/ui/src/routes/applications/[id]/storages.svelte @@ -36,14 +36,20 @@ if (service?.volumes) { for (const [_, volumeName] of Object.entries(service.volumes)) { let [volume, target] = volumeName.split(':'); - if (volume === '.') { - volume = target; - } - if (!target) { - target = volume; - volume = `${application.id}${volume.replace(/\//gi, '-').replace(/\./gi, '')}`; + if ( + volume.startsWith('.') || + volume.startsWith('..') || + volume.startsWith('/') || + volume.startsWith('~') + ) { + // Nothing to do here, host path } else { - volume = `${application.id}${volume.replace(/\//gi, '-').replace(/\./gi, '')}`; + if (!target) { + target = volume; + volume = `${application.id}${volume.replace(/\//gi, '-').replace(/\./gi, '')}`; + } else { + volume = `${application.id}${volume.replace(/\//gi, '-').replace(/\./gi, '')}`; + } } predefinedVolumes.push({ id: volume, path: target, predefined: true }); } @@ -88,14 +94,14 @@ {/key} {/each} {#if $appSession.isAdmin} -
0}> - Add New Volume -
- - +
0}> + Add New Volume +
+ + {/if} diff --git a/package.json b/package.json index 4da878fce..e518e9dda 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "coolify", "description": "An open-source & self-hostable Heroku / Netlify alternative.", - "version": "3.12.23", + "version": "3.12.26", "license": "Apache-2.0", "repository": "github:coollabsio/coolify", "scripts": { From edb66620c13ac863aa23a2321df193021df0a7c0 Mon Sep 17 00:00:00 2001 From: David Koch Gregersen Date: Tue, 7 Mar 2023 10:43:34 +0100 Subject: [PATCH 3/7] Adding a check when reading ssh config file Also adds comments to the createRemoteEngineConfiguration function --- apps/api/src/lib/common.ts | 39 +++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/apps/api/src/lib/common.ts b/apps/api/src/lib/common.ts index 4ef8790f9..8832fdb8a 100644 --- a/apps/api/src/lib/common.ts +++ b/apps/api/src/lib/common.ts @@ -11,7 +11,7 @@ import { promises as dns } from 'dns'; import * as Sentry from '@sentry/node'; import { PrismaClient } from '@prisma/client'; import os from 'os'; -import sshConfig from 'ssh-config'; +import * as SSHConfig from 'ssh-config/src/ssh-config'; import jsonwebtoken from 'jsonwebtoken'; import { checkContainer, removeContainer } from './docker'; import { day } from './dayjs'; @@ -498,35 +498,56 @@ export async function getFreeSSHLocalPort(id: string): Promise return false; } +/** + * Update the ssh config file with a host + * + * @param id Destination ID + * @returns + */ export async function createRemoteEngineConfiguration(id: string) { - const homedir = os.homedir(); const sshKeyFile = `/tmp/id_rsa-${id}`; const localPort = await getFreeSSHLocalPort(id); const { sshKey: { privateKey }, - network, remoteIpAddress, remotePort, remoteUser } = await prisma.destinationDocker.findFirst({ where: { id }, include: { sshKey: true } }); + + // Write new keyfile await fs.writeFile(sshKeyFile, decrypt(privateKey) + '\n', { encoding: 'utf8', mode: 400 }); - const currentConfigFileContent = (await fs.readFile(`${homedir}/.ssh/config`)).toString(); - const config = sshConfig.parse(currentConfigFileContent.toString()); const Host = `${remoteIpAddress}-remote`; + // Removes previous ssh-keys try { await executeCommand({ command: `ssh-keygen -R ${Host}` }); await executeCommand({ command: `ssh-keygen -R ${remoteIpAddress}` }); await executeCommand({ command: `ssh-keygen -R localhost:${localPort}` }); - } catch (error) { } + } catch (error) { + // + } + const homedir = os.homedir(); + let currentConfigFileContent = ''; + try { + // Read the current config file + currentConfigFileContent = (await fs.readFile(`${homedir}/.ssh/config`)).toString(); + } catch (error) { + // File doesn't exist, so we do nothing, a new one is going to be created + } + + // Parse the config file + const config = SSHConfig.parse(currentConfigFileContent); + + // Remove current config for the given host const found = config.find({ Host }); const foundIp = config.find({ Host: remoteIpAddress }); if (found) config.remove({ Host }); if (foundIp) config.remove({ Host: remoteIpAddress }); + // Create the new config config.append({ Host, Hostname: remoteIpAddress, @@ -539,13 +560,17 @@ export async function createRemoteEngineConfiguration(id: string) { ControlPersist: '10m' }); + // Check if .ssh folder exists, and if not create one try { await fs.stat(`${homedir}/.ssh/`); } catch (error) { await fs.mkdir(`${homedir}/.ssh/`); } - return await fs.writeFile(`${homedir}/.ssh/config`, sshConfig.stringify(config)); + + // Write the config + return await fs.writeFile(`${homedir}/.ssh/config`, SSHConfig.stringify(config)); } + export async function executeCommand({ command, dockerId = null, From 3e81d7e9cb00932e5e486c84888b0bbbc5cf02d4 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 7 Mar 2023 10:44:53 +0100 Subject: [PATCH 4/7] fix: replace . & .. & $PWD with ~ --- apps/api/src/lib/buildPacks/compose.ts | 5 +++-- apps/ui/src/routes/applications/[id]/storages.svelte | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/apps/api/src/lib/buildPacks/compose.ts b/apps/api/src/lib/buildPacks/compose.ts index 2d16c1f8d..70a2ba8af 100644 --- a/apps/api/src/lib/buildPacks/compose.ts +++ b/apps/api/src/lib/buildPacks/compose.ts @@ -82,9 +82,10 @@ export default async function (data) { v.startsWith('.') || v.startsWith('..') || v.startsWith('/') || - v.startsWith('~') + v.startsWith('~') || + v.startsWith('$PWD') ) { - // Nothing to do here, host path + v = v.replace('$.', `~`).replace('$..', '~').replace('$$PWD', '~'); } else { if (!path) { path = v; diff --git a/apps/ui/src/routes/applications/[id]/storages.svelte b/apps/ui/src/routes/applications/[id]/storages.svelte index 7f16645a2..8a810b635 100644 --- a/apps/ui/src/routes/applications/[id]/storages.svelte +++ b/apps/ui/src/routes/applications/[id]/storages.svelte @@ -40,9 +40,10 @@ volume.startsWith('.') || volume.startsWith('..') || volume.startsWith('/') || - volume.startsWith('~') + volume.startsWith('~') || + volume.startsWith('$PWD') ) { - // Nothing to do here, host path + volume = volume.replace('$.', `~`).replace('$..', '~').replace('$$PWD', '~'); } else { if (!target) { target = volume; From 1c237affb40ece0bdc55cebd5f9bb13dc73d18f9 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 7 Mar 2023 11:15:05 +0100 Subject: [PATCH 5/7] feat: add host path to any container --- .../migration.sql | 2 ++ apps/api/prisma/schema.prisma | 1 + apps/api/src/jobs/deployApplication.ts | 18 ++++++++-- apps/api/src/lib/buildPacks/compose.ts | 11 +++--- apps/api/src/lib/common.ts | 3 ++ .../routes/api/v1/applications/handlers.ts | 6 ++-- .../src/routes/api/v1/applications/types.ts | 1 + .../routes/applications/[id]/_Storage.svelte | 36 ++++++++++++++----- 8 files changed, 60 insertions(+), 18 deletions(-) create mode 100644 apps/api/prisma/migrations/20230307101148_add_host_volumes/migration.sql diff --git a/apps/api/prisma/migrations/20230307101148_add_host_volumes/migration.sql b/apps/api/prisma/migrations/20230307101148_add_host_volumes/migration.sql new file mode 100644 index 000000000..220ad995c --- /dev/null +++ b/apps/api/prisma/migrations/20230307101148_add_host_volumes/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "ApplicationPersistentStorage" ADD COLUMN "hostPath" TEXT; diff --git a/apps/api/prisma/schema.prisma b/apps/api/prisma/schema.prisma index bb76d83b0..7dca8314b 100644 --- a/apps/api/prisma/schema.prisma +++ b/apps/api/prisma/schema.prisma @@ -195,6 +195,7 @@ model ApplicationSettings { model ApplicationPersistentStorage { id String @id @default(cuid()) applicationId String + hostPath String? path String oldPath Boolean @default(false) createdAt DateTime @default(now()) diff --git a/apps/api/src/jobs/deployApplication.ts b/apps/api/src/jobs/deployApplication.ts index 8fb71cfe6..d3de46268 100644 --- a/apps/api/src/jobs/deployApplication.ts +++ b/apps/api/src/jobs/deployApplication.ts @@ -110,6 +110,9 @@ import * as buildpacks from '../lib/buildPacks'; .replace(/\//gi, '-') .replace('-app', '')}:${storage.path}`; } + if (storage.hostPath) { + return `${storage.hostPath}:${storage.path}` + } return `${applicationId}${storage.path.replace(/\//gi, '-')}:${storage.path}`; }) || []; @@ -160,7 +163,11 @@ import * as buildpacks from '../lib/buildPacks'; port: exposePort ? `${exposePort}:${port}` : port }); try { - const composeVolumes = volumes.map((volume) => { + const composeVolumes = volumes.filter(v => { + if (!v.startsWith('.') && !v.startsWith('..') && !v.startsWith('/') && !v.startsWith('~')) { + return v; + } + }).map((volume) => { return { [`${volume.split(':')[0]}`]: { name: volume.split(':')[0] @@ -381,6 +388,9 @@ import * as buildpacks from '../lib/buildPacks'; .replace(/\//gi, '-') .replace('-app', '')}:${storage.path}`; } + if (storage.hostPath) { + return `${storage.hostPath}:${storage.path}` + } return `${applicationId}${storage.path.replace(/\//gi, '-')}:${storage.path}`; }) || []; @@ -691,7 +701,11 @@ import * as buildpacks from '../lib/buildPacks'; await saveDockerRegistryCredentials({ url, username, password, workdir }); } try { - const composeVolumes = volumes.map((volume) => { + const composeVolumes = volumes.filter(v => { + if (!v.startsWith('.') && !v.startsWith('..') && !v.startsWith('/') && !v.startsWith('~')) { + return v; + } + }).map((volume) => { return { [`${volume.split(':')[0]}`]: { name: volume.split(':')[0] diff --git a/apps/api/src/lib/buildPacks/compose.ts b/apps/api/src/lib/buildPacks/compose.ts index 70a2ba8af..49d4a715b 100644 --- a/apps/api/src/lib/buildPacks/compose.ts +++ b/apps/api/src/lib/buildPacks/compose.ts @@ -36,12 +36,13 @@ export default async function (data) { if (volumes.length > 0) { for (const volume of volumes) { let [v, path] = volume.split(':'); - composeVolumes[v] = { - name: v - }; + if (!v.startsWith('.') && !v.startsWith('..') && !v.startsWith('/') && !v.startsWith('~')) { + composeVolumes[v] = { + name: v + }; + } } } - let networks = {}; for (let [key, value] of Object.entries(dockerComposeYaml.services)) { value['container_name'] = `${applicationId}-${key}`; @@ -78,6 +79,7 @@ export default async function (data) { if (value['volumes']?.length > 0) { value['volumes'] = value['volumes'].map((volume) => { let [v, path, permission] = volume.split(':'); + console.log(v, path, permission) if ( v.startsWith('.') || v.startsWith('..') || @@ -106,6 +108,7 @@ export default async function (data) { value['volumes'].push(volume); } } + console.log({ volumes, composeVolumes }) if (dockerComposeConfiguration[key]?.port) { value['expose'] = [dockerComposeConfiguration[key].port]; } diff --git a/apps/api/src/lib/common.ts b/apps/api/src/lib/common.ts index 8b931653d..5626d16ec 100644 --- a/apps/api/src/lib/common.ts +++ b/apps/api/src/lib/common.ts @@ -1633,6 +1633,9 @@ export function errorHandler({ type?: string | null; }) { if (message.message) message = message.message; + if (message.includes('Unique constraint failed')) { + message = 'This data is unique and already exists. Please try again with a different value.'; + } if (type === 'normal') { Sentry.captureException(message); } diff --git a/apps/api/src/routes/api/v1/applications/handlers.ts b/apps/api/src/routes/api/v1/applications/handlers.ts index 886b7bc88..8621a93ca 100644 --- a/apps/api/src/routes/api/v1/applications/handlers.ts +++ b/apps/api/src/routes/api/v1/applications/handlers.ts @@ -1340,16 +1340,16 @@ export async function getStorages(request: FastifyRequest) { export async function saveStorage(request: FastifyRequest, reply: FastifyReply) { try { const { id } = request.params; - const { path, newStorage, storageId } = request.body; + const { hostPath, path, newStorage, storageId } = request.body; if (newStorage) { await prisma.applicationPersistentStorage.create({ - data: { path, application: { connect: { id } } } + data: { hostPath, path, application: { connect: { id } } } }); } else { await prisma.applicationPersistentStorage.update({ where: { id: storageId }, - data: { path } + data: { hostPath, path } }); } return reply.code(201).send(); diff --git a/apps/api/src/routes/api/v1/applications/types.ts b/apps/api/src/routes/api/v1/applications/types.ts index 517194bd6..1c42f468a 100644 --- a/apps/api/src/routes/api/v1/applications/types.ts +++ b/apps/api/src/routes/api/v1/applications/types.ts @@ -96,6 +96,7 @@ export interface DeleteSecret extends OnlyId { } export interface SaveStorage extends OnlyId { Body: { + hostPath?: string; path: string; newStorage: boolean; storageId: string; diff --git a/apps/ui/src/routes/applications/[id]/_Storage.svelte b/apps/ui/src/routes/applications/[id]/_Storage.svelte index 0e0e13b8b..6eeb2c66b 100644 --- a/apps/ui/src/routes/applications/[id]/_Storage.svelte +++ b/apps/ui/src/routes/applications/[id]/_Storage.svelte @@ -12,6 +12,7 @@ import { errorNotification } from '$lib/common'; import { addToast } from '$lib/store'; import CopyVolumeField from '$lib/components/CopyVolumeField.svelte'; + import SimpleExplainer from '$lib/components/SimpleExplainer.svelte'; const { id } = $page.params; let isHttps = browser && window.location.protocol === 'https:'; export let value: string; @@ -33,11 +34,13 @@ storage.path.replace(/\/\//g, '/'); await post(`/applications/${id}/storages`, { path: storage.path, + hostPath: storage.hostPath, storageId: storage.id, newStorage }); dispatch('refresh'); if (isNew) { + storage.hostPath = null; storage.path = null; storage.id = null; } @@ -80,27 +83,42 @@
{#if storage.applicationId} {#if storage.oldPath} - - + {:else if !storage.hostPath} + - {:else} - - {/if} {/if} + + {#if isNew} +
+ + + +
+ {:else if storage.hostPath} + + {/if} -
+
{#if isNew}
{:else if $isDeploymentEnabled && !$page.url.pathname.startsWith(`/applications/${id}/configuration/`)} - {#if $status.application.overallStatus === 'degraded'} - - {/if} + {/if} {/if} {#if $location && $status.application.overallStatus === 'healthy'}