Shiloh/lasthourcloud
Shiloh/lasthourcloud
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 4 Wiki Activity

fix: remote traefik webhook

Browse Source
This commit is contained in:
Andras Bacsai 2022-07-22 20:23:16 +00:00
parent c0e513127d
commit bb2864a83f
8 changed files with 339 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
apps/api/package.json
View File

@ -44,6 +44,7 @@
"node-forge": "1.3.1",
"node-os-utils": "1.3.7",
"p-queue": "7.2.0",
"public-ip": "6.0.1",
"ssh-config": "4.1.6",
"strip-ansi": "7.0.1",
"unique-names-generator": "4.7.1"

4
apps/api/src/jobs/checkProxies.ts
View File

@ -1,6 +1,6 @@
import { parentPort } from 'node:worker_threads';
import { prisma, startTraefikTCPProxy, generateDatabaseConfiguration, startTraefikProxy, asyncExecShell, executeDockerCmd } from '../lib/common';
import { checkContainer, getEngine } from '../lib/docker';
import { prisma, startTraefikTCPProxy, generateDatabaseConfiguration, startTraefikProxy, executeDockerCmd } from '../lib/common';
import { checkContainer } from '../lib/docker';
(async () => {
if (parentPort) {

5
apps/api/src/lib/buildPacks/common.ts
View File

@ -1,8 +1,7 @@
import { asyncExecShell, base64Encode, executeDockerCmd, generateTimestamp, getDomain, isDev, prisma, version } from "../common";
import { scheduler } from "../scheduler";
import { base64Encode, executeDockerCmd, generateTimestamp, getDomain, isDev, prisma, version } from "../common";
import { promises as fs } from 'fs';
import { day } from "../dayjs";
import { spawn } from 'node:child_process'
const staticApps = ['static', 'react', 'vuejs', 'svelte', 'gatsby', 'astro', 'eleventy'];
const nodeBased = [
'react',

42
apps/api/src/lib/common.ts
View File

@ -505,7 +505,7 @@ export async function executeDockerCmd({ dockerId, command }: { dockerId: string
);
}
export async function startTraefikProxy(id: string): Promise<void> {
const { engine, network, remoteEngine } = await prisma.destinationDocker.findUnique({ where: { id } })
const { engine, network, remoteEngine, remoteIpAddress } = await prisma.destinationDocker.findUnique({ where: { id } })
const found = await checkContainer({ dockerId: id, container: 'coolify-proxy', remove: true });
const { id: settingsId } = await listSettings();
@ -513,6 +513,17 @@ export async function startTraefikProxy(id: string): Promise<void> {
if (!found) {
const { stdout: Config } = await executeDockerCmd({ dockerId: id, command: `docker network inspect ${network} --format '{{json .IPAM.Config }}'` })
const ip = JSON.parse(Config)[0].Gateway;
const { publicIp } = await import('public-ip')
let traefikUrl = mainTraefikEndpoint
if (remoteEngine) {
let ip = null
if (isDev) {
ip = getAPIUrl()
} else {
ip = `http://${await publicIp({ timeout: 2000 })}`
}
traefikUrl = `${ip}/webhooks/traefik/remote/${id}`
}
await executeDockerCmd({
dockerId: id,
command: `docker run --restart always \
@ -531,7 +542,7 @@ export async function startTraefikProxy(id: string): Promise<void> {
--entrypoints.websecure.forwardedHeaders.insecure=true \
--providers.docker=true \
--providers.docker.exposedbydefault=false \
--providers.http.endpoint=${mainTraefikEndpoint} \
--providers.http.endpoint=${traefikUrl} \
--providers.http.pollTimeout=5s \
--certificatesresolvers.letsencrypt.acme.httpchallenge=true \
--certificatesresolvers.letsencrypt.acme.storage=/etc/traefik/acme/acme.json \
@ -544,21 +555,32 @@ export async function startTraefikProxy(id: string): Promise<void> {
data: { isCoolifyProxyUsed: true }
});
}
if (!remoteEngine) await configureNetworkTraefikProxy(engine, id);
// Configure networks for local docker engine
if (engine) {
const destinations = await prisma.destinationDocker.findMany({ where: { engine } });
for (const destination of destinations) {
await configureNetworkTraefikProxy(destination);
}
}
// Configure networks for remote docker engine
if (remoteEngine) {
const destinations = await prisma.destinationDocker.findMany({ where: { remoteIpAddress } });
for (const destination of destinations) {
await configureNetworkTraefikProxy(destination);
}
}
}
export async function configureNetworkTraefikProxy(engine: string, id: string): Promise<void> {
const destinations = await prisma.destinationDocker.findMany({ where: { engine } });
export async function configureNetworkTraefikProxy(destination: any): Promise<void> {
const { id } = destination
const { stdout: networks } = await executeDockerCmd({
dockerId: id,
command:
`docker ps -a --filter name=coolify-proxy --format '{{json .Networks}}'`
});
const configuredNetworks = networks.replace(/"/g, '').replace('\n', '').split(',');
for (const destination of destinations) {
if (!configuredNetworks.includes(destination.network)) {
await executeDockerCmd({ dockerId: destination.id, command: `docker network connect ${destination.network} coolify-proxy` })
}
if (!configuredNetworks.includes(destination.network)) {
await executeDockerCmd({ dockerId: destination.id, command: `docker network connect ${destination.network} coolify-proxy` })
}
}
@ -1529,7 +1551,7 @@ export function convertTolOldVolumeNames(type) {
// export async function getAvailableServices(): Promise<any> {
// const { data } = await axios.get(`https://gist.githubusercontent.com/andrasbacsai/4aac36d8d6214dbfc34fa78110554a50/raw/5b27e6c37d78aaeedc1148d797112c827a2f43cf/availableServices.json`)
// return data
// }
//
export async function cleanupDockerStorage(dockerId, lowDiskSpace, force) {
// Cleanup old coolify images
try {

2
apps/api/src/lib/docker.ts
View File

@ -1,4 +1,4 @@
import { asyncExecShell, executeDockerCmd } from './common';
import { executeDockerCmd } from './common';
import Dockerode from 'dockerode';
export function getEngine(engine: string): string {
return engine === '/var/run/docker.sock' ? 'unix:///var/run/docker.sock' : engine;

244
apps/api/src/routes/webhooks/traefik/handlers.ts
View File

@ -1,5 +1,5 @@
import { FastifyRequest } from "fastify";
import { asyncExecShell, errorHandler, getDomain, isDev, listServicesWithIncludes, prisma, supportedServiceTypesAndVersions } from "../../../lib/common";
import { asyncExecShell, errorHandler, getDomain, isDev, listServicesWithIncludes, prisma, supportedServiceTypesAndVersions, include } from "../../../lib/common";
import { getEngine } from "../../../lib/docker";
import { TraefikOtherConfiguration } from "./types";
@ -167,6 +167,7 @@ export async function traefikConfiguration(request, reply) {
}
};
const applications = await prisma.application.findMany({
where: { destinationDocker: { remoteEngine: false } },
include: { destinationDocker: true, settings: true }
});
const data = {
@ -235,7 +236,11 @@ export async function traefikConfiguration(request, reply) {
}
}
}
const services = await listServicesWithIncludes();
const services: any = await prisma.service.findMany({
where: { destinationDocker: { remoteEngine: false } },
include,
orderBy: { createdAt: 'desc' },
});
for (const service of services) {
const {
@ -487,4 +492,239 @@ export async function traefikOtherConfiguration(request: FastifyRequest<TraefikO
console.log(status, message);
return errorHandler({ status, message })
}
}
export async function remoteTraefikConfiguration(request: FastifyRequest) {
const { id } = request.params
try {
const traefik = {
http: {
routers: {},
services: {},
middlewares: {
'redirect-to-https': {
redirectscheme: {
scheme: 'https'
}
},
'redirect-to-http': {
redirectscheme: {
scheme: 'http'
}
},
'redirect-to-non-www': {
redirectregex: {
regex: '^https?://www\\.(.+)',
replacement: 'http://${1}'
}
},
'redirect-to-www': {
redirectregex: {
regex: '^https?://(?:www\\.)?(.+)',
replacement: 'http://www.${1}'
}
}
}
}
};
const applications = await prisma.application.findMany({
where: { destinationDocker: { id } },
include: { destinationDocker: true, settings: true }
});
const data = {
applications: [],
services: [],
coolify: []
};
for (const application of applications) {
const {
fqdn,
id,
port,
destinationDocker,
destinationDockerId,
settings: { previews, dualCerts }
} = application;
if (destinationDockerId) {
const { engine, network } = destinationDocker;
const isRunning = true;
if (fqdn) {
const domain = getDomain(fqdn);
const nakedDomain = domain.replace(/^www\./, '');
const isHttps = fqdn.startsWith('https://');
const isWWW = fqdn.includes('www.');
if (isRunning) {
data.applications.push({
id,
container: id,
port: port || 3000,
domain,
nakedDomain,
isRunning,
isHttps,
isWWW,
isDualCerts: dualCerts
});
}
if (previews) {
const host = getEngine(engine);
const { stdout } = await asyncExecShell(
`DOCKER_HOST=${host} docker container ls --filter="status=running" --filter="network=${network}" --filter="name=${id}-" --format="{{json .Names}}"`
);
const containers = stdout
.trim()
.split('\n')
.filter((a) => a)
.map((c) => c.replace(/"/g, ''));
if (containers.length > 0) {
for (const container of containers) {
const previewDomain = `${container.split('-')[1]}.${domain}`;
const nakedDomain = previewDomain.replace(/^www\./, '');
data.applications.push({
id: container,
container,
port: port || 3000,
domain: previewDomain,
isRunning,
nakedDomain,
isHttps,
isWWW,
isDualCerts: dualCerts
});
}
}
}
}
}
}
const services: any = await prisma.service.findMany({
where: { destinationDocker: { id } },
include,
orderBy: { createdAt: 'desc' }
});
for (const service of services) {
const {
fqdn,
id,
type,
destinationDocker,
destinationDockerId,
dualCerts,
plausibleAnalytics
} = service;
if (destinationDockerId) {
const { engine } = destinationDocker;
const found = supportedServiceTypesAndVersions.find((a) => a.name === type);
if (found) {
const port = found.ports.main;
const publicPort = service[type]?.publicPort;
const isRunning = true;
if (fqdn) {
const domain = getDomain(fqdn);
const nakedDomain = domain.replace(/^www\./, '');
const isHttps = fqdn.startsWith('https://');
const isWWW = fqdn.includes('www.');
if (isRunning) {
// Plausible Analytics custom script
let scriptName = false;
if (type === 'plausibleanalytics' && plausibleAnalytics.scriptName !== 'plausible.js') {
scriptName = plausibleAnalytics.scriptName;
}
let container = id;
let otherDomain = null;
let otherNakedDomain = null;
let otherIsHttps = null;
let otherIsWWW = null;
if (type === 'minio' && service.minio.apiFqdn) {
otherDomain = getDomain(service.minio.apiFqdn);
otherNakedDomain = otherDomain.replace(/^www\./, '');
otherIsHttps = service.minio.apiFqdn.startsWith('https://');
otherIsWWW = service.minio.apiFqdn.includes('www.');
}
data.services.push({
id,
container,
type,
otherDomain,
otherNakedDomain,
otherIsHttps,
otherIsWWW,
port,
publicPort,
domain,
nakedDomain,
isRunning,
isHttps,
isWWW,
isDualCerts: dualCerts,
scriptName
});
}
}
}
}
}
const { fqdn, dualCerts } = await prisma.setting.findFirst();
if (fqdn) {
const domain = getDomain(fqdn);
const nakedDomain = domain.replace(/^www\./, '');
const isHttps = fqdn.startsWith('https://');
const isWWW = fqdn.includes('www.');
data.coolify.push({
id: isDev ? 'host.docker.internal' : 'coolify',
container: isDev ? 'host.docker.internal' : 'coolify',
port: 3000,
domain,
nakedDomain,
isHttps,
isWWW,
isDualCerts: dualCerts
});
}
for (const application of data.applications) {
configureMiddleware(application, traefik);
}
for (const service of data.services) {
const { id, scriptName } = service;
configureMiddleware(service, traefik);
if (service.type === 'minio') {
service.id = id + '-minio';
service.container = id;
service.domain = service.otherDomain;
service.nakedDomain = service.otherNakedDomain;
service.isHttps = service.otherIsHttps;
service.isWWW = service.otherIsWWW;
service.port = 9000;
configureMiddleware(service, traefik);
}
if (scriptName) {
traefik.http.middlewares[`${id}-redir`] = {
replacepathregex: {
regex: `/js/${scriptName}`,
replacement: '/js/plausible.js'
}
};
}
}
for (const coolify of data.coolify) {
configureMiddleware(coolify, traefik);
}
if (Object.keys(traefik.http.routers).length === 0) {
traefik.http.routers = null;
}
if (Object.keys(traefik.http.services).length === 0) {
traefik.http.services = null;
}
return {
...traefik
}
} catch ({ status, message }) {
return errorHandler({ status, message })
}
}

4
apps/api/src/routes/webhooks/traefik/index.ts
View File

@ -1,10 +1,12 @@
import { FastifyPluginAsync } from 'fastify';
import { traefikConfiguration, traefikOtherConfiguration } from './handlers';
import { remoteTraefikConfiguration, traefikConfiguration, traefikOtherConfiguration } from './handlers';
import { TraefikOtherConfiguration } from './types';
const root: FastifyPluginAsync = async (fastify): Promise<void> => {
fastify.get('/main.json', async (request, reply) => traefikConfiguration(request, reply));
fastify.get<TraefikOtherConfiguration>('/other.json', async (request, reply) => traefikOtherConfiguration(request));
fastify.get('/remote/:id', async (request) => remoteTraefikConfiguration(request));
};
export default root;

57
pnpm-lock.yaml
View File

@ -44,7 +44,7 @@ importers:
get-port: 6.1.2
got: 12.1.0
is-ip: 4.0.0
is-port-reachable: ^4.0.0
is-port-reachable: 4.0.0
js-yaml: 4.1.0
jsonwebtoken: 8.5.1
node-forge: 1.3.1
@ -53,6 +53,7 @@ importers:
p-queue: 7.2.0
prettier: 2.7.1
prisma: 3.15.2
public-ip: ^6.0.1
rimraf: 3.0.2
ssh-config: 4.1.6
strip-ansi: 7.0.1
@ -90,6 +91,7 @@ importers:
node-forge: 1.3.1
node-os-utils: 1.3.7
p-queue: 7.2.0
public-ip: 6.0.1
ssh-config: 4.1.6
strip-ansi: 7.0.1
unique-names-generator: 4.7.1
@ -349,6 +351,10 @@ packages:
resolution: {integrity: sha512-hZere0rUga8kTzSTFbHREXpD9E/jwi94+B5RyLAmMIzl/w/EK1z7rFEnMHzPkU4AZkL42JWSsGXoV8LXMihybg==}
dev: false
/@leichtgewicht/ip-codec/2.0.4:
resolution: {integrity: sha512-Hcv+nVC0kZnQ3tD9GVu5xSMR4VVYOteQIr/hwFPVEvPdlXqgGEuRjiheChHgdM+JyqdgNcmzZOX/tnl0JOiI7A==}
dev: false
/@lukeed/ms/2.0.0:
resolution: {integrity: sha512-NOlhE40rGptwLwJhE0ZW259hcoa+nkpQRQ1FUKV4Sr2z1Eh2WfkHQ3jjBNF7YEqOrF0TOpqnyU1wClvWBrXByg==}
engines: {node: '>=8'}
@ -751,6 +757,14 @@ packages:
engines: {node: '>=0.4.0'}
hasBin: true
/aggregate-error/4.0.1:
resolution: {integrity: sha512-0poP0T7el6Vq3rstR8Mn4V/IQrpBLO6POkUSrN7RhyY+GF/InCFShQzsQ39T25gkHhLgSLByyAz+Kjb+c2L98w==}
engines: {node: '>=12'}
dependencies:
clean-stack: 4.2.0
indent-string: 5.0.0
dev: false
/ajv-formats/2.1.1:
resolution: {integrity: sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==}
peerDependenciesMeta:
@ -1827,6 +1841,13 @@ packages:
resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==}
dev: false
/clean-stack/4.2.0:
resolution: {integrity: sha512-LYv6XPxoyODi36Dp976riBtSY27VmFo+MKqEU9QCCWyTrdEPDog+RWA7xQWHi6Vbp61j5c4cdzzX1NidnwtUWg==}
engines: {node: '>=12'}
dependencies:
escape-string-regexp: 5.0.0
dev: false
/clf-date/0.2.0:
resolution: {integrity: sha512-KmV+reIoSINOik5moU6eOqSUy3r/9t6J6Dbl4TCndg1g0R6Z3S3xHzd3u0ZeoTUSbUFr9hHbpiZ+36MrhlNEHQ==}
hasBin: true
@ -2149,6 +2170,20 @@ packages:
resolution: {integrity: sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==}
dev: true
/dns-packet/5.4.0:
resolution: {integrity: sha512-EgqGeaBB8hLiHLZtp/IbaDQTL8pZ0+IvwzSHA6d7VyMDM+B9hgddEMa9xjK5oYnw0ci0JQ6g2XCD7/f6cafU6g==}
engines: {node: '>=6'}
dependencies:
'@leichtgewicht/ip-codec': 2.0.4
dev: false
/dns-socket/4.2.2:
resolution: {integrity: sha512-BDeBd8najI4/lS00HSKpdFia+OvUMytaVjfzR9n5Lq8MlZRSvtbI+uLtx1+XmQFls5wFU9dssccTmQQ6nfpjdg==}
engines: {node: '>=6'}
dependencies:
dns-packet: 5.4.0
dev: false
/docker-modem/3.0.5:
resolution: {integrity: sha512-x1E6jxWdtoK3+ifAUWj4w5egPdTDGBpesSCErm+aKET5BnnEOvDtTP6GxcnMB1zZiv2iQ0qJZvJie+1wfIRg6Q==}
engines: {node: '>= 8.0'}
@ -2521,6 +2556,11 @@ packages:
engines: {node: '>=10'}
dev: true
/escape-string-regexp/5.0.0:
resolution: {integrity: sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==}
engines: {node: '>=12'}
dev: false
/eslint-config-prettier/8.5.0_eslint@8.20.0:
resolution: {integrity: sha512-obmWKLUNCnhtQRKc+tmnYuQl0pFU1ibYJQ5BGhTVB08bHe9wC8qUeG7c08dj9XX+AuPj1YSGSQIHl1pnDHZR0Q==}
hasBin: true
@ -3199,6 +3239,11 @@ packages:
engines: {node: '>=0.8.19'}
dev: true
/indent-string/5.0.0:
resolution: {integrity: sha512-m6FAo/spmsW2Ab2fU35JTYwtOKa2yAwXSwgjSv1TJzh4Mh7mC3lzAOVLBprb72XsTrgkEIsl7YrFNAiDiRhIGg==}
engines: {node: '>=12'}
dev: false
/inflight/1.0.6:
resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==}
dependencies:
@ -4399,6 +4444,16 @@ packages:
resolution: {integrity: sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==}
dev: true
/public-ip/6.0.1:
resolution: {integrity: sha512-1/Mxa1MKrAQ4jF5IalECSBtB0W1FAtnG+9c5X16jjvV/Gx9fiRy7xXIrHlBGYjnTlai0zdZkM3LrpmASavmAEg==}
engines: {node: '>=14.16'}
dependencies:
aggregate-error: 4.0.1
dns-socket: 4.2.2
got: 12.1.0
is-ip: 4.0.0
dev: false
/pump/3.0.0:
resolution: {integrity: sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==}
dependencies: