Shiloh/lasthourcloud
Shiloh/lasthourcloud
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 4 Wiki Activity

Merge pull request #487 from coollabsio/next

Browse Source 
v3.1.1
This commit is contained in:
Andras Bacsai 2022-07-13 15:43:46 +02:00 committed by GitHub
commit d5f9d57be9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 229 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

116
apps/api/src/jobs/cleanupStorage.ts
View File

@ -1,5 +1,5 @@
import { parentPort } from 'node:worker_threads';
import { asyncExecShell, isDev, prisma, version } from '../lib/common';
import { asyncExecShell, cleanupDockerStorage, isDev, prisma, version } from '../lib/common';
import { getEngine } from '../lib/docker';
(async () => {
@ -9,82 +9,52 @@ import { getEngine } from '../lib/docker';
for (const engine of engines) {
let lowDiskSpace = false;
const host = getEngine(engine);
// try {
// let stdout = null
// if (!isDev) {
// const output = await asyncExecShell(
// `DOCKER_HOST=${host} docker exec coolify sh -c 'df -kPT /'`
// );
// stdout = output.stdout;
// } else {
// const output = await asyncExecShell(
// `df -kPT /`
// );
// stdout = output.stdout;
// }
// let lines = stdout.trim().split('\n');
// let header = lines[0];
// let regex =
// /^Filesystem\s+|Type\s+|1024-blocks|\s+Used|\s+Available|\s+Capacity|\s+Mounted on\s*$/g;
// const boundaries = [];
// let match;
// while ((match = regex.exec(header))) {
// boundaries.push(match[0].length);
// }
// boundaries[boundaries.length - 1] = -1;
// const data = lines.slice(1).map((line) => {
// const cl = boundaries.map((boundary) => {
// const column = boundary > 0 ? line.slice(0, boundary) : line;
// line = line.slice(boundary);
// return column.trim();
// });
// return {
// capacity: Number.parseInt(cl[5], 10) / 100
// };
// });
// if (data.length > 0) {
// const { capacity } = data[0];
// if (capacity > 0.6) {
// lowDiskSpace = true;
// }
// }
// } catch (error) {
// console.log(error);
// }
if (!isDev) {
// Cleanup old coolify images
try {
let { stdout: images } = await asyncExecShell(
`DOCKER_HOST=${host} docker images coollabsio/coolify --filter before="coollabsio/coolify:${version}" -q | xargs `
try {
let stdout = null
if (!isDev) {
const output = await asyncExecShell(
`DOCKER_HOST=${host} docker exec coolify sh -c 'df -kPT /'`
);
images = images.trim();
if (images) {
await asyncExecShell(`DOCKER_HOST=${host} docker rmi -f ${images}`);
stdout = output.stdout;
} else {
const output = await asyncExecShell(
`df -kPT /`
);
stdout = output.stdout;
}
let lines = stdout.trim().split('\n');
let header = lines[0];
let regex =
/^Filesystem\s+|Type\s+|1024-blocks|\s+Used|\s+Available|\s+Capacity|\s+Mounted on\s*$/g;
const boundaries = [];
let match;
while ((match = regex.exec(header))) {
boundaries.push(match[0].length);
}
boundaries[boundaries.length - 1] = -1;
const data = lines.slice(1).map((line) => {
const cl = boundaries.map((boundary) => {
const column = boundary > 0 ? line.slice(0, boundary) : line;
line = line.slice(boundary);
return column.trim();
});
return {
capacity: Number.parseInt(cl[5], 10) / 100
};
});
if (data.length > 0) {
const { capacity } = data[0];
if (capacity > 0.8) {
lowDiskSpace = true;
}
} catch (error) {
//console.log(error);
}
try {
await asyncExecShell(`DOCKER_HOST=${host} docker container prune -f`);
} catch (error) {
//console.log(error);
}
try {
await asyncExecShell(`DOCKER_HOST=${host} docker image prune -f`);
} catch (error) {
//console.log(error);
}
try {
await asyncExecShell(`DOCKER_HOST=${host} docker image prune -a -f`);
} catch (error) {
//console.log(error);
}
} else {
console.log(`[DEV MODE] Low disk space: ${lowDiskSpace}`);
} catch (error) {
console.log(error);
}
await cleanupDockerStorage(host, lowDiskSpace, false)
}
await prisma.$disconnect();
await prisma.$disconnect();
} else process.exit(0);
})();

68
apps/api/src/lib/common.ts
View File

@ -14,8 +14,9 @@ import cuid from 'cuid';
import { checkContainer, getEngine, removeContainer } from './docker';
import { day } from './dayjs';
import * as serviceFields from './serviceFields'
import axios from 'axios';
export const version = '3.1.0';
export const version = '3.1.1';
export const isDev = process.env.NODE_ENV === 'development';
const algorithm = 'aes-256-ctr';
@ -30,13 +31,29 @@ export const defaultProxyImage = `coolify-haproxy-alpine:latest`;
export const defaultProxyImageTcp = `coolify-haproxy-tcp-alpine:latest`;
export const defaultProxyImageHttp = `coolify-haproxy-http-alpine:latest`;
export const defaultTraefikImage = `traefik:v2.6`;
export function getAPIUrl() {
if (process.env.GITPOD_WORKSPACE_URL) {
const { href } = new URL(process.env.GITPOD_WORKSPACE_URL)
const newURL = href.replace('https://', 'https://3001-').replace(/\/$/, '')
return newURL
}
return isDev ? 'http://localhost:3001' : 'http://localhost:3000';
}
export function getUIUrl() {
if (process.env.GITPOD_WORKSPACE_URL) {
const { href } = new URL(process.env.GITPOD_WORKSPACE_URL)
const newURL = href.replace('https://', 'https://3000-').replace(/\/$/, '')
return newURL
}
return 'http://localhost:3000';
}
const mainTraefikEndpoint = isDev
? 'http://host.docker.internal:3001/webhooks/traefik/main.json'
? `${getAPIUrl()}/webhooks/traefik/main.json`
: 'http://coolify:3000/webhooks/traefik/main.json';
const otherTraefikEndpoint = isDev
? 'http://host.docker.internal:3001/webhooks/traefik/other.json'
? `${getAPIUrl()}/webhooks/traefik/other.json`
: 'http://coolify:3000/webhooks/traefik/other.json';
@ -1478,4 +1495,49 @@ async function cleanupDB(buildId: string) {
if (data?.status === 'queued' || data?.status === 'running') {
await prisma.build.update({ where: { id: buildId }, data: { status: 'failed' } });
}
}
export function convertTolOldVolumeNames(type) {
if (type === 'nocodb') {
return 'nc'
}
}
export async function getAvailableServices(): Promise<any> {
const { data } = await axios.get(`https://gist.githubusercontent.com/andrasbacsai/4aac36d8d6214dbfc34fa78110554a50/raw/291a957ee6ac01d480465623e183a30230ad921f/availableServices.json`)
return data
}
export async function cleanupDockerStorage(host, lowDiskSpace, force) {
// Cleanup old coolify images
try {
let { stdout: images } = await asyncExecShell(
`DOCKER_HOST=${host} docker images coollabsio/coolify --filter before="coollabsio/coolify:${version}" -q | xargs `
);
images = images.trim();
if (images) {
await asyncExecShell(`DOCKER_HOST=${host} docker rmi -f ${images}`);
}
} catch (error) {
//console.log(error);
}
if (lowDiskSpace || force) {
if (isDev) {
if (!force) console.log(`[DEV MODE] Low disk space: ${lowDiskSpace}`);
return
}
try {
await asyncExecShell(`DOCKER_HOST=${host} docker container prune -f`);
} catch (error) {
//console.log(error);
}
try {
await asyncExecShell(`DOCKER_HOST=${host} docker image prune -f`);
} catch (error) {
//console.log(error);
}
try {
await asyncExecShell(`DOCKER_HOST=${host} docker image prune -a -f`);
} catch (error) {
//console.log(error);
}
}
}

11
apps/api/src/routes/api/v1/handlers.ts
View File

@ -4,7 +4,7 @@ import axios from 'axios';
import compare from 'compare-versions';
import cuid from 'cuid';
import bcrypt from 'bcryptjs';
import { asyncExecShell, asyncSleep, errorHandler, isDev, prisma, uniqueName, version } from '../../../lib/common';
import { asyncExecShell, asyncSleep, cleanupDockerStorage, errorHandler, isDev, prisma, uniqueName, version } from '../../../lib/common';
import type { FastifyReply, FastifyRequest } from 'fastify';
import type { Login, Update } from '.';
@ -15,7 +15,14 @@ export async function hashPassword(password: string): Promise<string> {
return bcrypt.hash(password, saltRounds);
}
export async function cleanupManually() {
try {
await cleanupDockerStorage('unix:///var/run/docker.sock', true, true)
return {}
} catch ({ status, message }) {
return errorHandler({ status, message })
}
}
export async function checkUpdate(request: FastifyRequest) {
try {
const currentVersion = version;

2
apps/api/src/routes/api/v1/iam/handlers.ts
View File

@ -445,7 +445,7 @@ export async function setPermission(request: FastifyRequest, reply: FastifyReply
export async function changePassword(request: FastifyRequest, reply: FastifyReply) {
try {
const { id } = request.body;
await prisma.user.update({ where: { id: undefined }, data: { password: 'RESETME' } });
await prisma.user.update({ where: { id }, data: { password: 'RESETME' } });
return reply.code(201).send()
} catch ({ status, message }) {
return errorHandler({ status, message })

6
apps/api/src/routes/api/v1/index.ts
View File

@ -1,6 +1,6 @@
import { FastifyPluginAsync } from 'fastify';
import { scheduler } from '../../../lib/scheduler';
import { checkUpdate, login, showDashboard, update, showUsage, getCurrentUser } from './handlers';
import { checkUpdate, login, showDashboard, update, showUsage, getCurrentUser, cleanupManually } from './handlers';
export interface Update {
Body: { latestVersion: string }
@ -46,6 +46,10 @@ const root: FastifyPluginAsync = async (fastify, opts): Promise<void> => {
fastify.get('/usage', {
onRequest: [fastify.authenticate]
}, async () => await showUsage());
fastify.post('/internal/cleanup', {
onRequest: [fastify.authenticate]
}, async () => await cleanupManually());
};
export default root;

66
apps/api/src/routes/api/v1/services/handlers.ts
View File

@ -2,11 +2,75 @@ import type { FastifyReply, FastifyRequest } from 'fastify';
import fs from 'fs/promises';
import yaml from 'js-yaml';
import bcrypt from 'bcryptjs';
import { prisma, uniqueName, asyncExecShell, getServiceImage, getServiceImages, configureServiceType, getServiceFromDB, getContainerUsage, removeService, isDomainConfigured, saveUpdateableFields, fixType, decrypt, encrypt, getServiceMainPort, createDirectories, ComposeFile, makeLabelForServices, getFreePort, getDomain, errorHandler, supportedServiceTypesAndVersions, generatePassword, isDev, stopTcpHttpProxy } from '../../../../lib/common';
import { prisma, uniqueName, asyncExecShell, getServiceImage, getServiceImages, configureServiceType, getServiceFromDB, getContainerUsage, removeService, isDomainConfigured, saveUpdateableFields, fixType, decrypt, encrypt, getServiceMainPort, createDirectories, ComposeFile, makeLabelForServices, getFreePort, getDomain, errorHandler, supportedServiceTypesAndVersions, generatePassword, isDev, stopTcpHttpProxy, getAvailableServices } from '../../../../lib/common';
import { day } from '../../../../lib/dayjs';
import { checkContainer, dockerInstance, getEngine, removeContainer } from '../../../../lib/docker';
import cuid from 'cuid';
async function startServiceNew(request: FastifyRequest) {
try {
const { id } = request.params;
const teamId = request.user.teamId;
const service = await getServiceFromDB({ id, teamId });
const { type, version, destinationDockerId, destinationDocker, serviceSecret, exposePort } =
service;
const network = destinationDockerId && destinationDocker.network;
const host = getEngine(destinationDocker.engine);
const port = getServiceMainPort(type);
const { workdir } = await createDirectories({ repository: type, buildId: id });
const image = getServiceImage(type);
const config = (await getAvailableServices()).find((name) => name.name === type).compose
const environmentVariables = {}
if (serviceSecret.length > 0) {
serviceSecret.forEach((secret) => {
environmentVariables[secret.name] = secret.value;
});
}
config.services[id] = JSON.parse(JSON.stringify(config.services[type]))
config.services[id].container_name = id
config.services[id].image = `${image}:${version}`
config.services[id].ports = (exposePort ? [`${exposePort}:${port}`] : []),
config.services[id].restart = "always"
config.services[id].networks = [network]
config.services[id].labels = makeLabelForServices(type)
config.services[id].deploy = {
restart_policy: {
condition: 'on-failure',
delay: '5s',
max_attempts: 3,
window: '120s'
}
}
config.networks = {
[network]: {
external: true
}
}
config.volumes = {}
config.services[id].volumes.forEach((volume, index) => {
let oldVolumeName = volume.split(':')[0]
const path = volume.split(':')[1]
oldVolumeName = convertTolOldVolumeNames(type)
const volumeName = `${id}-${oldVolumeName}`
config.volumes[volumeName] = {
name: volumeName
}
config.services[id].volumes[index] = `${volumeName}:${path}`
})
delete config.services[type]
config.services[id].environment = environmentVariables
const composeFileDestination = `${workdir}/docker-compose.yaml`;
await fs.writeFile(composeFileDestination, yaml.dump(config));
await asyncExecShell(`DOCKER_HOST=${host} docker compose -f ${composeFileDestination} pull`);
await asyncExecShell(`DOCKER_HOST=${host} docker compose -f ${composeFileDestination} up -d`);
return {}
} catch ({ status, message }) {
return errorHandler({ status, message })
}
}
export async function listServices(request: FastifyRequest) {
try {
const userId = request.user.userId;

6
apps/api/src/routes/webhooks/github/handlers.ts
View File

@ -2,7 +2,7 @@ import axios from "axios";
import cuid from "cuid";
import crypto from "crypto";
import type { FastifyReply, FastifyRequest } from "fastify";
import { encrypt, errorHandler, isDev, prisma } from "../../../lib/common";
import { encrypt, errorHandler, getAPIUrl, getUIUrl, isDev, prisma } from "../../../lib/common";
import { checkContainer, removeContainer } from "../../../lib/docker";
import { scheduler } from "../../../lib/scheduler";
import { getApplicationFromDB, getApplicationFromDBWebhook } from "../../api/v1/applications/handlers";
@ -19,7 +19,7 @@ export async function installGithub(request: FastifyRequest, reply: FastifyReply
data: { installationId: Number(installation_id) }
});
if (isDev) {
return reply.redirect(`http://localhost:3000/sources/${gitSourceId}`)
return reply.redirect(`${getUIUrl()}/sources/${gitSourceId}`)
} else {
return reply.redirect(`/sources/${gitSourceId}`)
}
@ -55,7 +55,7 @@ export async function configureGitHubApp(request, reply) {
}
});
if (isDev) {
return reply.redirect(`http://localhost:3000/sources/${state}`)
return reply.redirect(`${getUIUrl()}/sources/${state}`)
} else {
return reply.redirect(`/sources/${state}`)
}

6
apps/api/src/routes/webhooks/gitlab/handlers.ts
View File

@ -2,7 +2,7 @@ import axios from "axios";
import cuid from "cuid";
import crypto from "crypto";
import type { FastifyReply, FastifyRequest } from "fastify";
import { encrypt, errorHandler, isDev, listSettings, prisma } from "../../../lib/common";
import { encrypt, errorHandler, getAPIUrl, isDev, listSettings, prisma } from "../../../lib/common";
import { checkContainer, removeContainer } from "../../../lib/docker";
import { scheduler } from "../../../lib/scheduler";
import { getApplicationFromDB, getApplicationFromDBWebhook } from "../../api/v1/applications/handlers";
@ -16,7 +16,7 @@ export async function configureGitLabApp(request: FastifyRequest, reply: Fastify
let domain = `http://${request.hostname}`;
if (fqdn) domain = fqdn;
if (isDev) {
domain = `http://localhost:3001`;
domain = getAPIUrl();
}
const params = new URLSearchParams({
client_id: appId,
@ -28,7 +28,7 @@ export async function configureGitLabApp(request: FastifyRequest, reply: Fastify
});
const { data } = await axios.post(`${htmlUrl}/oauth/token`, params)
if (isDev) {
return reply.redirect(`http://localhost:3000/webhooks/success?token=${data.access_token}`)
return reply.redirect(`${getAPIUrl()}/webhooks/success?token=${data.access_token}`)
}
return reply.redirect(`/webhooks/success?token=${data.access_token}`)
} catch ({ status, message, ...other }) {

20
apps/ui/src/lib/api.ts
View File

@ -1,14 +1,28 @@
import { browser, dev } from '$app/env';
import { dev } from '$app/env';
import Cookies from 'js-cookie';
export function getAPIUrl() {
if (GITPOD_WORKSPACE_URL) {
const {href} = new URL(GITPOD_WORKSPACE_URL)
const newURL = href.replace('https://','https://3001-').replace(/\/$/,'')
const { href } = new URL(GITPOD_WORKSPACE_URL)
const newURL = href.replace('https://', 'https://3001-').replace(/\/$/, '')
return newURL
}
return dev ? 'http://localhost:3001' : 'http://localhost:3000';
}
export function getWebhookUrl(type: string) {
console.log(GITPOD_WORKSPACE_URL)
if (GITPOD_WORKSPACE_URL) {
const { href } = new URL(GITPOD_WORKSPACE_URL)
const newURL = href.replace('https://', 'https://3001-').replace(/\/$/, '')
if (type === 'github') {
return `${newURL}/webhooks/github/events`
}
if (type === 'gitlab') {
return `${newURL}/webhooks/gitlab/events`
}
}
return `https://webhook.site/0e5beb2c-4e9b-40e2-a89e-32295e570c21/events`;
}
async function send({
method,
path,

9
apps/ui/src/lib/components/Usage.svelte
View File

@ -23,7 +23,7 @@
};
import { appSession } from '$lib/store';
import { onDestroy, onMount } from 'svelte';
import { get } from '$lib/api';
import { get, post } from '$lib/api';
import { errorNotification } from '$lib/common';
import Trend from './Trend.svelte';
async function getStatus() {
@ -59,6 +59,9 @@
cpu: 'stable',
disk: 'stable'
};
async function manuallyCleanupStorage() {
return await post('/internal/cleanup', {});
}
</script>
{#if $appSession.teamId === '0'}
@ -129,6 +132,9 @@
<dd class="mt-1 text-3xl font-semibold text-white">
{usage?.disk.usedGb}<span class="text-sm">GB</span>
</dd>
<button on:click={manuallyCleanupStorage} class="bg-coollabs hover:bg-coollabs-100"
>Cleanup Storage</button
>
</div>
<div
class="overflow-hidden rounded px-4 py-5 text-center sm:p-6 sm:text-left"
@ -143,5 +149,6 @@
</dd>
</div>
</dl>
<div class="px-6 pt-20 text-2xl font-bold">Resources</div>
{/if}

6
apps/ui/src/routes/applications/[id]/configuration/_GitlabRepositories.svelte
View File

@ -8,7 +8,7 @@
import { onMount } from 'svelte';
import { dev } from '$app/env';
import { goto } from '$app/navigation';
import { del, get, post } from '$lib/api';
import { del, get, getAPIUrl, getWebhookUrl, post } from '$lib/api';
import { t } from '$lib/translations';
import { errorNotification } from '$lib/common';
import { appSession } from '$lib/store';
@ -18,7 +18,7 @@
const from = $page.url.searchParams.get('from');
let url = settings?.fqdn ? settings.fqdn : window.location.origin;
if (dev) url = `http://localhost:3001`;
if (dev) url = getAPIUrl();
const updateDeployKeyIdUrl = `/applications/${id}/configuration/deploykey`;
@ -228,7 +228,7 @@
}
async function setWebhook(url: any, webhookToken: any) {
const host = dev
? 'https://webhook.site/0e5beb2c-4e9b-40e2-a89e-32295e570c21'
? getWebhookUrl('gitlab')
: `${window.location.origin}/webhooks/gitlab/events`;
try {
await post(

6
apps/ui/src/routes/sources/[id]/_Github.svelte
View File

@ -2,7 +2,7 @@
export let source: any;
export let settings: any;
import { page } from '$app/stores';
import { post } from '$lib/api';
import { getAPIUrl, getWebhookUrl, post } from '$lib/api';
import Explainer from '$lib/components/Explainer.svelte';
import { toast } from '@zerodevx/svelte-toast';
import { t } from '$lib/translations';
@ -67,7 +67,7 @@
const { organization, htmlUrl } = source;
const { fqdn } = settings;
const host = dev
? 'http://localhost:3001'
? getAPIUrl()
: fqdn
? fqdn
: `http://${window.location.host}` || '';
@ -81,7 +81,7 @@
url: host,
hook_attributes: {
url: dev
? 'https://webhook.site/0e5beb2c-4e9b-40e2-a89e-32295e570c21/events'
? getWebhookUrl('github')
: `${host}/webhooks/github/events`
},
redirect_url: `${host}/webhooks/github`,

4
apps/ui/src/routes/sources/[id]/_Gitlab.svelte
View File

@ -4,7 +4,7 @@
import Explainer from '$lib/components/Explainer.svelte';
import { page } from '$app/stores';
import { onMount } from 'svelte';
import { post } from '$lib/api';
import { getAPIUrl, post } from '$lib/api';
import { dev } from '$app/env';
import CopyPasswordField from '$lib/components/CopyPasswordField.svelte';
import { toast } from '@zerodevx/svelte-toast';
@ -17,7 +17,7 @@
let url = settings.fqdn ? settings.fqdn : window.location.origin;
if (dev) {
url = `http://localhost:3001`;
url = getAPIUrl();
}
let loading = false;

3
package.json
View File

@ -1,7 +1,7 @@
{
"name": "coolify",
"description": "An open-source & self-hostable Heroku / Netlify alternative.",
"version": "3.1.0",
"version": "3.1.1",
"license": "AGPL-3.0",
"scripts": {
"db:studio": "pnpm run --filter coolify-api db:studio",
@ -18,6 +18,7 @@
"build": "NODE_ENV=production run-p -n build:*",
"build:api": "NODE_ENV=production pnpm run --filter coolify-api build",
"build:ui": "NODE_ENV=production pnpm run --filter coolify-ui build",
"dockerlogin":"echo $DOCKER_PASS | docker login --username=$DOCKER_USER --password-stdin",
"release:staging:amd": "cross-var docker buildx build --platform linux/amd64 -t coollabsio/coolify:$npm_package_version --push .",
"release:local":"rm -fr ./local-serve && mkdir ./local-serve && pnpm build && cp -Rp apps/api/build/* ./local-serve && cp -Rp apps/ui/build/ ./local-serve/public && cp -Rp apps/api/prisma/ ./local-serve/prisma && cp -Rp apps/api/package.json ./local-serve && cp .env ./local-serve && cd ./local-serve && pnpm install . && pnpm start"
},