From 4416646954fb4777a97edf2f55e11a42aedaa19a Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 26 Oct 2022 15:50:10 +0200 Subject: [PATCH] package updates + tags selector --- .gitignore | 1 + apps/api/package.json | 32 +- ...nvert.mjs => convertCaproverTemplates.mjs} | 0 apps/api/scripts/generateTags.mjs | 67 ++ apps/api/src/index.ts | 21 +- apps/api/src/lib/services.ts | 9 + apps/api/src/routes/api/v1/handlers.ts | 28 +- .../src/routes/api/v1/services/handlers.ts | 20 +- apps/api/src/routes/api/v1/services/types.ts | 2 + apps/ui/package.json | 38 +- .../src/routes/services/[id]/__layout.svelte | 5 +- apps/ui/src/routes/services/[id]/index.svelte | 36 +- apps/ui/src/routes/services/[id]/utils.ts | 5 +- pnpm-lock.yaml | 1037 ++++++----------- 14 files changed, 571 insertions(+), 730 deletions(-) rename apps/api/scripts/{convert.mjs => convertCaproverTemplates.mjs} (100%) create mode 100644 apps/api/scripts/generateTags.mjs diff --git a/.gitignore b/.gitignore index 2a8f1da22..0ef2f5e13 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ apps/api/core* logs others/certificates apps/api/template.json +apps/api/tags.json diff --git a/apps/api/package.json b/apps/api/package.json index 49d5f34be..67634775c 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -3,7 +3,7 @@ "description": "Coolify's Fastify API", "license": "Apache-2.0", "scripts": { - "db:generate":"prisma generate", + "db:generate": "prisma generate", "db:push": "prisma db push && prisma generate", "db:seed": "prisma db seed", "db:studio": "prisma studio", @@ -16,18 +16,18 @@ }, "dependencies": { "@breejs/ts-worker": "2.0.0", - "@fastify/autoload": "5.4.0", + "@fastify/autoload": "5.4.1", "@fastify/cookie": "8.3.0", - "@fastify/cors": "8.1.0", + "@fastify/cors": "8.1.1", "@fastify/env": "4.1.0", "@fastify/jwt": "6.3.2", - "@fastify/multipart": "7.2.0", + "@fastify/multipart": "7.3.0", "@fastify/static": "6.5.0", "@iarna/toml": "2.2.5", "@ladjs/graceful": "3.0.2", - "@prisma/client": "4.4.0", - "prisma": "4.4.0", - "axios": "0.27.2", + "@prisma/client": "4.5.0", + "prisma": "4.5.0", + "axios": "1.1.3", "bcryptjs": "2.4.3", "bree": "9.1.2", "cabin": "9.1.2", @@ -35,12 +35,12 @@ "csv-parse": "5.3.1", "csvtojson": "2.0.10", "cuid": "2.1.8", - "dayjs": "1.11.5", + "dayjs": "1.11.6", "dockerode": "3.3.4", "dotenv-extended": "2.9.0", "execa": "6.1.0", - "fastify": "4.8.1", - "fastify-plugin": "4.2.1", + "fastify": "4.9.2", + "fastify-plugin": "4.3.0", "generate-password": "1.7.0", "got": "12.5.2", "is-ip": "5.0.0", @@ -58,17 +58,17 @@ "unique-names-generator": "4.7.1" }, "devDependencies": { - "@types/node": "18.8.5", + "semver-sort": "1.0.0", + "@types/node": "18.11.6", "@types/node-os-utils": "1.3.0", - "@typescript-eslint/eslint-plugin": "5.38.1", - "@typescript-eslint/parser": "5.38.1", - "esbuild": "0.15.10", - "eslint": "8.25.0", + "@typescript-eslint/eslint-plugin": "5.41.0", + "@typescript-eslint/parser": "5.41.0", + "esbuild": "0.15.12", + "eslint": "8.26.0", "eslint-config-prettier": "8.5.0", "eslint-plugin-prettier": "4.2.1", "nodemon": "2.0.20", "prettier": "2.7.1", - "rimraf": "3.0.2", "tsconfig-paths": "4.1.0", "typescript": "4.8.4" diff --git a/apps/api/scripts/convert.mjs b/apps/api/scripts/convertCaproverTemplates.mjs similarity index 100% rename from apps/api/scripts/convert.mjs rename to apps/api/scripts/convertCaproverTemplates.mjs diff --git a/apps/api/scripts/generateTags.mjs b/apps/api/scripts/generateTags.mjs new file mode 100644 index 000000000..e74d3fca0 --- /dev/null +++ b/apps/api/scripts/generateTags.mjs @@ -0,0 +1,67 @@ +import fs from 'fs/promises'; +import yaml from 'js-yaml'; +import got from 'got'; +import semverSort from 'semver-sort'; + +const repositories = []; +const templates = await fs.readFile('../devTemplates.yaml', 'utf8'); +const devTemplates = yaml.load(templates); +for (const template of devTemplates) { + let image = template.services['$$id'].image.replaceAll(':$$core_version', ''); + const name = template.name + if (!image.includes('/')) { + image = `library/${image}`; + } + repositories.push({ image, name: name.toLowerCase().replaceAll(' ', '') }); +} +const services = [] + +const numberOfTags = 30; +// const semverRegex = new RegExp(/^v?(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/g) +const semverRegex = new RegExp(/^v?(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/g) +for (const repository of repositories) { + console.log('Querying', repository.name, 'at', repository.image); + if (repository.image.includes('ghcr.io')) { + const { execaCommand } = await import('execa'); + const { stdout } = await execaCommand(`docker run --rm quay.io/skopeo/stable list-tags docker://${repository.image}`); + if (stdout) { + const json = JSON.parse(stdout); + const semverTags = json.Tags.filter((tag) => semverRegex.test(tag)) + let tags = semverTags.length > 10 ? semverTags.sort().reverse().slice(0, numberOfTags) : json.Tags.sort().reverse().slice(0, numberOfTags) + if (!tags.includes('latest')) { + tags.push('latest') + } + try { + tags = semverSort.desc(tags) + } catch (error) { } + services.push({ name: repository.name, image: repository.image, tags }) + } + } else { + const { token } = await got.get(`https://auth.docker.io/token?service=registry.docker.io&scope=repository:${repository.image}:pull`).json() + let data = await got.get(`https://registry-1.docker.io/v2/${repository.image}/tags/list`, { + headers: { + Authorization: `Bearer ${token}` + } + }).json() + const semverTags = data.tags.filter((tag) => semverRegex.test(tag)) + let tags = semverTags.length > 10 ? semverTags.sort().reverse().slice(0, numberOfTags) : data.tags.sort().reverse().slice(0, numberOfTags) + if (!tags.includes('latest')) { + tags.push('latest') + } + try { + tags = semverSort.desc(tags) + } catch (error) { } + + console.log({ + name: repository.name, + image: repository.image, + tags + }) + services.push({ + name: repository.name, + image: repository.image, + tags + }) + } +} +await fs.writeFile('./tags.json', JSON.stringify(services)); \ No newline at end of file diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 99c33d36b..40b5e7dfb 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -17,7 +17,7 @@ import { verifyRemoteDockerEngineFn } from './routes/api/v1/destinations/handler import { checkContainer } from './lib/docker'; import { migrateServicesToNewTemplate } from './lib'; import { getTemplates } from './lib/services'; -import { refreshTemplates } from './routes/api/v1/handlers'; +import { refreshTags, refreshTemplates } from './routes/api/v1/handlers'; declare module 'fastify' { interface FastifyInstance { config: { @@ -131,12 +131,17 @@ const host = '0.0.0.0'; try { const { default: got } = await import('got') try { + const tags = await got.get('https://get.coollabs.io/coolify/service-tags.json').text() + if (isDev) { - const response = await fs.readFile('./devTemplates.yaml', 'utf8') - await fs.writeFile('./template.json', JSON.stringify(yaml.load(response), null, 2)) + const templates = await fs.readFile('./devTemplates.yaml', 'utf8') + await fs.writeFile('./template.json', JSON.stringify(yaml.load(templates))) + const tags = await got.get('https://get.coollabs.io/coolify/service-tags.json').text() + await fs.writeFile('./tags.json', tags) } else { const response = await got.get('https://get.coollabs.io/coolify/service-templates.yaml').text() - await fs.writeFile('/app/template.json', JSON.stringify(yaml.load(response), null, 2)) + await fs.writeFile('/app/template.json', JSON.stringify(yaml.load(response))) + await fs.writeFile('/app/tags.json', tags) } } catch (error) { @@ -173,18 +178,18 @@ const host = '0.0.0.0'; setInterval(async () => { await checkProxies(); await checkFluentBit(); - - }, 10000) + }, 60000) // Refresh and check templates setInterval(async () => { await refreshTemplates() + await refreshTags() await migrateServicesToNewTemplate() - }, 10000) + }, 60000) setInterval(async () => { await copySSLCertificates(); - }, 2000) + }, 10000) await Promise.all([ getArch(), diff --git a/apps/api/src/lib/services.ts b/apps/api/src/lib/services.ts index 5461a8355..8538c8028 100644 --- a/apps/api/src/lib/services.ts +++ b/apps/api/src/lib/services.ts @@ -141,3 +141,12 @@ export async function getTemplates() { // } return templates } +export async function getTags(type?: string) { + let tags: any = []; + if (isDev) { + tags = JSON.parse(await (await fs.readFile('./tags.json')).toString()) + } else { + tags = JSON.parse(await (await fs.readFile('/app/tags.json')).toString()) + } + return tags.find((tag: any) => tag.name.includes(type)) +} diff --git a/apps/api/src/routes/api/v1/handlers.ts b/apps/api/src/routes/api/v1/handlers.ts index 8645e3856..2a59f3b24 100644 --- a/apps/api/src/routes/api/v1/handlers.ts +++ b/apps/api/src/routes/api/v1/handlers.ts @@ -37,18 +37,42 @@ export async function cleanupManually(request: FastifyRequest) { return errorHandler({ status, message }); } } +export async function refreshTags() { + try { + const { default: got } = await import('got') + try { + const tags = await got.get('https://get.coollabs.io/coolify/service-tags.json').text() + if (isDev) { + await fs.writeFile('./tags.json', tags) + } else { + await fs.writeFile('/app/tags.json', tags) + } + } catch (error) { + console.log(error) + throw { + status: 500, + message: 'Could not fetch templates from get.coollabs.io' + }; + } + + return {}; + } catch ({ status, message }) { + return errorHandler({ status, message }); + } +} export async function refreshTemplates() { try { const { default: got } = await import('got') try { if (isDev) { const response = await fs.readFile('./devTemplates.yaml', 'utf8') - await fs.writeFile('./template.json', JSON.stringify(yaml.load(response), null, 2)) + await fs.writeFile('./template.json', JSON.stringify(yaml.load(response))) } else { const response = await got.get('https://get.coollabs.io/coolify/service-templates.yaml').text() - await fs.writeFile('/app/template.json', JSON.stringify(yaml.load(response), null, 2)) + await fs.writeFile('/app/template.json', JSON.stringify(yaml.load(response))) } } catch (error) { + console.log(error) throw { status: 500, message: 'Could not fetch templates from get.coollabs.io' diff --git a/apps/api/src/routes/api/v1/services/handlers.ts b/apps/api/src/routes/api/v1/services/handlers.ts index 9721af81c..b4f43ea55 100644 --- a/apps/api/src/routes/api/v1/services/handlers.ts +++ b/apps/api/src/routes/api/v1/services/handlers.ts @@ -2,17 +2,16 @@ import type { FastifyReply, FastifyRequest } from 'fastify'; import fs from 'fs/promises'; import yaml from 'js-yaml'; import bcrypt from 'bcryptjs'; -import crypto from 'crypto'; -import { prisma, uniqueName, asyncExecShell, getServiceFromDB, getContainerUsage, isDomainConfigured, saveUpdateableFields, fixType, decrypt, encrypt, ComposeFile, getFreePublicPort, getDomain, errorHandler, generatePassword, isDev, stopTcpHttpProxy, executeDockerCmd, checkDomainsIsValidInDNS, checkExposedPort, listSettings } from '../../../../lib/common'; -import { day } from '../../../../lib/dayjs'; -import { checkContainer, isContainerExited } from '../../../../lib/docker'; import cuid from 'cuid'; -import type { OnlyId } from '../../../../types'; +import { prisma, uniqueName, asyncExecShell, getServiceFromDB, getContainerUsage, isDomainConfigured, saveUpdateableFields, fixType, decrypt, encrypt, ComposeFile, getFreePublicPort, getDomain, errorHandler, generatePassword, isDev, stopTcpHttpProxy, executeDockerCmd, checkDomainsIsValidInDNS, checkExposedPort, listSettings } from '../../../../lib/common'; +import { day } from '../../../../lib/dayjs'; +import { checkContainer, } from '../../../../lib/docker'; +import { removeService } from '../../../../lib/services/common'; +import { getTags, getTemplates } from '../../../../lib/services'; + import type { ActivateWordpressFtp, CheckService, CheckServiceDomain, DeleteServiceSecret, DeleteServiceStorage, GetServiceLogs, SaveService, SaveServiceDestination, SaveServiceSecret, SaveServiceSettings, SaveServiceStorage, SaveServiceType, SaveServiceVersion, ServiceStartStop, SetGlitchTipSettings, SetWordpressSettings } from './types'; -import { configureServiceType, removeService } from '../../../../lib/services/common'; -import { hashPassword } from '../handlers'; -import { getTemplates } from '../../../../lib/services'; +import type { OnlyId } from '../../../../types'; export async function listServices(request: FastifyRequest) { try { @@ -224,10 +223,12 @@ export async function getService(request: FastifyRequest) { if (service.type) { template = await parseAndFindServiceTemplates(service) } + const tags = await getTags(service.type) return { settings: await listSettings(), service, template, + tags } } catch ({ status, message }) { console.log(status, message) @@ -470,7 +471,7 @@ export async function checkService(request: FastifyRequest) { export async function saveService(request: FastifyRequest, reply: FastifyReply) { try { const { id } = request.params; - let { name, fqdn, exposePort, type, serviceSetting } = request.body; + let { name, fqdn, exposePort, type, serviceSetting, version } = request.body; if (fqdn) fqdn = fqdn.toLowerCase(); if (exposePort) exposePort = Number(exposePort); type = fixType(type) @@ -479,6 +480,7 @@ export async function saveService(request: FastifyRequest, reply: F fqdn, name, exposePort, + version, } const templates = await getTemplates() const service = await prisma.service.findUnique({ where: { id } }) diff --git a/apps/api/src/routes/api/v1/services/types.ts b/apps/api/src/routes/api/v1/services/types.ts index f2ecbed8f..5839724c8 100644 --- a/apps/api/src/routes/api/v1/services/types.ts +++ b/apps/api/src/routes/api/v1/services/types.ts @@ -48,6 +48,8 @@ export interface SaveService extends OnlyId { name: string, fqdn: string, exposePort: number, + version: string, + serviceSetting: any type: string } } diff --git a/apps/ui/package.json b/apps/ui/package.json index fdf37b8aa..63a429dba 100644 --- a/apps/ui/package.json +++ b/apps/ui/package.json @@ -14,39 +14,39 @@ "format": "prettier --write --plugin-search-dir=. ." }, "devDependencies": { - "@floating-ui/dom": "1.0.1", - "@playwright/test": "1.25.1", + "@floating-ui/dom": "1.0.3", + "@playwright/test": "1.27.1", "@popperjs/core": "2.11.6", "@sveltejs/kit": "1.0.0-next.405", "@types/js-cookie": "3.0.2", - "@typescript-eslint/eslint-plugin": "5.36.1", - "@typescript-eslint/parser": "5.36.1", - "autoprefixer": "10.4.8", - "classnames": "2.3.1", - "eslint": "8.23.0", + "@typescript-eslint/eslint-plugin": "5.41.0", + "@typescript-eslint/parser": "5.41.0", + "autoprefixer": "10.4.12", + "classnames": "2.3.2", + "eslint": "8.26.0", "eslint-config-prettier": "8.5.0", "eslint-plugin-svelte3": "4.0.0", - "flowbite": "1.5.2", - "flowbite-svelte": "0.26.2", - "postcss": "8.4.16", + "flowbite": "1.5.3", + "flowbite-svelte": "0.27.11", + "postcss": "8.4.18", "prettier": "2.7.1", - "prettier-plugin-svelte": "2.7.0", - "svelte": "3.50.0", - "svelte-check": "2.9.0", + "prettier-plugin-svelte": "2.8.0", + "svelte": "3.52.0", + "svelte-check": "2.9.2", "svelte-preprocess": "4.10.7", - "tailwindcss": "3.1.8", + "tailwindcss": "3.2.1", "tailwindcss-scrollbar": "0.1.0", "tslib": "2.4.0", - "typescript": "4.8.2", - "vite": "3.1.0" + "typescript": "4.8.4", + "vite": "3.2.0" }, "type": "module", "dependencies": { - "@sveltejs/adapter-static": "1.0.0-next.39", + "@sveltejs/adapter-static": "1.0.0-next.46", "@tailwindcss/typography": "^0.5.7", "cuid": "2.1.8", - "daisyui": "2.24.2", - "dayjs": "1.11.5", + "daisyui": "2.33.0", + "dayjs": "1.11.6", "js-cookie": "3.0.1", "js-yaml": "4.1.0", "p-limit": "4.0.0", diff --git a/apps/ui/src/routes/services/[id]/__layout.svelte b/apps/ui/src/routes/services/[id]/__layout.svelte index 337b56d60..6c4ed5620 100644 --- a/apps/ui/src/routes/services/[id]/__layout.svelte +++ b/apps/ui/src/routes/services/[id]/__layout.svelte @@ -13,7 +13,7 @@ try { let readOnly = false; const response = await get(`/services/${params.id}`); - const { service, settings, template } = await response; + const { service, settings, template, tags } = await response; if (!service || Object.entries(service).length === 0) { return { status: 302, @@ -43,7 +43,8 @@ service, template, readOnly, - settings + settings, + tags } }; } catch (error) { diff --git a/apps/ui/src/routes/services/[id]/index.svelte b/apps/ui/src/routes/services/[id]/index.svelte index 838b87492..69a066c54 100644 --- a/apps/ui/src/routes/services/[id]/index.svelte +++ b/apps/ui/src/routes/services/[id]/index.svelte @@ -10,6 +10,7 @@