From a6f457749b8aaff4bfe425095432c847355925b9 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 18 Oct 2022 11:32:38 +0200 Subject: [PATCH] lots of changes --- .../migration.sql | 19 ++++ apps/api/prisma/schema.prisma | 29 +++--- apps/api/src/lib/services/handlers.ts | 20 ++-- apps/api/src/lib/templates.ts | 41 +++++--- .../src/routes/api/v1/services/handlers.ts | 23 +++-- .../src/routes/webhooks/traefik/handlers.ts | 10 +- apps/ui/src/lib/components/Setting.svelte | 2 +- .../routes/applications/[id]/_Storage.svelte | 23 ++++- .../routes/applications/[id]/storages.svelte | 2 +- .../src/routes/services/[id]/_Storage.svelte | 97 +++++++++++++------ .../src/routes/services/[id]/__layout.svelte | 42 ++++---- apps/ui/src/routes/services/[id]/index.svelte | 49 +++++++--- .../src/routes/services/[id]/storages.svelte | 12 ++- 13 files changed, 256 insertions(+), 113 deletions(-) create mode 100644 apps/api/prisma/migrations/20221018090939_service_peristent_volumes_predefined/migration.sql diff --git a/apps/api/prisma/migrations/20221018090939_service_peristent_volumes_predefined/migration.sql b/apps/api/prisma/migrations/20221018090939_service_peristent_volumes_predefined/migration.sql new file mode 100644 index 000000000..c7b935eb7 --- /dev/null +++ b/apps/api/prisma/migrations/20221018090939_service_peristent_volumes_predefined/migration.sql @@ -0,0 +1,19 @@ +-- RedefineTables +PRAGMA foreign_keys=OFF; +CREATE TABLE "new_ServicePersistentStorage" ( + "id" TEXT NOT NULL PRIMARY KEY, + "serviceId" TEXT NOT NULL, + "path" TEXT NOT NULL, + "volumeName" TEXT, + "predefined" BOOLEAN NOT NULL DEFAULT false, + "containerId" TEXT, + "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" DATETIME NOT NULL, + CONSTRAINT "ServicePersistentStorage_serviceId_fkey" FOREIGN KEY ("serviceId") REFERENCES "Service" ("id") ON DELETE RESTRICT ON UPDATE CASCADE +); +INSERT INTO "new_ServicePersistentStorage" ("createdAt", "id", "path", "serviceId", "updatedAt") SELECT "createdAt", "id", "path", "serviceId", "updatedAt" FROM "ServicePersistentStorage"; +DROP TABLE "ServicePersistentStorage"; +ALTER TABLE "new_ServicePersistentStorage" RENAME TO "ServicePersistentStorage"; +CREATE UNIQUE INDEX "ServicePersistentStorage_serviceId_path_key" ON "ServicePersistentStorage"("serviceId", "path"); +PRAGMA foreign_key_check; +PRAGMA foreign_keys=ON; diff --git a/apps/api/prisma/schema.prisma b/apps/api/prisma/schema.prisma index 28e2d9a3b..dcb20d876 100644 --- a/apps/api/prisma/schema.prisma +++ b/apps/api/prisma/schema.prisma @@ -193,12 +193,15 @@ model ApplicationPersistentStorage { } model ServicePersistentStorage { - id String @id @default(cuid()) - serviceId String - path String - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - service Service @relation(fields: [serviceId], references: [id]) + id String @id @default(cuid()) + serviceId String + path String + volumeName String? + predefined Boolean @default(false) + containerId String? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + service Service @relation(fields: [serviceId], references: [id]) @@unique([serviceId, path]) } @@ -419,13 +422,13 @@ model Service { } model ServiceSetting { - id String @id @default(cuid()) - serviceId String - name String - value String - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - service Service @relation(fields: [serviceId], references: [id]) + id String @id @default(cuid()) + serviceId String + name String + value String + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + service Service @relation(fields: [serviceId], references: [id]) @@unique([serviceId, name]) } diff --git a/apps/api/src/lib/services/handlers.ts b/apps/api/src/lib/services/handlers.ts index 07079e5f1..f5347f15f 100644 --- a/apps/api/src/lib/services/handlers.ts +++ b/apps/api/src/lib/services/handlers.ts @@ -8,7 +8,7 @@ import { defaultServiceConfigurations } from '../services'; import { OnlyId } from '../../types'; import templates from '../templates' import { parseAndFindServiceTemplates } from '../../routes/api/v1/services/handlers'; - +import path from 'path'; // export async function startService(request: FastifyRequest) { // try { // const { type } = request.params @@ -692,7 +692,7 @@ export async function startService(request: FastifyRequest) { const teamId = request.user.teamId; const service = await getServiceFromDB({ id, teamId }); - const { type, version, destinationDockerId, destinationDocker, serviceSecret,serviceSetting, exposePort, persistentStorage } = + const { type, version, destinationDockerId, destinationDocker, serviceSecret, serviceSetting, exposePort, persistentStorage } = service; const { workdir } = await createDirectories({ repository: type, buildId: id }); @@ -701,9 +701,10 @@ export async function startService(request: FastifyRequest) { const config = {}; for (const service in template.services) { - console.log(template.services[service]) config[service] = { container_name: service, + build: template.services[service].build || undefined, + command: template.services[service].command, image: template.services[service].image, expose: template.services[service].ports, // ...(exposePort ? { ports: [`${exposePort}:${port}`] } : {}), @@ -714,16 +715,23 @@ export async function startService(request: FastifyRequest) { labels: makeLabelForServices(type), ...defaultComposeConfiguration(network), } - + // Generate files for builds if (template.services[service].build) { if (template.services[service]?.extras?.files?.length > 0) { - console.log(template.services[service]?.extras?.files) + let Dockerfile = ` + FROM ${template.services[service].image}` + for (const file of template.services[service].extras.files) { + const { source, destination, content } = file; + await fs.writeFile(source, content); + Dockerfile += ` + COPY ./${path.basename(source)} ${destination}` + } + await fs.writeFile(`${workdir}/Dockerfile.${service}`, Dockerfile); } } } const { volumeMounts } = persistentVolumes(id, persistentStorage, config) - const composeFile: ComposeFile = { version: '3.8', services: config, diff --git a/apps/api/src/lib/templates.ts b/apps/api/src/lib/templates.ts index 339ceb6c8..92b0b2d16 100644 --- a/apps/api/src/lib/templates.ts +++ b/apps/api/src/lib/templates.ts @@ -43,7 +43,7 @@ export default [ "$$id": { "name": "Plausible Analytics", "documentation": "Taken from https://plausible.io/", - "command": ['sh -c "sleep 10 && /entrypoint.sh db createdb && /entrypoint.sh db migrate && /entrypoint.sh db init-admin && /entrypoint.sh run"'], + "command": 'sh -c "sleep 10 && /entrypoint.sh db createdb && /entrypoint.sh db migrate && /entrypoint.sh db init-admin && /entrypoint.sh run"', "depends_on": [ "$$id-postgresql", "$$id-clickhouse" @@ -52,7 +52,7 @@ export default [ "environment": [ "ADMIN_USER_EMAIL=$$config_admin_user_email", "ADMIN_USER_NAME=$$config_admin_user_name", - "ADMIN_USER_PASSWORD=$$secret_admin_user_password", + "ADMIN_USER_PWD=$$secret_admin_user_pwd", "BASE_URL=$$config_base_url", "SECRET_KEY_BASE=$$secret_secret_key_base", "DISABLE_AUTH=$$config_disable_auth", @@ -68,6 +68,9 @@ export default [ "name": "PostgreSQL", "documentation": "Taken from https://plausible.io/", "image": "bitnami/postgresql:13.2.0", + "volumes": [ + '$$id-postgresql-data:/bitnami/postgresql/', + ], "environment": [ "POSTGRESQL_PASSWORD=$$secret_postgresql_password", "POSTGRESQL_USERNAME=$$config_postgresql_username", @@ -78,7 +81,13 @@ export default [ "$$id-clickhouse": { "name": "Clickhouse", "documentation": "Taken from https://plausible.io/", - "build": "$$workdir", + "build": { + context: "$$workdir", + dockerfile: "Dockerfile.$$id-clickhouse" + }, + "volumes": [ + '$$id-clickhouse-data:/var/lib/clickhouse', + ], "image": "yandex/clickhouse-server:21.3.2.5", "ulimits": { "nofile": { @@ -87,21 +96,25 @@ export default [ } }, "extras": { - "files:": [ + "files": [ { - location: '$$workdir/clickhouse-config.xml', + source: "$$workdir/clickhouse-config.xml", + destination: '/etc/clickhouse-server/users.d/logging.xml', content: 'warningtrue' }, { - location: '$$workdir/clickhouse-user-config.xml', + source: "$$workdir/clickhouse-user-config.xml", + destination: '/etc/clickhouse-server/config.d/logging.xml', content: '00' }, { - location: '$$workdir/init.query', + source: "$$workdir/init.query", + destination: '/docker-entrypoint-initdb.d/init.query', content: 'CREATE DATABASE IF NOT EXISTS plausible;' }, { - location: '$$workdir/init-db.sh', + source: "$$workdir/init-db.sh", + destination: '/docker-entrypoint-initdb.d/init-db.sh', content: 'clickhouse client --queries-file /docker-entrypoint-initdb.d/init.query' } ] @@ -146,12 +159,14 @@ export default [ "description": "This is the admin username. Please change it.", }, { - "id": "$$secret_admin_user_password", - "name": "ADMIN_USER_PASSWORD", - "showAsConfiguration": true, + "id": "$$secret_admin_user_pwd", + "name": "ADMIN_USER_PWD", "label": "Admin User Password", "defaultValue": "$$generate_password", "description": "This is the admin password. Please change it.", + "extras": { + "isVisibleOnUI": true + } }, { "id": "$$secret_secret_key_base", @@ -159,8 +174,8 @@ export default [ "label": "Secret Key Base", "defaultValue": "$$generate_passphrase", "description": "", - "details": { - "length":64 + "extras": { + "length": 64 } }, { diff --git a/apps/api/src/routes/api/v1/services/handlers.ts b/apps/api/src/routes/api/v1/services/handlers.ts index 428a80465..ec723a30b 100644 --- a/apps/api/src/routes/api/v1/services/handlers.ts +++ b/apps/api/src/routes/api/v1/services/handlers.ts @@ -132,8 +132,8 @@ export async function parseAndFindServiceTemplates(service: any, workdir?: strin const label = foundTemplate.variables.find(v => v.name === envKey)?.label const description = foundTemplate.variables.find(v => v.name === envKey)?.description const defaultValue = foundTemplate.variables.find(v => v.name === envKey)?.defaultValue - const showAsConfiguration = foundTemplate.variables.find(v => v.name === envKey)?.showAsConfiguration - if (envValue.startsWith('$$config') || showAsConfiguration) { + const isVisibleOnUI = foundTemplate.variables.find(v => v.name === envKey)?.extras?.isVisibleOnUI + if (envValue.startsWith('$$config') || isVisibleOnUI) { parsedTemplate[realKey].environment.push( { name: envKey, value: envValue, label, description, defaultValue } ) @@ -220,12 +220,11 @@ export async function saveServiceType(request: FastifyRequest, foundTemplate.variables = foundTemplate.variables.map(variable => { let { id: variableId } = variable; if (variableId.startsWith('$$secret_')) { + const length = variable?.extras && variable.extras['length'] if (variable.defaultValue === '$$generate_password') { - const length = variable?.details['length'] || null - variable.value = generatePassword({length}); + variable.value = generatePassword({ length }); } else if (variable.defaultValue === '$$generate_passphrase') { - const length = variable?.details['length'] || null - variable.value = generatePassword({length}); + variable.value = generatePassword({ length }); } } if (variableId.startsWith('$$config_')) { @@ -267,6 +266,18 @@ export async function saveServiceType(request: FastifyRequest, } } } + for (const service of Object.keys(foundTemplate.services)) { + if (foundTemplate.services[service].volumes) { + for (const volume of foundTemplate.services[service].volumes) { + const [volumeName, path] = volume.split(':') + if (!volumeName.startsWith('/')) { + await prisma.servicePersistentStorage.create({ + data: { volumeName, path, containerId: service, predefined: true, service: { connect: { id } } } + }); + } + } + } + } await prisma.service.update({ where: { id }, data: { type, version: foundTemplate.serviceDefaultVersion } }) return reply.code(201).send() } else { diff --git a/apps/api/src/routes/webhooks/traefik/handlers.ts b/apps/api/src/routes/webhooks/traefik/handlers.ts index e6d4e474a..6d8747845 100644 --- a/apps/api/src/routes/webhooks/traefik/handlers.ts +++ b/apps/api/src/routes/webhooks/traefik/handlers.ts @@ -329,10 +329,9 @@ export async function traefikConfiguration(request, reply) { fqdn, id, type, - destinationDocker, destinationDockerId, dualCerts, - plausibleAnalytics + serviceSetting } = service; if (destinationDockerId) { const found = supportedServiceTypesAndVersions.find((a) => a.name === type); @@ -348,8 +347,11 @@ export async function traefikConfiguration(request, reply) { if (isRunning) { // Plausible Analytics custom script let scriptName = false; - if (type === 'plausibleanalytics' && plausibleAnalytics.scriptName !== 'plausible.js') { - scriptName = plausibleAnalytics.scriptName; + if (type === 'plausibleanalytics') { + const foundScriptName = serviceSetting.find((a) => a.name === 'SCRIPT_NAME')?.value; + if (foundScriptName) { + scriptName = foundScriptName; + } } let container = id; diff --git a/apps/ui/src/lib/components/Setting.svelte b/apps/ui/src/lib/components/Setting.svelte index ef5bab024..865b3dc38 100644 --- a/apps/ui/src/lib/components/Setting.svelte +++ b/apps/ui/src/lib/components/Setting.svelte @@ -8,7 +8,7 @@ export let setting: any; export let title: any; export let isBeta: any = false; - export let description: any; + export let description: any = null; export let isCenter = true; export let disabled = false; export let dataTooltip: any = null; diff --git a/apps/ui/src/routes/applications/[id]/_Storage.svelte b/apps/ui/src/routes/applications/[id]/_Storage.svelte index a20615a92..cf54e16df 100644 --- a/apps/ui/src/routes/applications/[id]/_Storage.svelte +++ b/apps/ui/src/routes/applications/[id]/_Storage.svelte @@ -59,18 +59,33 @@ } -
-
-
+
+
+ {#if storage.id} +
+ + +
+ {/if} +
+ +
+ +
{#if isNew}
-
diff --git a/apps/ui/src/routes/applications/[id]/storages.svelte b/apps/ui/src/routes/applications/[id]/storages.svelte index 6338deed3..06e3c87d2 100644 --- a/apps/ui/src/routes/applications/[id]/storages.svelte +++ b/apps/ui/src/routes/applications/[id]/storages.svelte @@ -42,7 +42,7 @@ />
- + {#each persistentStorages as storage} {#key storage.id} diff --git a/apps/ui/src/routes/services/[id]/_Storage.svelte b/apps/ui/src/routes/services/[id]/_Storage.svelte index 46e9ebbd9..a7c8a155b 100644 --- a/apps/ui/src/routes/services/[id]/_Storage.svelte +++ b/apps/ui/src/routes/services/[id]/_Storage.svelte @@ -59,35 +59,76 @@ } -
-
-
- - {#if isNew} -
- -
- {:else} -
-
- -
-
- -
+
+ {#if storage.predefined} +
+
+ +
+
+ +
+
+ {:else} +
+ {#if storage.id} +
+
{/if} +
+ {#if isNew} + + {/if} + +
+ +
+ {#if isNew} +
+ +
+ {:else} +
+
+ +
+
+ +
+
+ {/if} +
-
+ {/if}
diff --git a/apps/ui/src/routes/services/[id]/__layout.svelte b/apps/ui/src/routes/services/[id]/__layout.svelte index b44b096ac..40057aa50 100644 --- a/apps/ui/src/routes/services/[id]/__layout.svelte +++ b/apps/ui/src/routes/services/[id]/__layout.svelte @@ -252,27 +252,6 @@ Loading... {:else if $status.service.overallStatus === 'healthy'} - {:else if $status.service.overallStatus === 'degraded'}