Shiloh/lasthourcloud
Shiloh/lasthourcloud
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 4 Wiki Activity

fixes

Browse Source
This commit is contained in:
Andras Bacsai 2022-10-20 16:06:33 +02:00
parent 9f3732d35b
commit f4019db3d1
9 changed files with 191 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
apps/api/scripts/convert.mjs Normal file
View File

@ -0,0 +1,95 @@
import fs from 'fs/promises';
import yaml from 'js-yaml';
const templateYml = await fs.readFile('./caprover.yml', 'utf8')
const template = yaml.load(templateYml)
const newTemplate = {
"templateVersion": "1.0.0",
"serviceDefaultVersion": "latest",
"name": "",
"displayName": "",
"description": "",
"services": {
},
"variables": []
}
const version = template.caproverOneClickApp.variables.find(v => v.id === '$$cap_APP_VERSION').defaultValue || 'latest'
newTemplate.displayName = template.caproverOneClickApp.displayName
newTemplate.name = template.caproverOneClickApp.displayName.toLowerCase()
newTemplate.documentation = template.caproverOneClickApp.documentation
newTemplate.description = template.caproverOneClickApp.description
newTemplate.serviceDefaultVersion = version
const varSet = new Set()
const caproverVariables = template.caproverOneClickApp.variables
for (const service of Object.keys(template.services)) {
const serviceTemplate = template.services[service]
const newServiceName = service.replaceAll('cap_appname', 'id')
const newService = {
image: '',
command: '',
environment: [],
volumes: []
}
const FROM = serviceTemplate.caproverExtra?.dockerfileLines?.find((line) => line.startsWith('FROM'))
if (serviceTemplate.image) {
newService.image = serviceTemplate.image.replaceAll('cap_APP_VERSION', 'core_version')
} else if (FROM) {
newService.image = FROM.split(' ')[1].replaceAll('cap_APP_VERSION', 'core_version')
}
const CMD = serviceTemplate.caproverExtra?.dockerfileLines?.find((line) => line.startsWith('CMD'))
if (serviceTemplate.command) {
newService.command = serviceTemplate.command
} else if (CMD) {
newService.command = CMD.replace('CMD ', '').replaceAll('"', '').replaceAll('[', '').replaceAll(']', '').replaceAll(',', ' ').replace(/\s+/g, ' ')
} else {
delete newService.command
}
const ENTRYPOINT = serviceTemplate.caproverExtra?.dockerfileLines?.find((line) => line.startsWith('ENTRYPOINT'))
if (serviceTemplate.entrypoint) {
newService.command = serviceTemplate.entrypoint
} else if (ENTRYPOINT) {
newService.entrypoint = ENTRYPOINT.replace('ENTRYPOINT ', '').replaceAll('"', '').replaceAll('[', '').replaceAll(']', '').replaceAll(',', ' ').replace(/\s+/g, ' ')
} else {
delete newService.entrypoint
}
if (serviceTemplate.environment && Object.keys(serviceTemplate.environment).length > 0) {
for (const env of Object.keys(serviceTemplate.environment)) {
if (serviceTemplate.environment[env].startsWith('srv-captain--$$cap_appname')) {
continue;
}
const value = '$$config_' + serviceTemplate.environment[env].replaceAll('srv-captain--$$cap_appname', '$$$id').replace('$$cap', '').replaceAll('captain-overlay-network', `$$$config_${env}`).toLowerCase()
newService.environment.push(`${env}=${value}`)
const foundVariable = varSet.has(env)
if (!foundVariable) {
const foundCaproverVariable = caproverVariables.find((item) => item.id === serviceTemplate.environment[env])
const defaultValue = foundCaproverVariable?.defaultValue ? foundCaproverVariable?.defaultValue.toString()?.replace('$$cap_gen_random_hex', '$$$generate_hex') : ''
if (defaultValue && defaultValue !== foundCaproverVariable?.defaultValue) {
console.log('changed')
}
newTemplate.variables.push({
"id": value,
"name": env,
"label": foundCaproverVariable?.label || '',
"defaultValue": defaultValue,
"description": foundCaproverVariable?.description || '',
})
}
varSet.add(env)
}
}
if (serviceTemplate.volumes && serviceTemplate.volumes.length > 0) {
for (const volume of serviceTemplate.volumes) {
const [source, target] = volume.split(':')
newService.volumes.push(`${source.replaceAll('$$cap_appname-', '$$$id-')}:${target}`)
}
}
newTemplate.services[newServiceName] = newService
}
await fs.writeFile('./caprover_new.yml', yaml.dump([{ ...newTemplate }]))

10
apps/api/src/index.ts
View File

@ -11,6 +11,7 @@ import { scheduler } from './lib/scheduler';
import { compareVersions } from 'compare-versions';
import Graceful from '@ladjs/graceful'
import axios from 'axios';
import yaml from 'js-yaml'
import fs from 'fs/promises';
import { verifyRemoteDockerEngineFn } from './routes/api/v1/destinations/handlers';
import { checkContainer } from './lib/docker';
@ -123,7 +124,14 @@ const host = '0.0.0.0';
}
})
try {
await migrateServicesToNewTemplate()
const templateYaml = await axios.get('https://gist.githubusercontent.com/andrasbacsai/701c450ef4272a929215cab11d737e3d/raw/4f021329d22934b90c5d67a0e49839a32bd629fd/template.yaml')
const templateJson = yaml.load(templateYaml.data)
if (isDev) {
await fs.writeFile('./template.json', JSON.stringify(templateJson, null, 2))
} else {
await fs.writeFile('/app/template.json', JSON.stringify(templateJson, null, 2))
}
await migrateServicesToNewTemplate(templateJson)
await fastify.listen({ port, host })
console.log(`Coolify's API is listening on ${host}:${port}`);

54
apps/api/src/lib.ts
View File

@ -1,8 +1,7 @@
import { decrypt, encrypt, getDomain, prisma } from "./lib/common";
import { includeServices } from "./lib/services/common";
import templates from "./lib/templates";
export async function migrateServicesToNewTemplate() {
export async function migrateServicesToNewTemplate(templates: any) {
// This function migrates old hardcoded services to the new template based services
try {
const services = await prisma.service.findMany({ include: includeServices })
@ -10,20 +9,25 @@ export async function migrateServicesToNewTemplate() {
if (!service.type) {
continue;
}
if (service.type === 'plausibleanalytics' && service.plausibleAnalytics) await plausibleAnalytics(service)
if (service.type === 'fider' && service.fider) await fider(service)
if (service.type === 'minio' && service.minio) await minio(service)
if (service.type === 'vscodeserver' && service.vscodeserver) await vscodeserver(service)
if (service.type === 'wordpress' && service.wordpress) await wordpress(service)
if (service.type === 'ghost' && service.ghost) await ghost(service)
if (service.type === 'meilisearch' && service.meiliSearch) await meilisearch(service)
if (service.type === 'umami' && service.umami) await umami(service)
if (service.type === 'hasura' && service.hasura) await hasura(service)
if (service.type === 'glitchTip' && service.glitchTip) await glitchtip(service)
if (service.type === 'searxng' && service.searxng) await searxng(service)
if (service.type === 'weblate' && service.weblate) await weblate(service)
let template = templates.find(t => t.name === service.type.toLowerCase());
if (template) {
template = JSON.parse(JSON.stringify(template).replaceAll('$$id', service.id))
if (service.type === 'plausibleanalytics' && service.plausibleAnalytics) await plausibleAnalytics(service)
if (service.type === 'fider' && service.fider) await fider(service)
if (service.type === 'minio' && service.minio) await minio(service)
if (service.type === 'vscodeserver' && service.vscodeserver) await vscodeserver(service)
if (service.type === 'wordpress' && service.wordpress) await wordpress(service)
if (service.type === 'ghost' && service.ghost) await ghost(service)
if (service.type === 'meilisearch' && service.meiliSearch) await meilisearch(service)
if (service.type === 'umami' && service.umami) await umami(service)
if (service.type === 'hasura' && service.hasura) await hasura(service)
if (service.type === 'glitchTip' && service.glitchTip) await glitchtip(service)
if (service.type === 'searxng' && service.searxng) await searxng(service)
if (service.type === 'weblate' && service.weblate) await weblate(service)
await createVolumes(service, template);
}
await createVolumes(service);
}
} catch (error) {
console.log(error)
@ -321,19 +325,15 @@ async function migrateSecrets(secrets: any[], service: any) {
await prisma.serviceSecret.findFirst({ where: { name, serviceId: service.id } }) || await prisma.serviceSecret.create({ data: { name, value, service: { connect: { id: service.id } } } })
}
}
async function createVolumes(service: any) {
async function createVolumes(service: any, template: any) {
const volumes = [];
let template = templates.find(t => t.name === service.type.toLowerCase());
if (template) {
template = JSON.parse(JSON.stringify(template).replaceAll('$$id', service.id))
for (const s of Object.keys(template.services)) {
if (template.services[s].volumes && template.services[s].volumes.length > 0) {
for (const volume of template.services[s].volumes) {
const volumeName = volume.split(':')[0]
const volumePath = volume.split(':')[1]
const volumeService = service.id
volumes.push(`${volumeName}@@@${volumePath}@@@${volumeService}`)
}
for (const s of Object.keys(template.services)) {
if (template.services[s].volumes && template.services[s].volumes.length > 0) {
for (const volume of template.services[s].volumes) {
const volumeName = volume.split(':')[0]
const volumePath = volume.split(':')[1]
const volumeService = service.id
volumes.push(`${volumeName}@@@${volumePath}@@@${volumeService}`)
}
}
}

10
apps/api/src/lib/services.ts
View File

@ -1,5 +1,13 @@
import { createDirectories, getServiceFromDB, getServiceImage, getServiceMainPort, makeLabelForServices } from "./common";
import { createDirectories, getServiceFromDB, getServiceImage, getServiceMainPort, isDev, makeLabelForServices } from "./common";
import fs from 'fs/promises';
export async function getTemplates() {
let templates = [];
if (isDev) {
templates = JSON.parse((await fs.readFile('./template.json')).toString())
}
return templates
}
export async function defaultServiceConfigurations({ id, teamId }) {
const service = await getServiceFromDB({ id, teamId });
const { destinationDockerId, destinationDocker, type, serviceSecret } = service;

3
apps/api/src/lib/services/handlers.ts
View File

@ -6,7 +6,7 @@ import { ServiceStartStop } from '../../routes/api/v1/services/types';
import { asyncSleep, ComposeFile, createDirectories, defaultComposeConfiguration, errorHandler, executeDockerCmd, getDomain, getFreePublicPort, getServiceFromDB, getServiceImage, getServiceMainPort, isARM, isDev, makeLabelForServices, persistentVolumes, prisma } from '../common';
import { defaultServiceConfigurations } from '../services';
import { OnlyId } from '../../types';
import templates from '../templates'
import { parseAndFindServiceTemplates } from '../../routes/api/v1/services/handlers';
import path from 'path';
// export async function startService(request: FastifyRequest<ServiceStartStop>) {
@ -711,6 +711,7 @@ export async function startService(request: FastifyRequest<ServiceStartStop>) {
container_name: service,
build: template.services[service].build || undefined,
command: template.services[service].command,
entrypoint: template.services[service]?.entrypoint,
image: template.services[service].image,
expose: template.services[service].ports,
// ...(exposePort ? { ports: [`${exposePort}:${port}`] } : {}),

15
apps/api/src/lib/templates.ts
View File

@ -21,7 +21,7 @@ export default [
`WEBLATE_ADMIN_PASSWORD=$$secret_weblate_admin_password`,
`POSTGRES_PASSWORD=$$secret_postgres_password`,
`POSTGRES_USER=$$config_postgres_user`,
`POSTGRES_DATABASE=$$config_postgres_database`,
`POSTGRES_DATABASE=$$config_postgres_db`,
`POSTGRES_HOST=$$id-postgresql`,
`POSTGRES_PORT=5432`,
`REDIS_HOST=$$id-redis`,
@ -94,13 +94,7 @@ export default [
"defaultValue": "weblate",
"description": "",
},
{
"id": "$$config_postgres_database",
"name": "POSTGRES_DATABASE",
"label": "PostgreSQL Database",
"defaultValue": "$$config_postgres_db",
"description": ""
},
]
},
{
@ -121,7 +115,6 @@ export default [
],
"environment": [
"SEARXNG_BASE_URL=$$config_searxng_base_url",
"SECRET_KEY=$$secret_secret_key",
],
"ports": [
"8080"
@ -1462,9 +1455,7 @@ export default [
"label": "Secret Key Base",
"defaultValue": "$$generate_passphrase",
"description": "",
"extras": {
"length": 64
}
},
{
"id": "$$config_disable_auth",

64
apps/api/src/routes/api/v1/services/handlers.ts
View File

@ -2,6 +2,7 @@ import type { FastifyReply, FastifyRequest } from 'fastify';
import fs from 'fs/promises';
import yaml from 'js-yaml';
import bcrypt from 'bcryptjs';
import crypto from 'crypto';
import { prisma, uniqueName, asyncExecShell, getServiceFromDB, getContainerUsage, isDomainConfigured, saveUpdateableFields, fixType, decrypt, encrypt, ComposeFile, getFreePublicPort, getDomain, errorHandler, generatePassword, isDev, stopTcpHttpProxy, executeDockerCmd, checkDomainsIsValidInDNS, checkExposedPort, listSettings } from '../../../../lib/common';
import { day } from '../../../../lib/dayjs';
import { checkContainer, isContainerExited } from '../../../../lib/docker';
@ -12,7 +13,7 @@ import type { ActivateWordpressFtp, CheckService, CheckServiceDomain, DeleteServ
import { supportedServiceTypesAndVersions } from '../../../../lib/services/supportedVersions';
import { configureServiceType, removeService } from '../../../../lib/services/common';
import { hashPassword } from '../handlers';
import templates from '../../../../lib/templates';
import { getTemplates } from '../../../../lib/services';
export async function listServices(request: FastifyRequest) {
try {
@ -113,6 +114,7 @@ export async function getServiceStatus(request: FastifyRequest<OnlyId>) {
}
}
export async function parseAndFindServiceTemplates(service: any, workdir?: string, isDeploy: boolean = false) {
const templates = await getTemplates()
const foundTemplate = templates.find(t => t.name === service.type.toLowerCase())
let parsedTemplate = {}
if (foundTemplate) {
@ -162,7 +164,6 @@ export async function parseAndFindServiceTemplates(service: any, workdir?: strin
parsedTemplate = JSON.parse(JSON.stringify(parsedTemplate).replaceAll(regex, getDomain(service.fqdn) + "\""))
} else {
parsedTemplate = JSON.parse(JSON.stringify(parsedTemplate).replaceAll(regex, value + "\""))
}
}
@ -203,7 +204,7 @@ export async function getService(request: FastifyRequest<OnlyId>) {
export async function getServiceType(request: FastifyRequest) {
try {
return {
services: templates
services: await getTemplates()
}
} catch ({ status, message }) {
return errorHandler({ status, message })
@ -213,6 +214,7 @@ export async function saveServiceType(request: FastifyRequest<SaveServiceType>,
try {
const { id } = request.params;
const { type } = request.body;
const templates = await getTemplates()
let foundTemplate = templates.find(t => t.name === type)
if (foundTemplate) {
let generatedVariables = new Set()
@ -223,22 +225,32 @@ export async function saveServiceType(request: FastifyRequest<SaveServiceType>,
if (foundTemplate.variables.length > 0) {
foundTemplate.variables = foundTemplate.variables.map(variable => {
let { id: variableId } = variable;
console.log(variableId)
if (variableId.startsWith('$$secret_')) {
const length = variable?.extras && variable.extras['length']
if (variable.defaultValue === '$$generate_password') {
variable.value = generatePassword({ length });
} else if (variable.defaultValue === '$$generate_passphrase') {
variable.value = generatePassword({ length });
if (variable.defaultValue.startsWith('$$generate_password')) {
const length = variable.defaultValue.replace('$$generate_password(', '').replace('\)', '') || 16
variable.value = generatePassword({ length: Number(length) });
} else if (variable.defaultValue.startsWith('$$generate_hex')) {
const length = variable.defaultValue.replace('$$generate_hex(', '').replace('\)', '') || 16
variable.value = crypto.randomBytes(Number(length)).toString('hex');
} else if (!variable.defaultValue) {
variable.defaultValue = undefined
}
}
if (variableId.startsWith('$$config_')) {
if (variable.defaultValue === '$$generate_username') {
variable.value = cuid();
if (variable.defaultValue.startsWith('$$generate_username')) {
const length = variable.defaultValue.replace('$$generate_username(', '').replace('\)', '')
if (length !== '$$generate_username') {
variable.value = crypto.randomBytes(Number(length)).toString('hex');
} else {
variable.value = cuid();
}
} else {
variable.value = variable.defaultValue || ''
if (variable.defaultValue.startsWith('$$generate_hex')) {
const length = variable.defaultValue.replace('$$generate_hex(', '').replace('\)', '') || 16
variable.value = crypto.randomBytes(Number(length)).toString('hex');
} else {
variable.value = variable.defaultValue || ''
}
}
}
if (variable.value) {
@ -268,17 +280,24 @@ export async function saveServiceType(request: FastifyRequest<SaveServiceType>,
if (!variable.value) {
continue;
}
await prisma.serviceSecret.create({
data: { name: variable.name, value: encrypt(variable.value), service: { connect: { id } } }
})
const found = await prisma.serviceSecret.findFirst({ where: { name: variable.name, serviceId: id } })
if (!found) {
await prisma.serviceSecret.create({
data: { name: variable.name, value: encrypt(variable.value), service: { connect: { id } } }
})
}
}
if (variable.id.startsWith('$$config_')) {
if (!variable.value) {
variable.value = '';
}
await prisma.serviceSetting.create({
data: { name: variable.name, value: variable.value, service: { connect: { id } } }
})
const found = await prisma.serviceSetting.findFirst({ where: { name: variable.name, serviceId: id } })
if (!found) {
await prisma.serviceSetting.create({
data: { name: variable.name, value: variable.value.toString(), service: { connect: { id } } }
})
}
}
}
}
@ -287,9 +306,12 @@ export async function saveServiceType(request: FastifyRequest<SaveServiceType>,
for (const volume of foundTemplate.services[service].volumes) {
const [volumeName, path] = volume.split(':')
if (!volumeName.startsWith('/')) {
await prisma.servicePersistentStorage.create({
data: { volumeName, path, containerId: service, predefined: true, service: { connect: { id } } }
});
const found = await prisma.servicePersistentStorage.findFirst({ where: { volumeName, serviceId: id } })
if (!found) {
await prisma.servicePersistentStorage.create({
data: { volumeName, path, containerId: service, predefined: true, service: { connect: { id } } }
});
}
}
}
}

3
apps/ui/src/routes/services/[id]/index.svelte
View File

@ -381,7 +381,8 @@
class:border-b={template[oneService].environment.length > 0}
class:border-coolgray-500={template[oneService].environment.length > 0}
>
<div class="title font-bold pb-3">{template[oneService].name}</div>
<div class="title font-bold pb-3">{template[oneService].name || oneService.replace(`${id}-`,'').replace(id,service.type)}</div>
<ServiceStatus id={oneService} />
</div>
<div class="grid grid-flow-row gap-2 px-4">

2
apps/ui/src/routes/services/[id]/logs/index.svelte
View File

@ -93,7 +93,7 @@
<div class="title font-bold pb-3">Service Logs</div>
</div>
</div>
<div class="flex gap-2 lg:gap-8 pb-4">
<div class="grid grid-cols-4 gap-2 lg:gap-8 pb-4">
{#if template}
{#each Object.keys(template) as service}
<button