From 02c42a7e3a4260aded73e7706833f995607780da Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 5 Oct 2022 09:01:17 +0000 Subject: [PATCH 01/34] fix: pure docker based development --- Dockerfile-dev | 27 +++++++++++++++++++++++++++ apps/api/src/lib/common.ts | 2 +- docker-compose-dev.yaml | 24 ++++++++++++++++++++++++ package.json | 6 ++++-- pnpm-lock.yaml | 4 +++- 5 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 Dockerfile-dev diff --git a/Dockerfile-dev b/Dockerfile-dev new file mode 100644 index 000000000..1ea476959 --- /dev/null +++ b/Dockerfile-dev @@ -0,0 +1,27 @@ +ARG PNPM_VERSION=7.11.0 +ARG NPM_VERSION=8.19.1 + +FROM node:18-slim +ENV NODE_ENV development +ARG TARGETPLATFORM +WORKDIR /app + +RUN npm --no-update-notifier --no-fund --global install pnpm@${PNPM_VERSION} + +RUN apt update && apt -y install --no-install-recommends ca-certificates git git-lfs openssh-client curl jq cmake sqlite3 openssl psmisc python3 +RUN apt-get clean autoclean && apt-get autoremove --yes && rm -rf /var/lib/{apt,dpkg,cache,log}/ +RUN npm --no-update-notifier --no-fund --global install pnpm@${PNPM_VERSION} +RUN npm install -g npm@${PNPM_VERSION} + +RUN mkdir -p ~/.docker/cli-plugins/ +# https://download.docker.com/linux/static/stable/ +RUN curl -SL https://cdn.coollabs.io/bin/$TARGETPLATFORM/docker-20.10.9 -o /usr/bin/docker +# https://github.com/docker/compose/releases +# Reverted to 2.6.1 because of this https://github.com/docker/compose/issues/9704. 2.9.0 still has a bug. +RUN curl -SL https://cdn.coollabs.io/bin/$TARGETPLATFORM/docker-compose-linux-2.6.1 -o ~/.docker/cli-plugins/docker-compose +RUN chmod +x ~/.docker/cli-plugins/docker-compose /usr/bin/docker + +RUN (curl -sSL "https://github.com/buildpacks/pack/releases/download/v0.27.0/pack-v0.27.0-linux.tgz" | tar -C /usr/local/bin/ --no-same-owner -xzv pack) + +EXPOSE 3000 +ENV CHECKPOINT_DISABLE=1 \ No newline at end of file diff --git a/apps/api/src/lib/common.ts b/apps/api/src/lib/common.ts index 6ccd2f71a..32c9468a6 100644 --- a/apps/api/src/lib/common.ts +++ b/apps/api/src/lib/common.ts @@ -20,7 +20,7 @@ import { scheduler } from './scheduler'; import { supportedServiceTypesAndVersions } from './services/supportedVersions'; import { includeServices } from './services/common'; -export const version = '3.10.14'; +export const version = '3.10.15'; export const isDev = process.env.NODE_ENV === 'development'; const algorithm = 'aes-256-ctr'; diff --git a/docker-compose-dev.yaml b/docker-compose-dev.yaml index e1934322c..e35760989 100644 --- a/docker-compose-dev.yaml +++ b/docker-compose-dev.yaml @@ -1,6 +1,30 @@ version: '3.8' services: + coolify: + build: + context: . + dockerfile: Dockerfile-dev + command: pnpm dev:container + env_file: apps/api/.env + environment: + - GITPOD_WORKSPACE_URL=${GITPOD_WORKSPACE_URL} + - CODESANDBOX_HOST=${CODESANDBOX_HOST} + container_name: coolify + ports: + - target: 3000 + published: 3000 + protocol: tcp + mode: host + - target: 3001 + published: 3001 + protocol: tcp + mode: host + volumes: + - ./:/app + - '/var/run/docker.sock:/var/run/docker.sock' + networks: + - coolify-infra fluent-bit: image: coollabsio/coolify-fluent-bit:1.0.0 command: /fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit-dev.conf diff --git a/package.json b/package.json index 6f580d533..c183074c7 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "coolify", "description": "An open-source & self-hostable Heroku / Netlify alternative.", - "version": "3.10.14", + "version": "3.10.15", "license": "Apache-2.0", "repository": "github:coollabsio/coolify", "scripts": { @@ -15,7 +15,8 @@ "format:api": "NODE_ENV=development pnpm run --filter api format", "lint": "run-p -l -n lint:*", "lint:api": "NODE_ENV=development pnpm run --filter api lint", - "dev": "run-p -l -n dev:*", + "dev:container": "pnpm install && pnpm db:push && pnpm db:seed && pnpm dev", + "dev": "run-p -l -n dev:api dev:ui", "dev:api": "NODE_ENV=development pnpm run --filter api dev", "dev:ui": "NODE_ENV=development pnpm run --filter ui dev", "build": "NODE_ENV=production run-p -n build:*", @@ -34,6 +35,7 @@ "docker", "self-host", "iaas", + "paas", "heroku", "netlify", "open-source", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3c7fb8aea..0820847ab 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -104,6 +104,7 @@ importers: node-os-utils: 1.3.7 p-all: 4.0.0 p-throttle: 5.0.0 + prisma: 4.4.0 public-ip: 6.0.1 pump: 3.0.0 ssh-config: 4.1.6 @@ -120,7 +121,6 @@ importers: eslint-plugin-prettier: 4.2.1_tgumt6uwl2md3n6uqnggd6wvce nodemon: 2.0.20 prettier: 2.7.1 - prisma: 4.4.0 rimraf: 3.0.2 tsconfig-paths: 4.1.0 typescript: 4.8.4 @@ -532,6 +532,7 @@ packages: /@prisma/engines/4.4.0: resolution: {integrity: sha512-Fpykccxlt9MHrAs/QpPGpI2nOiRxuLA+LiApgA59ibbf24YICZIMWd3SI2YD+q0IAIso0jCGiHhirAIbxK3RyQ==} requiresBuild: true + dev: false /@rollup/pluginutils/4.2.1: resolution: {integrity: sha512-iKnFXr7NkdZAIHiIWE+BX5ULi/ucVFYWD6TbAV+rZctiRTY2PL6tsIKhoIOaoskiWAkgu+VsbXgUVDNLHf+InQ==} @@ -5078,6 +5079,7 @@ packages: requiresBuild: true dependencies: '@prisma/engines': 4.4.0 + dev: false /private/0.1.8: resolution: {integrity: sha512-VvivMrbvd2nKkiG38qjULzlc+4Vx4wm/whI9pQD35YrARNnhxeiRktSOhSukRLFNlzg6Br/cJPet5J/u19r/mg==} From cb478e0dc8457435f7d1587ac59b6d05fde6761e Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 5 Oct 2022 09:13:51 +0000 Subject: [PATCH 02/34] add contribution guide on container based development flow --- CONTRIBUTION.md | 12 ++++++++++-- docker-compose-dev.yaml | 6 ++++-- package.json | 2 +- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTION.md b/CONTRIBUTION.md index 78992243b..9f4bf9321 100644 --- a/CONTRIBUTION.md +++ b/CONTRIBUTION.md @@ -5,19 +5,27 @@ # Contribution You can ask for guidance anytime on our Discord server in the #contribution channel. ## Setup your development environment +### Container based development flow (recommended and the easiest) +All you need is to intall [Docker Engine 20.11+](https://docs.docker.com/engine/install/) on your local machine and run `pnpm dev:container`. It will build the base image for Coolify and start the development server inside Docker. All required ports (3000, 3001) will be exposed to your host. + ### Github codespaces If you have github codespaces enabled then you can just create a codespace and run `pnpm dev` to run your the dev environment. All the required dependencies and packages has been configured for you already. ### Gitpod +1. Use [container based development flow](#container-based-development-flow-easiest) +2. Or setup your workspace manually: -If you have a [Gitpod](https://gitpod.io), you can just create a workspace from this repository, run `pnpm install && pnpm db:push && pnpm db:seed` and then `pnpm dev`. All the required dependencies and packages has been configured for you already. +Create a workspace from this repository, run `pnpm install && pnpm db:push && pnpm db:seed` and then `pnpm dev`. All the required dependencies and packages has been configured for you already. + +> Some packages, just `pack` are not installed in this way. You cannot test all the features. Please use the [container based development flow](#container-based-development-flow-easiest). ### Local Machine > At the moment, Coolify `doesn't support Windows`. You must use `Linux` or `MacOS` or consider using Gitpod or Github Codespaces. -- Due to the lock file, this repository is best with [pnpm](https://pnpm.io). I recommend you try and use `pnpm` because it is cool and efficient! +Install all the prerequisites manually to your host system. If you would not like to install anything, I suggest to use the [container based development flow](#container-based-development-flow-easiest). +- Due to the lock file, this repository is best with [pnpm](https://pnpm.io). I recommend you try and use `pnpm` because it is cool and efficient! - You need to have [Docker Engine](https://docs.docker.com/engine/install/) installed locally. - You need to have [Docker Compose Plugin](https://docs.docker.com/compose/install/compose-plugin/) installed locally. - You need to have [GIT LFS Support](https://git-lfs.github.com/) installed locally. diff --git a/docker-compose-dev.yaml b/docker-compose-dev.yaml index e35760989..468d17efc 100644 --- a/docker-compose-dev.yaml +++ b/docker-compose-dev.yaml @@ -5,9 +5,11 @@ services: build: context: . dockerfile: Dockerfile-dev - command: pnpm dev:container - env_file: apps/api/.env + command: bash -c 'pnpm install && pnpm db:push && pnpm db:seed && pnpm dev' environment: + - COOLIFY_APP_ID=random-local-id + - COOLIFY_SECRET_KEY=12341234123412341234123412341234 + - COOLIFY_DATABASE_URL=file:../db/dev.db - GITPOD_WORKSPACE_URL=${GITPOD_WORKSPACE_URL} - CODESANDBOX_HOST=${CODESANDBOX_HOST} container_name: coolify diff --git a/package.json b/package.json index c183074c7..de44428ea 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "format:api": "NODE_ENV=development pnpm run --filter api format", "lint": "run-p -l -n lint:*", "lint:api": "NODE_ENV=development pnpm run --filter api lint", - "dev:container": "pnpm install && pnpm db:push && pnpm db:seed && pnpm dev", + "dev:container": "docker-compose -f docker-compose-dev.yaml up", "dev": "run-p -l -n dev:api dev:ui", "dev:api": "NODE_ENV=development pnpm run --filter api dev", "dev:ui": "NODE_ENV=development pnpm run --filter ui dev", From 3f1841a1881f9685b940ab7dcbe848bfed84cca5 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 5 Oct 2022 10:27:12 +0000 Subject: [PATCH 03/34] init: docker-compose support --- apps/api/src/lib/buildPacks/compose.ts | 34 ++++++++++++++++++++++++++ apps/api/src/lib/buildPacks/index.ts | 4 ++- apps/api/src/lib/common.ts | 2 +- apps/ui/src/lib/templates.ts | 8 ++++++ 4 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 apps/api/src/lib/buildPacks/compose.ts diff --git a/apps/api/src/lib/buildPacks/compose.ts b/apps/api/src/lib/buildPacks/compose.ts new file mode 100644 index 000000000..30af1f10b --- /dev/null +++ b/apps/api/src/lib/buildPacks/compose.ts @@ -0,0 +1,34 @@ +import { promises as fs } from 'fs'; +import { executeDockerCmd } from '../common'; +import { buildImage } from './common'; +import yaml from 'js-yaml'; + +export default async function (data) { + let { + applicationId, + dockerId, + debug, + tag, + workdir, + buildId, + baseDirectory, + secrets, + pullmergeRequestId, + dockerFileLocation + } = data + const file = `${workdir}${baseDirectory}/docker-compose.yml`; + const dockerComposeRaw = await fs.readFile(`${file}`, 'utf8') + const dockerComposeYaml = yaml.load(dockerComposeRaw) + if (!dockerComposeYaml.services) { + throw 'No Services found in docker-compose file.' + } + for (let [key, value] of Object.entries(dockerComposeYaml.services)) { + value['container_name'] = `${applicationId}-${key}` + console.log({key, value}); + } + + throw 'Halting' + // await executeDockerCmd({ debug, buildId, applicationId, dockerId, command: `docker compose --project-directory ${workdir} pull` }) + // await executeDockerCmd({ debug, buildId, applicationId, dockerId, command: `docker compose --project-directory ${workdir} build --progress plain --pull` }) + // await executeDockerCmd({ debug, buildId, applicationId, dockerId, command: `docker compose --project-directory ${workdir} up -d` }) +} diff --git a/apps/api/src/lib/buildPacks/index.ts b/apps/api/src/lib/buildPacks/index.ts index b8eb4d609..8e82047fe 100644 --- a/apps/api/src/lib/buildPacks/index.ts +++ b/apps/api/src/lib/buildPacks/index.ts @@ -16,6 +16,7 @@ import python from './python'; import deno from './deno'; import laravel from './laravel'; import heroku from './heroku'; +import compose from './compose' export { node, @@ -35,5 +36,6 @@ export { python, deno, laravel, - heroku + heroku, + compose }; diff --git a/apps/api/src/lib/common.ts b/apps/api/src/lib/common.ts index 32c9468a6..bb52979ec 100644 --- a/apps/api/src/lib/common.ts +++ b/apps/api/src/lib/common.ts @@ -598,7 +598,7 @@ export async function executeDockerCmd({ debug, buildId, applicationId, dockerId command = command.replace(/docker compose/gi, 'docker-compose'); } } - if (command.startsWith(`docker build --progress plain`) || command.startsWith(`pack build`)) { + if (command.startsWith(`docker build`) || command.startsWith(`pack build`)|| command.startsWith(`docker compose build`)) { return await asyncExecShellStream({ debug, buildId, applicationId, command, engine }); } return await execaCommand(command, { env: { DOCKER_BUILDKIT: "1", DOCKER_HOST: engine }, shell: true }) diff --git a/apps/ui/src/lib/templates.ts b/apps/ui/src/lib/templates.ts index 671bf9d5a..5219879e5 100644 --- a/apps/ui/src/lib/templates.ts +++ b/apps/ui/src/lib/templates.ts @@ -235,6 +235,14 @@ export const buildPacks = [ color: 'bg-sky-700', isCoolifyBuildPack: true, }, + { + name: 'compose', + type: 'base', + fancyName: 'Docker Compose', + hoverColor: 'hover:bg-sky-700', + color: 'bg-sky-700', + isCoolifyBuildPack: true, + }, { name: 'svelte', type: 'specific', From d8206c0e3ecc20f06d5179b0601f6d952d78549b Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 5 Oct 2022 15:34:52 +0200 Subject: [PATCH 04/34] wip: docker compose --- .../migration.sql | 3 + .../migration.sql | 2 + apps/api/prisma/schema.prisma | 77 +- apps/api/src/jobs/deployApplication.ts | 251 +++--- apps/api/src/lib/buildPacks/common.ts | 1 + apps/api/src/lib/buildPacks/compose.ts | 107 ++- .../routes/api/v1/applications/handlers.ts | 13 +- .../src/routes/api/v1/applications/types.ts | 5 +- apps/ui/package.json | 1 + apps/ui/src/lib/templates.ts | 13 +- .../[id]/configuration/_BuildPack.svelte | 12 +- .../[id]/configuration/buildpack.svelte | 98 ++- .../src/routes/applications/[id]/index.svelte | 754 ++++++++++-------- docker-compose-dev.yaml | 4 + package.json | 4 +- pnpm-lock.yaml | 2 + 16 files changed, 831 insertions(+), 516 deletions(-) create mode 100644 apps/api/prisma/migrations/20221005120323_initial_docker_compose/migration.sql create mode 100644 apps/api/prisma/migrations/20221005132352_docker_compose_configuration/migration.sql diff --git a/apps/api/prisma/migrations/20221005120323_initial_docker_compose/migration.sql b/apps/api/prisma/migrations/20221005120323_initial_docker_compose/migration.sql new file mode 100644 index 000000000..bb93e1aaf --- /dev/null +++ b/apps/api/prisma/migrations/20221005120323_initial_docker_compose/migration.sql @@ -0,0 +1,3 @@ +-- AlterTable +ALTER TABLE "Application" ADD COLUMN "dockerComposeFile" TEXT; +ALTER TABLE "Application" ADD COLUMN "dockerComposeFileLocation" TEXT; diff --git a/apps/api/prisma/migrations/20221005132352_docker_compose_configuration/migration.sql b/apps/api/prisma/migrations/20221005132352_docker_compose_configuration/migration.sql new file mode 100644 index 000000000..e7368dc1a --- /dev/null +++ b/apps/api/prisma/migrations/20221005132352_docker_compose_configuration/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "Application" ADD COLUMN "dockerComposeConfiguration" TEXT; diff --git a/apps/api/prisma/schema.prisma b/apps/api/prisma/schema.prisma index eba0ac215..d782bceae 100644 --- a/apps/api/prisma/schema.prisma +++ b/apps/api/prisma/schema.prisma @@ -94,43 +94,46 @@ model TeamInvitation { } model Application { - id String @id @default(cuid()) - name String - fqdn String? - repository String? - configHash String? - branch String? - buildPack String? - projectId Int? - port Int? - exposePort Int? - installCommand String? - buildCommand String? - startCommand String? - baseDirectory String? - publishDirectory String? - deploymentType String? - phpModules String? - pythonWSGI String? - pythonModule String? - pythonVariable String? - dockerFileLocation String? - denoMainFile String? - denoOptions String? - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - destinationDockerId String? - gitSourceId String? - baseImage String? - baseBuildImage String? - gitSource GitSource? @relation(fields: [gitSourceId], references: [id]) - destinationDocker DestinationDocker? @relation(fields: [destinationDockerId], references: [id]) - persistentStorage ApplicationPersistentStorage[] - settings ApplicationSettings? - secrets Secret[] - teams Team[] - connectedDatabase ApplicationConnectedDatabase? - previewApplication PreviewApplication[] + id String @id @default(cuid()) + name String + fqdn String? + repository String? + configHash String? + branch String? + buildPack String? + projectId Int? + port Int? + exposePort Int? + installCommand String? + buildCommand String? + startCommand String? + baseDirectory String? + publishDirectory String? + deploymentType String? + phpModules String? + pythonWSGI String? + pythonModule String? + pythonVariable String? + dockerFileLocation String? + denoMainFile String? + denoOptions String? + dockerComposeFile String? + dockerComposeFileLocation String? + dockerComposeConfiguration String? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + destinationDockerId String? + gitSourceId String? + baseImage String? + baseBuildImage String? + gitSource GitSource? @relation(fields: [gitSourceId], references: [id]) + destinationDocker DestinationDocker? @relation(fields: [destinationDockerId], references: [id]) + persistentStorage ApplicationPersistentStorage[] + settings ApplicationSettings? + secrets Secret[] + teams Team[] + connectedDatabase ApplicationConnectedDatabase? + previewApplication PreviewApplication[] } model PreviewApplication { diff --git a/apps/api/src/jobs/deployApplication.ts b/apps/api/src/jobs/deployApplication.ts index a2790aadb..f25c4bbc3 100644 --- a/apps/api/src/jobs/deployApplication.ts +++ b/apps/api/src/jobs/deployApplication.ts @@ -212,17 +212,37 @@ import * as buildpacks from '../lib/buildPacks'; // } await copyBaseConfigurationFiles(buildPack, workdir, buildId, applicationId, baseImage); + const labels = makeLabelForStandaloneApplication({ + applicationId, + fqdn, + name, + type, + pullmergeRequestId, + buildPack, + repository, + branch, + projectId, + port: exposePort ? `${exposePort}:${port}` : port, + commit, + installCommand, + buildCommand, + startCommand, + baseDirectory, + publishDirectory + }); if (forceRebuild) deployNeeded = true if (!imageFound || deployNeeded) { - // if (true) { if (buildpacks[buildPack]) await buildpacks[buildPack]({ dockerId: destinationDocker.id, + network: destinationDocker.network, buildId, applicationId, domain, name, type, + volumes, + labels, pullmergeRequestId, buildPack, repository, @@ -248,7 +268,7 @@ import * as buildpacks from '../lib/buildPacks'; denoOptions, baseImage, baseBuildImage, - deploymentType + deploymentType, }); else { await saveBuildLog({ line: `Build pack ${buildPack} not found`, buildId, applicationId }); @@ -257,112 +277,137 @@ import * as buildpacks from '../lib/buildPacks'; } else { await saveBuildLog({ line: 'Build image already available - no rebuild required.', buildId, applicationId }); } - try { - await executeDockerCmd({ dockerId: destinationDocker.id, command: `docker stop -t 0 ${imageId}` }) - await executeDockerCmd({ dockerId: destinationDocker.id, command: `docker rm ${imageId}` }) - } catch (error) { - // - } - const envs = [ - `PORT=${port}` - ]; - if (secrets.length > 0) { - secrets.forEach((secret) => { - if (pullmergeRequestId) { - const isSecretFound = secrets.filter(s => s.name === secret.name && s.isPRMRSecret) - if (isSecretFound.length > 0) { - envs.push(`${secret.name}=${isSecretFound[0].value}`); - } else { - envs.push(`${secret.name}=${secret.value}`); - } - } else { - if (!secret.isPRMRSecret) { - envs.push(`${secret.name}=${secret.value}`); - } + + if (buildPack === 'compose') { + try { + await executeDockerCmd({ + dockerId: destinationDockerId, + command: `docker ps -a --filter 'label=coolify.applicationId=${applicationId}' --format {{.ID}}|xargs -r -n 1 docker stop -t 0` + }) + await executeDockerCmd({ + dockerId: destinationDockerId, + command: `docker ps -a --filter 'label=coolify.applicationId=${applicationId}' --format {{.ID}}|xargs -r -n 1 docker rm --force` + }) + } catch (error) { + // + } + try { + await executeDockerCmd({ debug, buildId, applicationId, dockerId: destinationDocker.id, command: `docker compose --project-directory ${workdir} up -d` }) + await saveBuildLog({ line: 'Deployment successful!', buildId, applicationId }); + await saveBuildLog({ line: 'Proxy will be updated shortly.', buildId, applicationId }); + await prisma.build.update({ where: { id: buildId }, data: { status: 'success' } }); + await prisma.application.update({ + where: { id: applicationId }, + data: { configHash: currentHash } + }); + } catch (error) { + await saveBuildLog({ line: error, buildId, applicationId }); + const foundBuild = await prisma.build.findUnique({ where: { id: buildId } }) + if (foundBuild) { + await prisma.build.update({ + where: { id: buildId }, + data: { + status: 'failed' + } + }); } - }); - } - await fs.writeFile(`${workdir}/.env`, envs.join('\n')); - const labels = makeLabelForStandaloneApplication({ - applicationId, - fqdn, - name, - type, - pullmergeRequestId, - buildPack, - repository, - branch, - projectId, - port: exposePort ? `${exposePort}:${port}` : port, - commit, - installCommand, - buildCommand, - startCommand, - baseDirectory, - publishDirectory - }); - let envFound = false; - try { - envFound = !!(await fs.stat(`${workdir}/.env`)); - } catch (error) { - // - } - try { - await saveBuildLog({ line: 'Deployment started.', buildId, applicationId }); - const composeVolumes = volumes.map((volume) => { - return { - [`${volume.split(':')[0]}`]: { - name: volume.split(':')[0] - } - }; - }); - const composeFile = { - version: '3.8', - services: { - [imageId]: { - image: `${applicationId}:${tag}`, - container_name: imageId, - volumes, - env_file: envFound ? [`${workdir}/.env`] : [], - labels, - depends_on: [], - expose: [port], - ...(exposePort ? { ports: [`${exposePort}:${port}`] } : {}), - // logging: { - // driver: 'fluentd', - // }, - ...defaultComposeConfiguration(destinationDocker.network), - } - }, - networks: { - [destinationDocker.network]: { - external: true - } - }, - volumes: Object.assign({}, ...composeVolumes) - }; - await fs.writeFile(`${workdir}/docker-compose.yml`, yaml.dump(composeFile)); - await executeDockerCmd({ dockerId: destinationDocker.id, command: `docker compose --project-directory ${workdir} up -d` }) - await saveBuildLog({ line: 'Deployment successful!', buildId, applicationId }); - } catch (error) { - await saveBuildLog({ line: error, buildId, applicationId }); - const foundBuild = await prisma.build.findUnique({ where: { id: buildId } }) - if (foundBuild) { - await prisma.build.update({ - where: { id: buildId }, - data: { - status: 'failed' + throw new Error(error); + } + + } else { + try { + await executeDockerCmd({ + dockerId: destinationDockerId, + command: `docker ps -a --filter 'label=com.docker.compose.service=${applicationId}' --format {{.ID}}|xargs -r -n 1 docker stop -t 0` + }) + await executeDockerCmd({ + dockerId: destinationDockerId, + command: `docker ps -a --filter 'label=com.docker.compose.service=${applicationId}' --format {{.ID}}|xargs -r -n 1 docker rm --force` + }) + } catch (error) { + // + } + const envs = [ + `PORT=${port}` + ]; + if (secrets.length > 0) { + secrets.forEach((secret) => { + if (pullmergeRequestId) { + const isSecretFound = secrets.filter(s => s.name === secret.name && s.isPRMRSecret) + if (isSecretFound.length > 0) { + envs.push(`${secret.name}=${isSecretFound[0].value}`); + } else { + envs.push(`${secret.name}=${secret.value}`); + } + } else { + if (!secret.isPRMRSecret) { + envs.push(`${secret.name}=${secret.value}`); + } } }); } - throw new Error(error); + await fs.writeFile(`${workdir}/.env`, envs.join('\n')); + + let envFound = false; + try { + envFound = !!(await fs.stat(`${workdir}/.env`)); + } catch (error) { + // + } + try { + await saveBuildLog({ line: 'Deployment started.', buildId, applicationId }); + const composeVolumes = volumes.map((volume) => { + return { + [`${volume.split(':')[0]}`]: { + name: volume.split(':')[0] + } + }; + }); + const composeFile = { + version: '3.8', + services: { + [imageId]: { + image: `${applicationId}:${tag}`, + container_name: imageId, + volumes, + env_file: envFound ? [`${workdir}/.env`] : [], + labels, + depends_on: [], + expose: [port], + ...(exposePort ? { ports: [`${exposePort}:${port}`] } : {}), + ...defaultComposeConfiguration(destinationDocker.network), + } + }, + networks: { + [destinationDocker.network]: { + external: true + } + }, + volumes: Object.assign({}, ...composeVolumes) + }; + await fs.writeFile(`${workdir}/docker-compose.yml`, yaml.dump(composeFile)); + await executeDockerCmd({ dockerId: destinationDocker.id, command: `docker compose --project-directory ${workdir} up -d` }) + await saveBuildLog({ line: 'Deployment successful!', buildId, applicationId }); + } catch (error) { + await saveBuildLog({ line: error, buildId, applicationId }); + const foundBuild = await prisma.build.findUnique({ where: { id: buildId } }) + if (foundBuild) { + await prisma.build.update({ + where: { id: buildId }, + data: { + status: 'failed' + } + }); + } + throw new Error(error); + } + await saveBuildLog({ line: 'Proxy will be updated shortly.', buildId, applicationId }); + await prisma.build.update({ where: { id: buildId }, data: { status: 'success' } }); + if (!pullmergeRequestId) await prisma.application.update({ + where: { id: applicationId }, + data: { configHash: currentHash } + }); } - await saveBuildLog({ line: 'Proxy will be updated shortly.', buildId, applicationId }); - await prisma.build.update({ where: { id: buildId }, data: { status: 'success' } }); - if (!pullmergeRequestId) await prisma.application.update({ - where: { id: applicationId }, - data: { configHash: currentHash } - }); } } catch (error) { diff --git a/apps/api/src/lib/buildPacks/common.ts b/apps/api/src/lib/buildPacks/common.ts index 98a387f2f..5fe971cc6 100644 --- a/apps/api/src/lib/buildPacks/common.ts +++ b/apps/api/src/lib/buildPacks/common.ts @@ -634,6 +634,7 @@ export function makeLabelForStandaloneApplication({ return [ 'coolify.managed=true', `coolify.version=${version}`, + `coolify.applicationId=${applicationId}`, `coolify.type=standalone-application`, `coolify.configuration=${base64Encode( JSON.stringify({ diff --git a/apps/api/src/lib/buildPacks/compose.ts b/apps/api/src/lib/buildPacks/compose.ts index 30af1f10b..71cb72b21 100644 --- a/apps/api/src/lib/buildPacks/compose.ts +++ b/apps/api/src/lib/buildPacks/compose.ts @@ -1,34 +1,99 @@ import { promises as fs } from 'fs'; -import { executeDockerCmd } from '../common'; -import { buildImage } from './common'; +import { defaultComposeConfiguration, executeDockerCmd } from '../common'; +import { buildImage, saveBuildLog } from './common'; import yaml from 'js-yaml'; +import { getSecrets } from '../../routes/api/v1/applications/handlers'; export default async function (data) { - let { - applicationId, + let { + applicationId, + debug, + buildId, dockerId, - debug, - tag, - workdir, - buildId, - baseDirectory, - secrets, - pullmergeRequestId, - dockerFileLocation - } = data - const file = `${workdir}${baseDirectory}/docker-compose.yml`; - const dockerComposeRaw = await fs.readFile(`${file}`, 'utf8') + network, + volumes, + labels, + workdir, + baseDirectory, + secrets, + pullmergeRequestId, + port + } = data + const fileYml = `${workdir}${baseDirectory}/docker-compose.yml`; + const fileYaml = `${workdir}${baseDirectory}/docker-compose.yaml`; + let dockerComposeRaw = null; + let isYml = false; + try { + dockerComposeRaw = await fs.readFile(`${fileYml}`, 'utf8') + isYml = true + } catch (error) { } + try { + dockerComposeRaw = await fs.readFile(`${fileYaml}`, 'utf8') + } catch (error) { } + + if (!dockerComposeRaw) { + throw ('docker-compose.yml or docker-compose.yaml are not found!'); + } const dockerComposeYaml = yaml.load(dockerComposeRaw) if (!dockerComposeYaml.services) { throw 'No Services found in docker-compose file.' } + const envs = [ + `PORT=${port}` + ]; + if (getSecrets.length > 0) { + secrets.forEach((secret) => { + if (secret.isBuildSecret) { + if (pullmergeRequestId) { + const isSecretFound = secrets.filter(s => s.name === secret.name && s.isPRMRSecret) + if (isSecretFound.length > 0) { + envs.push(`${secret.name}=${isSecretFound[0].value}`); + } else { + envs.push(`${secret.name}=${secret.value}`); + } + } else { + if (!secret.isPRMRSecret) { + envs.push(`${secret.name}=${secret.value}`); + } + } + } + }); + } + await fs.writeFile(`${workdir}/.env`, envs.join('\n')); + let envFound = false; + try { + envFound = !!(await fs.stat(`${workdir}/.env`)); + } catch (error) { + // + } + const composeVolumes = volumes.map((volume) => { + return { + [`${volume.split(':')[0]}`]: { + name: volume.split(':')[0] + } + }; + }); + let networks = {} for (let [key, value] of Object.entries(dockerComposeYaml.services)) { value['container_name'] = `${applicationId}-${key}` - console.log({key, value}); + value['env_file'] = envFound ? [`${workdir}/.env`] : [] + value['labels'] = labels + value['volumes'] = volumes + if (value['networks']?.length > 0) { + value['networks'].forEach((network) => { + networks[network] = { + name: network + } + }) + } + value['networks'] = [...value['networks'] || '', network] + dockerComposeYaml.services[key] = { ...dockerComposeYaml.services[key], restart: defaultComposeConfiguration(network).restart, deploy: defaultComposeConfiguration(network).deploy } } - - throw 'Halting' - // await executeDockerCmd({ debug, buildId, applicationId, dockerId, command: `docker compose --project-directory ${workdir} pull` }) - // await executeDockerCmd({ debug, buildId, applicationId, dockerId, command: `docker compose --project-directory ${workdir} build --progress plain --pull` }) - // await executeDockerCmd({ debug, buildId, applicationId, dockerId, command: `docker compose --project-directory ${workdir} up -d` }) + dockerComposeYaml['volumes'] = Object.assign({ ...dockerComposeYaml['volumes'] }, ...composeVolumes) + dockerComposeYaml['networks'] = Object.assign({ ...networks }, { [network]: { external: true } }) + await fs.writeFile(`${workdir}/docker-compose.${isYml ? 'yml' : 'yaml'}`, yaml.dump(dockerComposeYaml)); + await executeDockerCmd({ debug, buildId, applicationId, dockerId, command: `docker compose --project-directory ${workdir} pull` }) + await saveBuildLog({ line: 'Pulling images from Compose file.', buildId, applicationId }); + await executeDockerCmd({ debug, buildId, applicationId, dockerId, command: `docker compose --project-directory ${workdir} build --progress plain` }) + await saveBuildLog({ line: 'Building images from Compose file.', buildId, applicationId }); } diff --git a/apps/api/src/routes/api/v1/applications/handlers.ts b/apps/api/src/routes/api/v1/applications/handlers.ts index be06a3519..65b0750b2 100644 --- a/apps/api/src/routes/api/v1/applications/handlers.ts +++ b/apps/api/src/routes/api/v1/applications/handlers.ts @@ -289,13 +289,16 @@ export async function saveApplication(request: FastifyRequest, baseImage, baseBuildImage, deploymentType, - baseDatabaseBranch + baseDatabaseBranch, + dockerComposeFile, + dockerComposeFileLocation, + dockerComposeConfiguration } = request.body + console.log({dockerComposeConfiguration}) if (port) port = Number(port); if (exposePort) { exposePort = Number(exposePort); } - const { destinationDocker: { engine, remoteEngine, remoteIpAddress }, exposePort: configuredPort } = await prisma.application.findUnique({ where: { id }, include: { destinationDocker: true } }) if (exposePort) await checkExposedPort({ id, configuredPort, exposePort, engine, remoteEngine, remoteIpAddress }) if (denoOptions) denoOptions = denoOptions.trim(); @@ -324,6 +327,9 @@ export async function saveApplication(request: FastifyRequest, baseImage, baseBuildImage, deploymentType, + dockerComposeFile, + dockerComposeFileLocation, + dockerComposeConfiguration, ...defaultConfiguration, connectedDatabase: { update: { hostedDatabaseDBName: baseDatabaseBranch } } } @@ -342,6 +348,9 @@ export async function saveApplication(request: FastifyRequest, baseImage, baseBuildImage, deploymentType, + dockerComposeFile, + dockerComposeFileLocation, + dockerComposeConfiguration, ...defaultConfiguration } }); diff --git a/apps/api/src/routes/api/v1/applications/types.ts b/apps/api/src/routes/api/v1/applications/types.ts index 443deb00f..14ba30b78 100644 --- a/apps/api/src/routes/api/v1/applications/types.ts +++ b/apps/api/src/routes/api/v1/applications/types.ts @@ -21,7 +21,10 @@ export interface SaveApplication extends OnlyId { baseImage: string, baseBuildImage: string, deploymentType: string, - baseDatabaseBranch: string + baseDatabaseBranch: string, + dockerComposeFile: string, + dockerComposeFileLocation: string, + dockerComposeConfiguration: string } } export interface SaveApplicationSettings extends OnlyId { diff --git a/apps/ui/package.json b/apps/ui/package.json index 805539a5d..fdf37b8aa 100644 --- a/apps/ui/package.json +++ b/apps/ui/package.json @@ -48,6 +48,7 @@ "daisyui": "2.24.2", "dayjs": "1.11.5", "js-cookie": "3.0.1", + "js-yaml": "4.1.0", "p-limit": "4.0.0", "svelte-file-dropzone": "^1.0.0", "svelte-select": "4.4.7", diff --git a/apps/ui/src/lib/templates.ts b/apps/ui/src/lib/templates.ts index 5219879e5..d71fd9323 100644 --- a/apps/ui/src/lib/templates.ts +++ b/apps/ui/src/lib/templates.ts @@ -29,7 +29,7 @@ export function findBuildPack(pack: string, packageManager = 'npm') { port: 80 }; } - if (pack === 'docker') { + if (pack === 'docker' || pack === 'compose') { return { ...metaData, installCommand: null, @@ -39,6 +39,7 @@ export function findBuildPack(pack: string, packageManager = 'npm') { port: null }; } + if (pack === 'svelte') { return { ...metaData, @@ -236,13 +237,13 @@ export const buildPacks = [ isCoolifyBuildPack: true, }, { - name: 'compose', + name: 'compose', type: 'base', fancyName: 'Docker Compose', hoverColor: 'hover:bg-sky-700', color: 'bg-sky-700', isCoolifyBuildPack: true, - }, + }, { name: 'svelte', type: 'specific', @@ -357,14 +358,14 @@ export const buildPacks = [ color: 'bg-green-700', isCoolifyBuildPack: true, }, - { - name: 'heroku', + { + name: 'heroku', type: 'base', fancyName: 'Heroku', hoverColor: 'hover:bg-purple-700', color: 'bg-purple-700', isHerokuBuildPack: true, - } + } ]; export const scanningTemplates = { '@sveltejs/kit': { diff --git a/apps/ui/src/routes/applications/[id]/configuration/_BuildPack.svelte b/apps/ui/src/routes/applications/[id]/configuration/_BuildPack.svelte index 13db5a24a..f7771e026 100644 --- a/apps/ui/src/routes/applications/[id]/configuration/_BuildPack.svelte +++ b/apps/ui/src/routes/applications/[id]/configuration/_BuildPack.svelte @@ -14,6 +14,8 @@ export let foundConfig: any; export let scanning: any; export let packageManager: any; + export let dockerComposeFile: any = null; + export let dockerComposeFileLocation: string | null = null; async function handleSubmit(name: string) { try { @@ -25,10 +27,12 @@ delete tempBuildPack.fancyName; delete tempBuildPack.color; delete tempBuildPack.hoverColor; - - if (foundConfig?.buildPack !== name) { - await post(`/applications/${id}`, { ...tempBuildPack, buildPack: name }); - } + await post(`/applications/${id}`, { + ...tempBuildPack, + buildPack: name, + dockerComposeFile, + dockerComposeFileLocation + }); await post(`/applications/${id}/configuration/buildpack`, { buildPack: name }); return await goto(from || `/applications/${id}`); } catch (error) { diff --git a/apps/ui/src/routes/applications/[id]/configuration/buildpack.svelte b/apps/ui/src/routes/applications/[id]/configuration/buildpack.svelte index 3a4c41115..ad9b70be4 100644 --- a/apps/ui/src/routes/applications/[id]/configuration/buildpack.svelte +++ b/apps/ui/src/routes/applications/[id]/configuration/buildpack.svelte @@ -34,12 +34,15 @@ import { buildPacks, findBuildPack, scanningTemplates } from '$lib/templates'; import { errorNotification } from '$lib/common'; import BuildPack from './_BuildPack.svelte'; + import yaml from 'js-yaml'; const { id } = $page.params; - let scanning = true; + let scanning: boolean = true; let foundConfig: any = null; - let packageManager = 'npm'; + let packageManager: string = 'npm'; + let dockerComposeFile: any = null; + let dockerComposeFileLocation: string | null = null; export let apiUrl: any; export let projectId: any; @@ -60,10 +63,14 @@ } } } - async function scanRepository(): Promise { + async function scanRepository(isPublicRepository: boolean): Promise { try { if (type === 'gitlab') { - const files = await get(`${apiUrl}/v4/projects/${projectId}/repository/tree`, { + if (isPublicRepository) { + return; + } + const url = isPublicRepository ? `` : `/v4/projects/${projectId}/repository/tree`; + const files = await get(`${apiUrl}${url}`, { Authorization: `Bearer ${$appSession.tokens.gitlab}` }); const packageJson = files.find( @@ -82,6 +89,14 @@ (file: { name: string; type: string }) => file.name === 'Dockerfile' && file.type === 'blob' ); + const dockerComposeFileYml = files.find( + (file: { name: string; type: string }) => + file.name === 'docker-compose.yml' && file.type === 'blob' + ); + const dockerComposeFileYaml = files.find( + (file: { name: string; type: string }) => + file.name === 'docker-compose.yaml' && file.type === 'blob' + ); const cargoToml = files.find( (file: { name: string; type: string }) => file.name === 'Cargo.toml' && file.type === 'blob' @@ -105,11 +120,12 @@ const laravel = files.find( (file: { name: string; type: string }) => file.name === 'artisan' && file.type === 'blob' ); - if (yarnLock) packageManager = 'yarn'; if (pnpmLock) packageManager = 'pnpm'; - if (dockerfile) { + if (dockerComposeFileYml || dockerComposeFileYaml) { + foundConfig = findBuildPack('dockercompose', packageManager); + } else if (dockerfile) { foundConfig = findBuildPack('docker', packageManager); } else if (packageJson && !laravel) { const path = packageJson.path; @@ -135,8 +151,13 @@ foundConfig = findBuildPack('node', packageManager); } } else if (type === 'github') { + const headers = isPublicRepository + ? {} + : { + Authorization: `token ${$appSession.tokens.github}` + }; const files = await get(`${apiUrl}/repos/${repository}/contents?ref=${branch}`, { - Authorization: `Bearer ${$appSession.tokens.github}`, + ...headers, Accept: 'application/vnd.github.v2.json' }); const packageJson = files.find( @@ -155,6 +176,14 @@ (file: { name: string; type: string }) => file.name === 'Dockerfile' && file.type === 'file' ); + const dockerComposeFileYml = files.find( + (file: { name: string; type: string }) => + file.name === 'docker-compose.yml' && file.type === 'file' + ); + const dockerComposeFileYaml = files.find( + (file: { name: string; type: string }) => + file.name === 'docker-compose.yaml' && file.type === 'file' + ); const cargoToml = files.find( (file: { name: string; type: string }) => file.name === 'Cargo.toml' && file.type === 'file' @@ -182,7 +211,25 @@ if (yarnLock) packageManager = 'yarn'; if (pnpmLock) packageManager = 'pnpm'; - if (dockerfile) { + if (dockerComposeFileYml || dockerComposeFileYaml) { + foundConfig = findBuildPack('compose', packageManager); + const data = await get( + `${apiUrl}/repos/${repository}/contents/${ + dockerComposeFileYml ? 'docker-compose.yml' : 'docker-compose.yaml' + }?ref=${branch}`, + { + ...headers, + Accept: 'application/vnd.github.v2.json' + } + ); + if (data?.content) { + const content = atob(data.content); + dockerComposeFile = JSON.stringify(yaml.load(content) || null); + dockerComposeFileLocation = dockerComposeFileYml + ? 'docker-compose.yml' + : 'docker-compose.yaml'; + } + } else if (dockerfile) { foundConfig = findBuildPack('docker', packageManager); } else if (packageJson && !laravel) { const data: any = await get(`${packageJson.git_url}`, { @@ -237,7 +284,7 @@ if (error.message === 'Bad credentials') { const { token } = await get(`/applications/${id}/configuration/githubToken`); $appSession.tokens.github = token; - return await scanRepository(); + return await scanRepository(isPublicRepository); } return errorNotification(error); } finally { @@ -246,11 +293,7 @@ } } onMount(async () => { - if (!isPublicRepository) { - await scanRepository(); - } else { - scanning = false; - } + await scanRepository(isPublicRepository); }); @@ -266,7 +309,12 @@
{#each buildPacks.filter((bp) => bp.isHerokuBuildPack === true) as buildPack}
- +
{/each}
@@ -274,9 +322,16 @@
Coolify Base
- {#each buildPacks.filter((bp) => bp.isCoolifyBuildPack === true && bp.type ==='base') as buildPack} + {#each buildPacks.filter((bp) => bp.isCoolifyBuildPack === true && bp.type === 'base') as buildPack}
- +
{/each}
@@ -284,9 +339,14 @@
Coolify Specific
- {#each buildPacks.filter((bp) => bp.isCoolifyBuildPack === true && bp.type ==='specific') as buildPack} + {#each buildPacks.filter((bp) => bp.isCoolifyBuildPack === true && bp.type === 'specific') as buildPack}
- +
{/each}
diff --git a/apps/ui/src/routes/applications/[id]/index.svelte b/apps/ui/src/routes/applications/[id]/index.svelte index 5f09c7db6..c5ff925f0 100644 --- a/apps/ui/src/routes/applications/[id]/index.svelte +++ b/apps/ui/src/routes/applications/[id]/index.svelte @@ -29,7 +29,7 @@ export let application: any; export let settings: any; import { page } from '$app/stores'; - import { onDestroy, onMount } from 'svelte'; + import { onMount } from 'svelte'; import Select from 'svelte-select'; import { get, post } from '$lib/api'; import cuid from 'cuid'; @@ -45,10 +45,9 @@ import { t } from '$lib/translations'; import { errorNotification, getDomain, notNodeDeployments, staticDeployments } from '$lib/common'; import Setting from '$lib/components/Setting.svelte'; - import Tooltip from '$lib/components/Tooltip.svelte'; import Explainer from '$lib/components/Explainer.svelte'; import { goto } from '$app/navigation'; - import { fade } from 'svelte/transition'; + import yaml from 'js-yaml'; const { id } = $page.params; @@ -58,6 +57,10 @@ let loading = false; let fqdnEl: any = null; let forceSave = false; + let isPublicRepository = application.settings.isPublicRepository; + let apiUrl = application.gitSource.apiUrl; + let branch = application.branch; + let repository = application.repository; let debug = application.settings.debug; let previews = application.settings.previews; let dualCerts = application.settings.dualCerts; @@ -66,6 +69,11 @@ let isBot = application.settings.isBot; let isDBBranching = application.settings.isDBBranching; + let dockerComposeFile = JSON.parse(application.dockerComposeFile) || null; + let dockerComposeServices: any[] = []; + let dockerComposeFileLocation = application.dockerComposeFileLocation; + let dockerComposeConfiguration = JSON.parse(application.dockerComposeConfiguration) || {}; + let baseDatabaseBranch: any = application?.connectedDatabase?.hostedDatabaseDBName || null; let nonWWWDomain = application.fqdn && getDomain(application.fqdn).replace(/^www\./, ''); let isHttps = application.fqdn && application.fqdn.startsWith('https://'); @@ -86,6 +94,26 @@ label: 'Uvicorn' } ]; + + function normalizeDockerServices(services: any[]) { + const tempdockerComposeServices = []; + for (const [name, data] of Object.entries(services)) { + tempdockerComposeServices.push({ + name, + data + }); + } + for (const service of tempdockerComposeServices) { + if (!dockerComposeConfiguration[service.name]) { + dockerComposeConfiguration[service.name] = {}; + } + } + return tempdockerComposeServices; + } + if (dockerComposeFile?.services) { + dockerComposeServices = normalizeDockerServices(dockerComposeFile.services); + } + function containerClass() { return 'text-white bg-transparent font-thin px-0 w-full border border-dashed border-coolgray-200'; } @@ -214,7 +242,11 @@ dualCerts, exposePort: application.exposePort })); - await post(`/applications/${id}`, { ...application, baseDatabaseBranch }); + await post(`/applications/${id}`, { + ...application, + baseDatabaseBranch, + dockerComposeConfiguration: JSON.stringify(dockerComposeConfiguration) + }); setLocation(application, settings); $isDeploymentEnabled = checkIfDeploymentEnabledApplications($appSession.isAdmin, application); @@ -281,6 +313,36 @@ return false; } } + async function reloadCompose() { + try { + const headers = isPublicRepository + ? {} + : { + Authorization: `token ${$appSession.tokens.github}` + }; + const data = await get( + `${apiUrl}/repos/${repository}/contents/${dockerComposeFileLocation}?ref=${branch}`, + { + ...headers, + Accept: 'application/vnd.github.v2.json' + } + ); + if (data?.content) { + const content = atob(data.content); + let dockerComposeFileContent = JSON.stringify(yaml.load(content) || null); + let dockerComposeFileContentJSON = JSON.parse(dockerComposeFileContent); + dockerComposeServices = normalizeDockerServices(dockerComposeFileContentJSON?.services); + application.dockerComposeFile = dockerComposeFileContent; + await handleSubmit(); + } + addToast({ + message: 'Compose file reloaded.', + type: 'success' + }); + } catch (error) { + errorNotification(error); + } + }
@@ -372,18 +434,20 @@ />
-
- changeSettings('isBot')} - title="Is your application a bot?" - description="You can deploy applications without domains or make them to listen on the Exposed Port.

Useful to host Twitch bots, regular jobs, or anything that does not require an incoming HTTP connection." - disabled={$status.application.isRunning} - /> -
- {#if !isBot} + {#if application.buildPack !== 'compose'} +
+ changeSettings('isBot')} + title="Is your application a bot?" + description="You can deploy applications without domains or make them to listen on the Exposed Port.

Useful to host Twitch bots, regular jobs, or anything that does not require an incoming HTTP connection." + disabled={$status.application.isRunning} + /> +
+ {/if} + {#if !isBot && application.buildPack !== 'compose'}
- {#if isHttps} + {#if isHttps && application.buildPack !== 'compose'}
- -
Build & Deploy
-
- {#if application.buildCommand || application.buildPack === 'rust' || application.buildPack === 'laravel'} -
- -
- -
-
- {/if} - {#if application.buildPack !== 'docker' && (application.buildPack === 'nextjs' || application.buildPack === 'nuxtjs')} -
- -
- +
- {#if isDBBranching} - + - {#if application.connectedDatabase} -
- - -
-
- Connected to {application.connectedDatabase.databaseId} -
+
+ +
+
+ {/if} + {#if $features.beta} + {#if !application.settings.isBot && !application.settings.isPublicRepository} +
+ changeSettings('isDBBranching')} + title="Enable DB Branching" + description="Enable DB Branching" + /> +
+ {#if isDBBranching} + + {#if application.connectedDatabase} +
+ + +
+
+ Connected to {application.connectedDatabase.databaseId} +
+ {/if} {/if} {/if} {/if} - {/if} - {#if application.buildPack === 'python'} -
- -
- -
- {#if application.pythonWSGI?.toLowerCase() === 'gunicorn'} + {#if application.buildPack === 'python'}
- + +
+
+ {#if application.pythonWSGI?.toLowerCase() === 'gunicorn'} +
+ + +
+ {/if} + {#if application.pythonWSGI?.toLowerCase() === 'uvicorn'} +
+ + +
+ {/if} {/if} - {#if application.pythonWSGI?.toLowerCase() === 'uvicorn'} -
- + {#if !staticDeployments.includes(application.buildPack)} +
+
{/if} - {/if} - {#if !staticDeployments.includes(application.buildPack)} -
- - -
- {/if} -
- - -
- {#if !notNodeDeployments.includes(application.buildPack)} -
- - -
-
- - -
-
- - -
- {/if} - {#if application.buildPack === 'deno'} -
- - -
-
-
+ {/if}
diff --git a/docker-compose-dev.yaml b/docker-compose-dev.yaml index 468d17efc..c9588c7a8 100644 --- a/docker-compose-dev.yaml +++ b/docker-compose-dev.yaml @@ -22,6 +22,10 @@ services: published: 3001 protocol: tcp mode: host + - target: 5555 + published: 5555 + protocol: tcp + mode: host volumes: - ./:/app - '/var/run/docker.sock:/var/run/docker.sock' diff --git a/package.json b/package.json index de44428ea..e692b61f6 100644 --- a/package.json +++ b/package.json @@ -8,14 +8,16 @@ "oc": "opencollective-setup", "translate": "pnpm run --filter i18n-converter translate", "db:studio": "pnpm run --filter api db:studio", + "db:studio:container": "docker exec coolify pnpm run --filter api db:studio", "db:push": "pnpm run --filter api db:push", "db:seed": "pnpm run --filter api db:seed", "db:migrate": "pnpm run --filter api db:migrate", + "db:migrate:container": "docker exec coolify pnpm run --filter api db:migrate", "format": "run-p -l -n format:*", "format:api": "NODE_ENV=development pnpm run --filter api format", "lint": "run-p -l -n lint:*", "lint:api": "NODE_ENV=development pnpm run --filter api lint", - "dev:container": "docker-compose -f docker-compose-dev.yaml up", + "dev:container": "docker-compose -f docker-compose-dev.yaml up || docker compose -f docker-compose-dev.yaml up", "dev": "run-p -l -n dev:api dev:ui", "dev:api": "NODE_ENV=development pnpm run --filter api dev", "dev:ui": "NODE_ENV=development pnpm run --filter ui dev", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0820847ab..bac8b5276 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -159,6 +159,7 @@ importers: flowbite: 1.5.2 flowbite-svelte: 0.26.2 js-cookie: 3.0.1 + js-yaml: 4.1.0 p-limit: 4.0.0 postcss: 8.4.16 prettier: 2.7.1 @@ -181,6 +182,7 @@ importers: daisyui: 2.24.2_25hquoklqeoqwmt7fwvvcyxm5e dayjs: 1.11.5 js-cookie: 3.0.1 + js-yaml: 4.1.0 p-limit: 4.0.0 svelte-file-dropzone: 1.0.0 svelte-select: 4.4.7 From d27426fd8f7cff71bb32097dbfe407ffd7197650 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 6 Oct 2022 10:25:41 +0200 Subject: [PATCH 05/34] feat: docker compose support --- .../routes/api/v1/applications/handlers.ts | 83 +++++++++++--- .../src/routes/webhooks/traefik/handlers.ts | 65 ++++++++++- apps/ui/src/lib/store.ts | 6 +- .../routes/applications/[id]/__layout.svelte | 73 ++++++++---- .../src/routes/applications/[id]/index.svelte | 104 ++++++++++++++---- 5 files changed, 263 insertions(+), 68 deletions(-) diff --git a/apps/api/src/routes/api/v1/applications/handlers.ts b/apps/api/src/routes/api/v1/applications/handlers.ts index 65b0750b2..c5425e25f 100644 --- a/apps/api/src/routes/api/v1/applications/handlers.ts +++ b/apps/api/src/routes/api/v1/applications/handlers.ts @@ -110,23 +110,64 @@ export async function getApplicationStatus(request: FastifyRequest) { try { const { id } = request.params const { teamId } = request.user - let isRunning = false; - let isExited = false; - let isRestarting = false; + let payload = [] const application: any = await getApplicationFromDB(id, teamId); if (application?.destinationDockerId) { - const status = await checkContainer({ dockerId: application.destinationDocker.id, container: id }); - if (status?.found) { - isRunning = status.status.isRunning; - isExited = status.status.isExited; - isRestarting = status.status.isRestarting + if (application.buildPack === 'compose') { + const { stdout: containers } = await executeDockerCmd({ + dockerId: application.destinationDocker.id, + command: + `docker ps -a --filter "label=coolify.applicationId=${id}" --format '{{json .}}'` + }); + const containersArray = containers.trim().split('\n'); + if (containersArray.length > 0 && containersArray[0] !== '') { + for (const container of containersArray) { + let isRunning = false; + let isExited = false; + let isRestarting = false; + const containerObj = JSON.parse(container); + const status = containerObj.State + if (status === 'running') { + isRunning = true; + } + if (status === 'exited') { + isExited = true; + } + if (status === 'restarting') { + isRestarting = true; + } + payload.push({ + name: containerObj.Names, + status: { + isRunning, + isExited, + isRestarting + } + }) + } + } + } else { + let isRunning = false; + let isExited = false; + let isRestarting = false; + const status = await checkContainer({ dockerId: application.destinationDocker.id, container: id }); + if (status?.found) { + isRunning = status.status.isRunning; + isExited = status.status.isExited; + isRestarting = status.status.isRestarting + payload.push({ + name: id, + status: { + isRunning, + isExited, + isRestarting + } + }) + + } } } - return { - isRunning, - isRestarting, - isExited, - }; + return payload } catch ({ status, message }) { return errorHandler({ status, message }) } @@ -294,7 +335,6 @@ export async function saveApplication(request: FastifyRequest, dockerComposeFileLocation, dockerComposeConfiguration } = request.body - console.log({dockerComposeConfiguration}) if (port) port = Number(port); if (exposePort) { exposePort = Number(exposePort); @@ -515,6 +555,21 @@ export async function stopApplication(request: FastifyRequest, reply: Fa const application: any = await getApplicationFromDB(id, teamId); if (application?.destinationDockerId) { const { id: dockerId } = application.destinationDocker; + if (application.buildPack === 'compose') { + const { stdout: containers } = await executeDockerCmd({ + dockerId: application.destinationDocker.id, + command: + `docker ps -a --filter "label=coolify.applicationId=${id}" --format '{{json .}}'` + }); + const containersArray = containers.trim().split('\n'); + if (containersArray.length > 0 && containersArray[0] !== '') { + for (const container of containersArray) { + const containerObj = JSON.parse(container); + await removeContainer({ id: containerObj.ID, dockerId: application.destinationDocker.id }); + } + } + return + } const { found } = await checkContainer({ dockerId, container: id }); if (found) { await removeContainer({ id, dockerId: application.destinationDocker.id }); diff --git a/apps/api/src/routes/webhooks/traefik/handlers.ts b/apps/api/src/routes/webhooks/traefik/handlers.ts index 381869430..e6d4e474a 100644 --- a/apps/api/src/routes/webhooks/traefik/handlers.ts +++ b/apps/api/src/routes/webhooks/traefik/handlers.ts @@ -234,6 +234,8 @@ export async function traefikConfiguration(request, reply) { fqdn, id, port, + buildPack, + dockerComposeConfiguration, destinationDocker, destinationDockerId, settings: { previews, dualCerts, isCustomSSL } @@ -241,6 +243,33 @@ export async function traefikConfiguration(request, reply) { if (destinationDockerId) { const { network, id: dockerId } = destinationDocker; const isRunning = true; + if (buildPack === 'compose') { + const services = Object.entries(JSON.parse(dockerComposeConfiguration)) + for (const service of services) { + const [key, value] = service + const { port: customPort, fqdn } = value + if (fqdn) { + const domain = getDomain(fqdn); + const nakedDomain = domain.replace(/^www\./, ''); + const isHttps = fqdn.startsWith('https://'); + const isWWW = fqdn.includes('www.'); + data.applications.push({ + id: `${id}-${key}`, + container: `${id}-${key}`, + port: customPort ? customPort : port || 3000, + domain, + nakedDomain, + isRunning, + isHttps, + isWWW, + isDualCerts: dualCerts, + isCustomSSL + }); + } + } + continue; + } + if (fqdn) { const domain = getDomain(fqdn); const nakedDomain = domain.replace(/^www\./, ''); @@ -604,13 +633,41 @@ export async function remoteTraefikConfiguration(request: FastifyRequest fqdn, id, port, + buildPack, + dockerComposeConfiguration, destinationDocker, destinationDockerId, - settings: { previews, dualCerts } + settings: { previews, dualCerts, isCustomSSL } } = application; if (destinationDockerId) { const { id: dockerId, network } = destinationDocker; const isRunning = true; + if (buildPack === 'compose') { + const services = Object.entries(JSON.parse(dockerComposeConfiguration)) + for (const service of services) { + const [key, value] = service + const { port: customPort, fqdn } = value + if (fqdn) { + const domain = getDomain(fqdn); + const nakedDomain = domain.replace(/^www\./, ''); + const isHttps = fqdn.startsWith('https://'); + const isWWW = fqdn.includes('www.'); + data.applications.push({ + id: `${id}-${key}`, + container: `${id}-${key}`, + port: customPort ? customPort : port || 3000, + domain, + nakedDomain, + isRunning, + isHttps, + isWWW, + isDualCerts: dualCerts, + isCustomSSL + }); + } + } + continue; + } if (fqdn) { const domain = getDomain(fqdn); const nakedDomain = domain.replace(/^www\./, ''); @@ -626,7 +683,8 @@ export async function remoteTraefikConfiguration(request: FastifyRequest isRunning, isHttps, isWWW, - isDualCerts: dualCerts + isDualCerts: dualCerts, + isCustomSSL }); } if (previews) { @@ -649,7 +707,8 @@ export async function remoteTraefikConfiguration(request: FastifyRequest nakedDomain, isHttps, isWWW, - isDualCerts: dualCerts + isDualCerts: dualCerts, + isCustomSSL }); } } diff --git a/apps/ui/src/lib/store.ts b/apps/ui/src/lib/store.ts index aa3493ca5..0d399a11c 100644 --- a/apps/ui/src/lib/store.ts +++ b/apps/ui/src/lib/store.ts @@ -56,6 +56,7 @@ export const isDeploymentEnabled: Writable = writable(false); export function checkIfDeploymentEnabledApplications(isAdmin: boolean, application: any) { return ( isAdmin && + (application.buildPack === 'compose') || (application.fqdn || application.settings.isBot) && application.gitSource && application.repository && @@ -74,9 +75,8 @@ export function checkIfDeploymentEnabledServices(isAdmin: boolean, service: any) } export const status: Writable = writable({ application: { - isRunning: false, - isExited: false, - isRestarting: false, + statuses: [], + overallStatus: 'degraded', loading: false, initialLoading: true }, diff --git a/apps/ui/src/routes/applications/[id]/__layout.svelte b/apps/ui/src/routes/applications/[id]/__layout.svelte index 31aebdf5c..fce55a758 100644 --- a/apps/ui/src/routes/applications/[id]/__layout.svelte +++ b/apps/ui/src/routes/applications/[id]/__layout.svelte @@ -59,7 +59,6 @@ import { goto } from '$app/navigation'; import { onDestroy, onMount } from 'svelte'; import { t } from '$lib/translations'; - import DeleteIcon from '$lib/components/DeleteIcon.svelte'; import { appSession, status, @@ -140,13 +139,11 @@ async function stopApplication() { try { $status.application.initialLoading = true; - // $status.application.loading = true; await post(`/applications/${id}/stop`, {}); } catch (error) { return errorNotification(error); } finally { $status.application.initialLoading = false; - // $status.application.loading = false; await getStatus(); } } @@ -154,18 +151,45 @@ if ($status.application.loading) return; $status.application.loading = true; const data = await get(`/applications/${id}/status`); - $status.application.isRunning = data.isRunning; - $status.application.isExited = data.isExited; - $status.application.isRestarting = data.isRestarting; + + $status.application.statuses = data; + const numberOfApplications = + application.buildPack === 'compose' + ? Object.entries(JSON.parse(application.dockerComposeConfiguration)).length + : 1; + if ($status.application.statuses.length === 0) { + $status.application.overallStatus = 'stopped'; + } else { + if ($status.application.statuses.length !== numberOfApplications) { + $status.application.overallStatus = 'degraded'; + } else { + for (const oneStatus of $status.application.statuses) { + if (oneStatus.status.isExited || oneStatus.status.isRestarting) { + $status.application.overallStatus = 'degraded'; + break; + } + if (oneStatus.status.isRunning) { + $status.application.overallStatus = 'healthy'; + } + if ( + !oneStatus.status.isExited && + !oneStatus.status.isRestarting && + !oneStatus.status.isRunning + ) { + $status.application.overallStatus = 'stopped'; + } + } + } + } $status.application.loading = false; $status.application.initialLoading = false; } onDestroy(() => { $status.application.initialLoading = true; - $status.application.isRunning = false; - $status.application.isExited = false; - $status.application.isRestarting = false; + // $status.application.isRunning = false; + // $status.application.isExited = false; + // $status.application.isRestarting = false; $status.application.loading = false; $location = null; $isDeploymentEnabled = false; @@ -173,15 +197,11 @@ }); onMount(async () => { setLocation(application, settings); - $status.application.isRunning = false; - $status.application.isExited = false; - $status.application.isRestarting = false; + // $status.application.isRunning = false; + // $status.application.isExited = false; + // $status.application.isRestarting = false; $status.application.loading = false; - if ( - application.gitSourceId && - application.destinationDockerId && - (application.fqdn || application.settings.isBot) - ) { + if ($isDeploymentEnabled) { await getStatus(); statusInterval = setInterval(async () => { await getStatus(); @@ -208,10 +228,15 @@
Configurations
- {$status.application.isRunning ? 'Running' : 'Stopped'} + {$status.application.overallStatus === 'healthy' + ? 'Running' + : $status.application.overallStatus === 'degraded' + ? 'Degraded' + : 'Stopped'}
{/if} @@ -245,7 +270,7 @@
- {#if $status.application.isExited || $status.application.isRestarting} + {#if $status.application.overallStatus === 'degraded' && application.buildPack !== 'compose'} - {:else if $status.application.isRunning} + {:else if $status.application.overallStatus === 'healthy'} {/if} - {#if $location && $status.application.isRunning} + {#if $location && $status.application.overallStatus === 'healthy'} export let application: any; export let settings: any; + + import yaml from 'js-yaml'; import { page } from '$app/stores'; import { onMount } from 'svelte'; import Select from 'svelte-select'; @@ -47,13 +49,16 @@ import Setting from '$lib/components/Setting.svelte'; import Explainer from '$lib/components/Explainer.svelte'; import { goto } from '$app/navigation'; - import yaml from 'js-yaml'; const { id } = $page.params; $: isDisabled = - !$appSession.isAdmin || $status.application.isRunning || $status.application.initialLoading; + !$appSession.isAdmin || + $status.application.overallStatus === 'degraded' || + $status.application.overallStatus === 'healthy' || + $status.application.initialLoading; + let statues: any = {}; let loading = false; let fqdnEl: any = null; let forceSave = false; @@ -176,7 +181,7 @@ isCustomSSL = !isCustomSSL; } if (name === 'isBot') { - if ($status.application.isRunning) return; + if ($status.application.overallStatus !== 'stopped') return; isBot = !isBot; application.settings.isBot = isBot; application.fqdn = null; @@ -228,9 +233,9 @@ $isDeploymentEnabled = checkIfDeploymentEnabledApplications($appSession.isAdmin, application); } } - async function handleSubmit() { + async function handleSubmit(toast: boolean = true) { if (loading) return; - loading = true; + if (toast) loading = true; try { nonWWWDomain = application.fqdn && getDomain(application.fqdn).replace(/^www\./, ''); if (application.deploymentType) @@ -252,7 +257,7 @@ forceSave = false; - addToast({ + toast && addToast({ message: 'Configuration saved.', type: 'success' }); @@ -333,7 +338,7 @@ let dockerComposeFileContentJSON = JSON.parse(dockerComposeFileContent); dockerComposeServices = normalizeDockerServices(dockerComposeFileContentJSON?.services); application.dockerComposeFile = dockerComposeFileContent; - await handleSubmit(); + await handleSubmit(false); } addToast({ message: 'Compose file reloaded.', @@ -343,6 +348,30 @@ errorNotification(error); } } + $: if ($status.application.statuses) { + for (const service of dockerComposeServices) { + getStatus(service); + } + } + function getStatus(service: any) { + let foundStatus = null; + const foundService = $status.application.statuses.find( + (s: any) => s.name === `${application.id}-${service.name}` + ); + if (foundService) { + const statusText = foundService?.status; + if (statusText?.isRunning) { + foundStatus = 'Running'; + } + if (statusText?.isExited) { + foundStatus = 'Exited'; + } + if (statusText?.isRestarting) { + foundStatus = 'Restarting'; + } + } + statues[service.name] = foundStatus || 'Stopped'; + }
@@ -443,7 +472,7 @@ on:click={() => changeSettings('isBot')} title="Is your application a bot?" description="You can deploy applications without domains or make them to listen on the Exposed Port.

Useful to host Twitch bots, regular jobs, or anything that does not require an incoming HTTP connection." - disabled={$status.application.isRunning} + disabled={isDisabled} />
{/if} @@ -510,12 +539,12 @@ !$status.application.isRunning && changeSettings('dualCerts')} + on:click={() => !isDisabled && changeSettings('dualCerts')} />
{#if isHttps && application.buildPack !== 'compose'} @@ -552,7 +581,7 @@ {isDisabled} containerClasses={isDisabled && containerClass()} id="baseBuildImages" - showIndicator={!$status.application.isRunning} + showIndicator={!isDisabled} items={application.baseBuildImages} on:select={selectBaseBuildImage} value={application.baseBuildImage} @@ -572,7 +601,7 @@ {isDisabled} containerClasses={isDisabled && containerClass()} id="baseImages" - showIndicator={!$status.application.isRunning} + showIndicator={!isDisabled} items={application.baseImages} on:select={selectBaseImage} value={application.baseImage} @@ -594,7 +623,7 @@ {isDisabled} containerClasses={isDisabled && containerClass()} id="deploymentTypes" - showIndicator={!$status.application.isRunning} + showIndicator={!isDisabled} items={['static', 'node']} on:select={selectDeploymentType} value={application.deploymentType} @@ -705,7 +734,9 @@
Reload Docker Compose File {/if}
{#each dockerComposeServices as service} -
-
{service.name}
- {#if service.data?.image} -
{service.data.image}
- {:else} -
No image, build required
- {/if} +
+
+ {service.name} + {statues[service.name] || 'Loading...'} +
+
{application.id}-{service.name}
+
-
+
+
+ + +
{/each}
{/if} From b0b2657fe0decf4403ceb61a05c65b39b267f374 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 6 Oct 2022 10:26:40 +0200 Subject: [PATCH 06/34] debug: remove worker jobs --- apps/api/src/index.ts | 64 +++++++++++++++++------------------ apps/api/src/lib/scheduler.ts | 8 ++--- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index a40ea38f2..65e75cd67 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -7,7 +7,7 @@ import multipart from '@fastify/multipart'; import path, { join } from 'path'; import autoLoad from '@fastify/autoload'; import { asyncExecShell, createRemoteEngineConfiguration, getDomain, isDev, listSettings, prisma, version } from './lib/common'; -import { scheduler } from './lib/scheduler'; +// import { scheduler } from './lib/scheduler'; import { compareVersions } from 'compare-versions'; import Graceful from '@ladjs/graceful' import { verifyRemoteDockerEngineFn } from './routes/api/v1/destinations/handlers'; @@ -124,43 +124,43 @@ const host = '0.0.0.0'; console.log(`Coolify's API is listening on ${host}:${port}`); await initServer(); - const graceful = new Graceful({ brees: [scheduler] }); - graceful.listen(); + // const graceful = new Graceful({ brees: [scheduler] }); + // graceful.listen(); - setInterval(async () => { - if (!scheduler.workers.has('deployApplication')) { - scheduler.run('deployApplication'); - } - if (!scheduler.workers.has('infrastructure')) { - scheduler.run('infrastructure'); - } - }, 2000) + // setInterval(async () => { + // if (!scheduler.workers.has('deployApplication')) { + // scheduler.run('deployApplication'); + // } + // if (!scheduler.workers.has('infrastructure')) { + // scheduler.run('infrastructure'); + // } + // }, 2000) - // autoUpdater - setInterval(async () => { - scheduler.workers.has('infrastructure') && scheduler.workers.get('infrastructure').postMessage("action:autoUpdater") - }, 60000 * 15) + // // autoUpdater + // setInterval(async () => { + // scheduler.workers.has('infrastructure') && scheduler.workers.get('infrastructure').postMessage("action:autoUpdater") + // }, 60000 * 15) - // cleanupStorage - setInterval(async () => { - scheduler.workers.has('infrastructure') && scheduler.workers.get('infrastructure').postMessage("action:cleanupStorage") - }, 60000 * 10) + // // cleanupStorage + // setInterval(async () => { + // scheduler.workers.has('infrastructure') && scheduler.workers.get('infrastructure').postMessage("action:cleanupStorage") + // }, 60000 * 10) - // checkProxies and checkFluentBit - setInterval(async () => { - scheduler.workers.has('infrastructure') && scheduler.workers.get('infrastructure').postMessage("action:checkProxies") - scheduler.workers.has('infrastructure') && scheduler.workers.get('infrastructure').postMessage("action:checkFluentBit") - }, 10000) + // // checkProxies and checkFluentBit + // setInterval(async () => { + // scheduler.workers.has('infrastructure') && scheduler.workers.get('infrastructure').postMessage("action:checkProxies") + // scheduler.workers.has('infrastructure') && scheduler.workers.get('infrastructure').postMessage("action:checkFluentBit") + // }, 10000) - setInterval(async () => { - scheduler.workers.has('infrastructure') && scheduler.workers.get('infrastructure').postMessage("action:copySSLCertificates") - }, 2000) + // setInterval(async () => { + // scheduler.workers.has('infrastructure') && scheduler.workers.get('infrastructure').postMessage("action:copySSLCertificates") + // }, 2000) - await Promise.all([ - getArch(), - getIPAddress(), - configureRemoteDockers(), - ]) + // await Promise.all([ + // getArch(), + // getIPAddress(), + // configureRemoteDockers(), + // ]) } catch (error) { console.error(error); process.exit(1); diff --git a/apps/api/src/lib/scheduler.ts b/apps/api/src/lib/scheduler.ts index 743463757..cec129bef 100644 --- a/apps/api/src/lib/scheduler.ts +++ b/apps/api/src/lib/scheduler.ts @@ -18,10 +18,10 @@ const options: any = { } } }, - jobs: [ - { name: 'infrastructure' }, - { name: 'deployApplication' }, - ], + // jobs: [ + // { name: 'infrastructure' }, + // { name: 'deployApplication' }, + // ], }; if (isDev) options.root = path.join(__dirname, '../jobs'); From ec97e04fd415acc828778ce20c283354a37865e5 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 6 Oct 2022 11:37:28 +0200 Subject: [PATCH 07/34] revert last debug --- apps/api/src/index.ts | 65 +++++++++++++++++------------------ apps/api/src/lib/scheduler.ts | 8 ++--- 2 files changed, 36 insertions(+), 37 deletions(-) diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 65e75cd67..5ed8110fe 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -7,7 +7,7 @@ import multipart from '@fastify/multipart'; import path, { join } from 'path'; import autoLoad from '@fastify/autoload'; import { asyncExecShell, createRemoteEngineConfiguration, getDomain, isDev, listSettings, prisma, version } from './lib/common'; -// import { scheduler } from './lib/scheduler'; +import { scheduler } from './lib/scheduler'; import { compareVersions } from 'compare-versions'; import Graceful from '@ladjs/graceful' import { verifyRemoteDockerEngineFn } from './routes/api/v1/destinations/handlers'; @@ -72,7 +72,6 @@ const host = '0.0.0.0'; } }; - const options = { schema, dotenv: true @@ -124,43 +123,43 @@ const host = '0.0.0.0'; console.log(`Coolify's API is listening on ${host}:${port}`); await initServer(); - // const graceful = new Graceful({ brees: [scheduler] }); - // graceful.listen(); + const graceful = new Graceful({ brees: [scheduler] }); + graceful.listen(); - // setInterval(async () => { - // if (!scheduler.workers.has('deployApplication')) { - // scheduler.run('deployApplication'); - // } - // if (!scheduler.workers.has('infrastructure')) { - // scheduler.run('infrastructure'); - // } - // }, 2000) + setInterval(async () => { + if (!scheduler.workers.has('deployApplication')) { + scheduler.run('deployApplication'); + } + if (!scheduler.workers.has('infrastructure')) { + scheduler.run('infrastructure'); + } + }, 2000) - // // autoUpdater - // setInterval(async () => { - // scheduler.workers.has('infrastructure') && scheduler.workers.get('infrastructure').postMessage("action:autoUpdater") - // }, 60000 * 15) + // autoUpdater + setInterval(async () => { + scheduler.workers.has('infrastructure') && scheduler.workers.get('infrastructure').postMessage("action:autoUpdater") + }, 60000 * 15) - // // cleanupStorage - // setInterval(async () => { - // scheduler.workers.has('infrastructure') && scheduler.workers.get('infrastructure').postMessage("action:cleanupStorage") - // }, 60000 * 10) + // cleanupStorage + setInterval(async () => { + scheduler.workers.has('infrastructure') && scheduler.workers.get('infrastructure').postMessage("action:cleanupStorage") + }, 60000 * 10) - // // checkProxies and checkFluentBit - // setInterval(async () => { - // scheduler.workers.has('infrastructure') && scheduler.workers.get('infrastructure').postMessage("action:checkProxies") - // scheduler.workers.has('infrastructure') && scheduler.workers.get('infrastructure').postMessage("action:checkFluentBit") - // }, 10000) + // checkProxies and checkFluentBit + setInterval(async () => { + scheduler.workers.has('infrastructure') && scheduler.workers.get('infrastructure').postMessage("action:checkProxies") + scheduler.workers.has('infrastructure') && scheduler.workers.get('infrastructure').postMessage("action:checkFluentBit") + }, 10000) - // setInterval(async () => { - // scheduler.workers.has('infrastructure') && scheduler.workers.get('infrastructure').postMessage("action:copySSLCertificates") - // }, 2000) + setInterval(async () => { + scheduler.workers.has('infrastructure') && scheduler.workers.get('infrastructure').postMessage("action:copySSLCertificates") + }, 2000) - // await Promise.all([ - // getArch(), - // getIPAddress(), - // configureRemoteDockers(), - // ]) + await Promise.all([ + getArch(), + getIPAddress(), + configureRemoteDockers(), + ]) } catch (error) { console.error(error); process.exit(1); diff --git a/apps/api/src/lib/scheduler.ts b/apps/api/src/lib/scheduler.ts index cec129bef..743463757 100644 --- a/apps/api/src/lib/scheduler.ts +++ b/apps/api/src/lib/scheduler.ts @@ -18,10 +18,10 @@ const options: any = { } } }, - // jobs: [ - // { name: 'infrastructure' }, - // { name: 'deployApplication' }, - // ], + jobs: [ + { name: 'infrastructure' }, + { name: 'deployApplication' }, + ], }; if (isDev) options.root = path.join(__dirname, '../jobs'); From 680b20d199f9939a6eb1cc00a6283284c7217363 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 6 Oct 2022 11:37:42 +0200 Subject: [PATCH 08/34] update dev container flow --- apps/api/src/lib/buildPacks/common.ts | 4 ++-- docker-compose-dev.yaml | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/apps/api/src/lib/buildPacks/common.ts b/apps/api/src/lib/buildPacks/common.ts index 5fe971cc6..8849938c0 100644 --- a/apps/api/src/lib/buildPacks/common.ts +++ b/apps/api/src/lib/buildPacks/common.ts @@ -468,9 +468,9 @@ export const saveBuildLog = async ({ line = line.replace(regex, '@'); } const addTimestamp = `[${generateTimestamp()}] ${line}`; - const fluentBitUrl = isDev ? 'http://localhost:24224' : 'http://coolify-fluentbit:24224'; + const fluentBitUrl = isDev ? process.env.COOLIFY_CONTAINER_DEV === 'true' ? 'http://coolify-fluentbit:24224' : 'http://localhost:24224' : 'http://coolify-fluentbit:24224'; - if (isDev) { + if (isDev && !process.env.COOLIFY_CONTAINER_DEV) { console.debug(`[${applicationId}] ${addTimestamp}`); } try { diff --git a/docker-compose-dev.yaml b/docker-compose-dev.yaml index c9588c7a8..c3c6d8c54 100644 --- a/docker-compose-dev.yaml +++ b/docker-compose-dev.yaml @@ -7,6 +7,7 @@ services: dockerfile: Dockerfile-dev command: bash -c 'pnpm install && pnpm db:push && pnpm db:seed && pnpm dev' environment: + - COOLIFY_CONTAINER_DEV=true - COOLIFY_APP_ID=random-local-id - COOLIFY_SECRET_KEY=12341234123412341234123412341234 - COOLIFY_DATABASE_URL=file:../db/dev.db @@ -29,6 +30,7 @@ services: volumes: - ./:/app - '/var/run/docker.sock:/var/run/docker.sock' + - /tmp:/tmp networks: - coolify-infra fluent-bit: @@ -38,7 +40,10 @@ services: volumes: - ./logs:/logs ports: - - "24224:24224" + - target: 24224 + published: 24224 + protocol: tcp + mode: host networks: - coolify-infra networks: From e91ea4ecbe70b365a5f0e5488a99cfbf5cd2c7ba Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 6 Oct 2022 11:37:47 +0200 Subject: [PATCH 09/34] feat: docker compose --- apps/api/src/lib/buildPacks/compose.ts | 25 ++++++----- .../src/routes/applications/[id]/index.svelte | 43 +++++++++++++++---- 2 files changed, 47 insertions(+), 21 deletions(-) diff --git a/apps/api/src/lib/buildPacks/compose.ts b/apps/api/src/lib/buildPacks/compose.ts index 71cb72b21..11be25d13 100644 --- a/apps/api/src/lib/buildPacks/compose.ts +++ b/apps/api/src/lib/buildPacks/compose.ts @@ -2,7 +2,6 @@ import { promises as fs } from 'fs'; import { defaultComposeConfiguration, executeDockerCmd } from '../common'; import { buildImage, saveBuildLog } from './common'; import yaml from 'js-yaml'; -import { getSecrets } from '../../routes/api/v1/applications/handlers'; export default async function (data) { let { @@ -41,20 +40,18 @@ export default async function (data) { const envs = [ `PORT=${port}` ]; - if (getSecrets.length > 0) { + if (secrets.length > 0) { secrets.forEach((secret) => { - if (secret.isBuildSecret) { - if (pullmergeRequestId) { - const isSecretFound = secrets.filter(s => s.name === secret.name && s.isPRMRSecret) - if (isSecretFound.length > 0) { - envs.push(`${secret.name}=${isSecretFound[0].value}`); - } else { - envs.push(`${secret.name}=${secret.value}`); - } + if (pullmergeRequestId) { + const isSecretFound = secrets.filter(s => s.name === secret.name && s.isPRMRSecret) + if (isSecretFound.length > 0) { + envs.push(`${secret.name}=${isSecretFound[0].value}`); } else { - if (!secret.isPRMRSecret) { - envs.push(`${secret.name}=${secret.value}`); - } + envs.push(`${secret.name}=${secret.value}`); + } + } else { + if (!secret.isPRMRSecret) { + envs.push(`${secret.name}=${secret.value}`); } } }); @@ -66,6 +63,8 @@ export default async function (data) { } catch (error) { // } + console.log('envFound', envFound) + console.log(envs) const composeVolumes = volumes.map((volume) => { return { [`${volume.split(':')[0]}`]: { diff --git a/apps/ui/src/routes/applications/[id]/index.svelte b/apps/ui/src/routes/applications/[id]/index.svelte index 11f59dc4d..fe6c07e1f 100644 --- a/apps/ui/src/routes/applications/[id]/index.svelte +++ b/apps/ui/src/routes/applications/[id]/index.svelte @@ -49,6 +49,7 @@ import Setting from '$lib/components/Setting.svelte'; import Explainer from '$lib/components/Explainer.svelte'; import { goto } from '$app/navigation'; + import Beta from '$lib/components/Beta.svelte'; const { id } = $page.params; @@ -257,10 +258,11 @@ forceSave = false; - toast && addToast({ - message: 'Configuration saved.', - type: 'success' - }); + toast && + addToast({ + message: 'Configuration saved.', + type: 'success' + }); if (application.fqdn && application.fqdn.startsWith('https')) { isHttps = true; } else { @@ -436,7 +438,7 @@ {/if}
- + {#if isDisabled} {:else} @@ -910,7 +912,7 @@
{:else}
- Docker Compose + Stack {#if $appSession.isAdmin} {:else if $status.application.overallStatus === 'healthy'} - Stop - - - Restart (useful to change secrets) - - - Force Redeploy (without cache) + + + {#if application.buildPack !== 'compose'} + + {/if} {:else if $isDeploymentEnabled && !$page.url.pathname.startsWith(`/applications/${id}/configuration/`)} {:else if $status.application.overallStatus === 'healthy'} + + {#if application.buildPack !== 'compose'} + +{/if} + - {#if application.buildPack !== 'compose'} - - {/if} + {:else if $isDeploymentEnabled && !$page.url.pathname.startsWith(`/applications/${id}/configuration/`)}
diff --git a/apps/ui/src/routes/applications/[id]/configuration/_BuildPack.svelte b/apps/ui/src/routes/applications/[id]/configuration/_BuildPack.svelte index f7771e026..fb913b5a9 100644 --- a/apps/ui/src/routes/applications/[id]/configuration/_BuildPack.svelte +++ b/apps/ui/src/routes/applications/[id]/configuration/_BuildPack.svelte @@ -14,8 +14,9 @@ export let foundConfig: any; export let scanning: any; export let packageManager: any; - export let dockerComposeFile: any = null; + export let dockerComposeFile: string | null = null; export let dockerComposeFileLocation: string | null = null; + export let dockerComposeConfiguration: any = null; async function handleSubmit(name: string) { try { @@ -27,11 +28,19 @@ delete tempBuildPack.fancyName; delete tempBuildPack.color; delete tempBuildPack.hoverColor; + let composeConfiguration: any = {} + if (!dockerComposeConfiguration && dockerComposeFile) { + for (const [name, _] of Object.entries(JSON.parse(dockerComposeFile).services)) { + composeConfiguration[name] = {}; + } + + } await post(`/applications/${id}`, { ...tempBuildPack, buildPack: name, dockerComposeFile, - dockerComposeFileLocation + dockerComposeFileLocation, + dockerComposeConfiguration: JSON.stringify(composeConfiguration) || JSON.stringify({}) }); await post(`/applications/${id}/configuration/buildpack`, { buildPack: name }); return await goto(from || `/applications/${id}`); diff --git a/apps/ui/src/routes/applications/[id]/configuration/_PublicRepository.svelte b/apps/ui/src/routes/applications/[id]/configuration/_PublicRepository.svelte index 28d0ef1ce..32d5627d9 100644 --- a/apps/ui/src/routes/applications/[id]/configuration/_PublicRepository.svelte +++ b/apps/ui/src/routes/applications/[id]/configuration/_PublicRepository.svelte @@ -165,7 +165,7 @@ placeholder="eg: https://github.com/coollabsio/nodejs-example/tree/main" bind:value={publicRepositoryLink} /> -
diff --git a/apps/ui/src/routes/applications/[id]/configuration/buildpack.svelte b/apps/ui/src/routes/applications/[id]/configuration/buildpack.svelte index ad9b70be4..30ec09fe9 100644 --- a/apps/ui/src/routes/applications/[id]/configuration/buildpack.svelte +++ b/apps/ui/src/routes/applications/[id]/configuration/buildpack.svelte @@ -12,6 +12,7 @@ const response = await get(`/applications/${params.id}/configuration/buildpack`); return { props: { + application, ...response } }; @@ -25,6 +26,14 @@
-
+ handleSubmit()}>
General
@@ -440,7 +441,7 @@
{#if isDisabled} - + {:else} Stack {#if $appSession.isAdmin} - Reload Docker Compose File {/if}
diff --git a/apps/ui/src/routes/applications/[id]/logs/index.svelte b/apps/ui/src/routes/applications/[id]/logs/index.svelte index 4ec6ab28d..a0ef249b2 100644 --- a/apps/ui/src/routes/applications/[id]/logs/index.svelte +++ b/apps/ui/src/routes/applications/[id]/logs/index.svelte @@ -3,11 +3,11 @@ import { get } from '$lib/api'; import { t } from '$lib/translations'; import { errorNotification } from '$lib/common'; - import LoadingLogs from '$lib/components/LoadingLogs.svelte'; import { onMount, onDestroy } from 'svelte'; import Tooltip from '$lib/components/Tooltip.svelte'; import { status } from '$lib/store'; import { goto } from '$app/navigation'; + let application: any = {}; let logsLoading = false; let loadLogsInterval: any = null; @@ -17,26 +17,39 @@ let followingLogs: any; let logsEl: any; let position = 0; - if ( - !$status.application.isExited && - !$status.application.isRestarting && - !$status.application.isRunning - ) { - goto(`/applications/${$page.params.id}/`, { replaceState: true }); - } + let services: any = []; + let selectedService: any = null; const { id } = $page.params; onMount(async () => { const response = await get(`/applications/${id}`); application = response.application; - loadAllLogs(); - loadLogsInterval = setInterval(() => { - loadLogs(); - }, 1000); + if (response.application.dockerComposeFile) { + services = normalizeDockerServices( + JSON.parse(response.application.dockerComposeFile).services + ); + } else { + services = [ + { + name: '' + } + ]; + await selectService('') + } }); onDestroy(() => { clearInterval(loadLogsInterval); clearInterval(followingInterval); }); + function normalizeDockerServices(services: any[]) { + const tempdockerComposeServices = []; + for (const [name, data] of Object.entries(services)) { + tempdockerComposeServices.push({ + name, + data + }); + } + return tempdockerComposeServices; + } async function loadAllLogs() { try { logsLoading = true; @@ -55,7 +68,7 @@ if (logsLoading) return; try { const newLogs: any = await get( - `/applications/${id}/logs?since=${lastLog?.split(' ')[0] || 0}` + `/applications/${id}/logs/${selectedService}?since=${lastLog?.split(' ')[0] || 0}` ); if (newLogs?.logs && newLogs.logs[newLogs.logs.length - 1] !== logs[logs.length - 1]) { @@ -89,6 +102,22 @@ clearInterval(followingInterval); } } + async function selectService(service: any, init: boolean = false) { + if (services.length === 1 && init) return + + if (loadLogsInterval) clearInterval(loadLogsInterval); + if (followingInterval) clearInterval(followingInterval); + + logs = []; + lastLog = null; + followingLogs = false; + + selectedService = `${application.id}${service.name ? `-${service.name}` : ''}`; + loadLogs(); + loadLogsInterval = setInterval(() => { + loadLogs(); + }, 1000); + }
@@ -96,50 +125,67 @@
Application Logs
-
- {#if logs.length === 0} -
{$t('application.build.waiting_logs')}
- {:else} -
-
- - {#if loadLogsInterval} -
-
- {#each logs as log} -

{log + '\n'}

- {/each} -
-
- {/if} +
+ {#each services as service} + + {/each}
+ +{#if selectedService} +
+ {#if logs.length === 0} +
{$t('application.build.waiting_logs')}
+ {:else} +
+
+ + {#if loadLogsInterval} +
+
+ {#each logs as log} +

{log + '\n'}

+ {/each} +
+
+ {/if} +
+{/if} From 9e634fed1357bcc16ee2f0a437c2b59a024dc2ad Mon Sep 17 00:00:00 2001 From: Carl Mercier Date: Thu, 6 Oct 2022 15:38:15 -0400 Subject: [PATCH 15/34] Inject GIT SHA into build process As discussed at https://github.com/docker/hub-feedback/issues/600 --- apps/api/src/lib/buildPacks/common.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/api/src/lib/buildPacks/common.ts b/apps/api/src/lib/buildPacks/common.ts index 98a387f2f..47a4526e3 100644 --- a/apps/api/src/lib/buildPacks/common.ts +++ b/apps/api/src/lib/buildPacks/common.ts @@ -596,7 +596,8 @@ export async function buildImage({ } const dockerFile = isCache ? `${dockerFileLocation}-cache` : `${dockerFileLocation}` const cache = `${applicationId}:${tag}${isCache ? '-cache' : ''}` - await executeDockerCmd({ debug, buildId, applicationId, dockerId, command: `docker build --progress plain -f ${workdir}/${dockerFile} -t ${cache} ${workdir}` }) + const git_sha = `git rev-parse HEAD` + await executeDockerCmd({ debug, buildId, applicationId, dockerId, command: `docker build --progress plain -f ${workdir}/${dockerFile} -t ${cache} --build-arg SOURCE_COMMIT=${git_sha} ${workdir}` }) const { status } = await prisma.build.findUnique({ where: { id: buildId } }) if (status === 'canceled') { throw new Error('Deployment canceled.') @@ -758,4 +759,4 @@ export async function buildCacheImageWithCargo(data, imageForBuild) { Dockerfile.push('RUN cargo chef cook --release --recipe-path recipe.json'); await fs.writeFile(`${workdir}/Dockerfile-cache`, Dockerfile.join('\n')); await buildImage({ ...data, isCache: true }); -} \ No newline at end of file +} From 7090c1657576287eee43c8a19e2ca2021b7b7a79 Mon Sep 17 00:00:00 2001 From: Carl Mercier Date: Thu, 6 Oct 2022 15:42:57 -0400 Subject: [PATCH 16/34] Update common.ts --- apps/api/src/lib/buildPacks/common.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/lib/buildPacks/common.ts b/apps/api/src/lib/buildPacks/common.ts index 47a4526e3..3c04e0cb7 100644 --- a/apps/api/src/lib/buildPacks/common.ts +++ b/apps/api/src/lib/buildPacks/common.ts @@ -596,7 +596,7 @@ export async function buildImage({ } const dockerFile = isCache ? `${dockerFileLocation}-cache` : `${dockerFileLocation}` const cache = `${applicationId}:${tag}${isCache ? '-cache' : ''}` - const git_sha = `git rev-parse HEAD` + const { git_sha } = await asyncExecShell(`git rev-parse HEAD`) await executeDockerCmd({ debug, buildId, applicationId, dockerId, command: `docker build --progress plain -f ${workdir}/${dockerFile} -t ${cache} --build-arg SOURCE_COMMIT=${git_sha} ${workdir}` }) const { status } = await prisma.build.findUnique({ where: { id: buildId } }) if (status === 'canceled') { From 7683164ed274d938f3f34060e6b4e2bf54500468 Mon Sep 17 00:00:00 2001 From: Martin Saulis Date: Fri, 7 Oct 2022 14:59:16 +0300 Subject: [PATCH 17/34] fix deno options string --- apps/api/src/lib/buildPacks/deno.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/lib/buildPacks/deno.ts b/apps/api/src/lib/buildPacks/deno.ts index f255a5983..074464445 100644 --- a/apps/api/src/lib/buildPacks/deno.ts +++ b/apps/api/src/lib/buildPacks/deno.ts @@ -49,7 +49,7 @@ const createDockerfile = async (data, image): Promise => { Dockerfile.push(`RUN deno cache ${denoMainFile}`); Dockerfile.push(`ENV NO_COLOR true`); Dockerfile.push(`EXPOSE ${port}`); - Dockerfile.push(`CMD deno run ${denoOptions ? denoOptions.split(' ') : ''} ${denoMainFile}`); + Dockerfile.push(`CMD deno run ${denoOptions || ''} ${denoMainFile}`); await fs.writeFile(`${workdir}/Dockerfile`, Dockerfile.join('\n')); }; From f07868d24ee92895943d24ae7c187781986a9802 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Mon, 10 Oct 2022 15:28:02 +0200 Subject: [PATCH 18/34] fix: do not show nope as ip address for dbs --- .../[id]/_Databases/_Databases.svelte | 100 +++++++++++------- 1 file changed, 64 insertions(+), 36 deletions(-) diff --git a/apps/ui/src/routes/databases/[id]/_Databases/_Databases.svelte b/apps/ui/src/routes/databases/[id]/_Databases/_Databases.svelte index b3601a9e9..02c98e150 100644 --- a/apps/ui/src/routes/databases/[id]/_Databases/_Databases.svelte +++ b/apps/ui/src/routes/databases/[id]/_Databases/_Databases.svelte @@ -21,9 +21,11 @@ const { id } = $page.params; - let loading = false; - let publicLoading = false; - + let loading = { + main: false, + public: false + }; + let publicUrl = ''; let appendOnly = database.settings.appendOnly; let databaseDefault: any; @@ -47,23 +49,46 @@ databaseDbUser = ''; } } - function generateUrl(): string { - return `${database.type}://${ - databaseDbUser ? databaseDbUser + ':' : '' - }${databaseDbUserPassword}@${ - $status.database.isPublic - ? database.destinationDocker.remoteEngine - ? database.destinationDocker.remoteIpAddress - : $appSession.ipv4 - : database.id - }:${$status.database.isPublic ? database.publicPort : privatePort}/${databaseDefault}`; + function generateUrl() { + const ipAddress = () => { + if ($status.database.isPublic) { + if (database.destinationDocker.remoteEngine) { + return database.destinationDocker.remoteIpAddress; + } + if ($appSession.ipv6) { + return $appSession.ipv6; + } + if ($appSession.ipv4) { + return $appSession.ipv4; + } + return ''; + } else { + return database.id; + } + }; + const user = () => { + if (databaseDbUser) { + return databaseDbUser + ':'; + } + return ''; + }; + const port = () => { + if ($status.database.isPublic) { + return database.publicPort; + } else { + return privatePort; + } + }; + publicUrl = `${ + database.type + }://${user()}${databaseDbUserPassword}@${ipAddress()}:${port()}/${databaseDefault}`; } async function changeSettings(name: any) { if (name !== 'appendOnly') { - if (publicLoading || !$status.database.isRunning) return; + if (loading.public || !$status.database.isRunning) return; } - publicLoading = true; + loading.public = true; let data = { isPublic: $status.database.isPublic, appendOnly @@ -87,12 +112,12 @@ } catch (error) { return errorNotification(error); } finally { - publicLoading = false; + loading.public = false; } } async function handleSubmit() { try { - loading = true; + loading.main = true; await post(`/databases/${id}`, { ...database, isRunning: $status.database.isRunning }); generateDbDetails(); addToast({ @@ -102,7 +127,7 @@ } catch (error) { return errorNotification(error); } finally { - loading = false; + loading.main = false; } } @@ -115,9 +140,9 @@ {$t('forms.save')} {/if}
@@ -175,7 +200,7 @@ readonly disabled name="publicPort" - value={publicLoading + value={loading.public ? 'Loading...' : $status.database.isPublic ? database.publicPort @@ -198,8 +223,8 @@ {/if}
-
-
- + {#if publicUrl} + + {/if}
@@ -228,7 +256,7 @@
changeSettings('isPublic')} title={$t('database.set_public')} @@ -238,7 +266,7 @@ {#if database.type === 'redis'} changeSettings('appendOnly')} title={$t('database.change_append_only_mode')} From a2b4d400afdf2d7d77d9d76313739456537c7916 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Mon, 10 Oct 2022 15:28:16 +0200 Subject: [PATCH 19/34] fix: add git sha to build args --- apps/api/src/lib/buildPacks/common.ts | 8 +++++--- apps/api/src/lib/importers/github.ts | 2 -- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/api/src/lib/buildPacks/common.ts b/apps/api/src/lib/buildPacks/common.ts index f2d76c249..fd36a2ea6 100644 --- a/apps/api/src/lib/buildPacks/common.ts +++ b/apps/api/src/lib/buildPacks/common.ts @@ -580,7 +580,8 @@ export async function buildImage({ dockerId, isCache = false, debug = false, - dockerFileLocation = '/Dockerfile' + dockerFileLocation = '/Dockerfile', + commit }) { if (isCache) { await saveBuildLog({ line: `Building cache image started.`, buildId, applicationId }); @@ -596,8 +597,9 @@ export async function buildImage({ } const dockerFile = isCache ? `${dockerFileLocation}-cache` : `${dockerFileLocation}` const cache = `${applicationId}:${tag}${isCache ? '-cache' : ''}` - const { git_sha } = await asyncExecShell(`git rev-parse HEAD`) - await executeDockerCmd({ debug, buildId, applicationId, dockerId, command: `docker build --progress plain -f ${workdir}/${dockerFile} -t ${cache} --build-arg SOURCE_COMMIT=${git_sha} ${workdir}` }) + + await executeDockerCmd({ debug, buildId, applicationId, dockerId, command: `docker build --progress plain -f ${workdir}/${dockerFile} -t ${cache} --build-arg SOURCE_COMMIT=${commit} ${workdir}` }) + const { status } = await prisma.build.findUnique({ where: { id: buildId } }) if (status === 'canceled') { throw new Error('Deployment canceled.') diff --git a/apps/api/src/lib/importers/github.ts b/apps/api/src/lib/importers/github.ts index 798931d7a..f42461939 100644 --- a/apps/api/src/lib/importers/github.ts +++ b/apps/api/src/lib/importers/github.ts @@ -73,6 +73,4 @@ export default async function ({ const { stdout: commit } = await asyncExecShell(`cd ${workdir}/ && git rev-parse HEAD`); return commit.replace('\n', ''); - - } From d4b731841315c8dbe9bdfb0b1d1c5634658dd31b Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Mon, 10 Oct 2022 15:28:36 +0200 Subject: [PATCH 20/34] fix: smart search for new services --- .../api/src/lib/services/supportedVersions.ts | 60 ++++++++++++------- .../services/[id]/configuration/type.svelte | 49 ++++++++++++++- 2 files changed, 86 insertions(+), 23 deletions(-) diff --git a/apps/api/src/lib/services/supportedVersions.ts b/apps/api/src/lib/services/supportedVersions.ts index 64f049961..d84d1bd65 100644 --- a/apps/api/src/lib/services/supportedVersions.ts +++ b/apps/api/src/lib/services/supportedVersions.ts @@ -29,7 +29,8 @@ export const supportedServiceTypesAndVersions = [ recommendedVersion: 'stable', ports: { main: 8000 - } + }, + labels: ['analytics', 'plausible', 'plausible-analytics', 'gdpr', 'no-cookie'] }, { name: 'nocodb', @@ -39,7 +40,8 @@ export const supportedServiceTypesAndVersions = [ recommendedVersion: 'latest', ports: { main: 8080 - } + }, + labels: ['nocodb', 'airtable', 'database'] }, { name: 'minio', @@ -49,7 +51,8 @@ export const supportedServiceTypesAndVersions = [ recommendedVersion: 'latest', ports: { main: 9001 - } + }, + labels: ['minio', 's3', 'storage'] }, { name: 'vscodeserver', @@ -59,7 +62,8 @@ export const supportedServiceTypesAndVersions = [ recommendedVersion: 'latest', ports: { main: 8080 - } + }, + labels: ['vscodeserver', 'vscode', 'code-server', 'ide'] }, { name: 'wordpress', @@ -70,7 +74,8 @@ export const supportedServiceTypesAndVersions = [ recommendedVersion: 'latest', ports: { main: 80 - } + }, + labels: ['wordpress', 'blog', 'cms'] }, { name: 'vaultwarden', @@ -80,7 +85,8 @@ export const supportedServiceTypesAndVersions = [ recommendedVersion: 'latest', ports: { main: 80 - } + }, + labels: ['vaultwarden', 'password-manager', 'passwords'] }, { name: 'languagetool', @@ -90,7 +96,8 @@ export const supportedServiceTypesAndVersions = [ recommendedVersion: 'latest', ports: { main: 8010 - } + }, + labels: ['languagetool', 'grammar', 'spell-checker'] }, { name: 'n8n', @@ -100,7 +107,8 @@ export const supportedServiceTypesAndVersions = [ recommendedVersion: 'latest', ports: { main: 5678 - } + }, + labels: ['n8n', 'workflow', 'automation', 'ifttt', 'zapier', 'nodered'] }, { name: 'uptimekuma', @@ -110,7 +118,8 @@ export const supportedServiceTypesAndVersions = [ recommendedVersion: 'latest', ports: { main: 3001 - } + }, + labels: ['uptimekuma', 'uptime', 'monitoring'] }, { name: 'ghost', @@ -121,7 +130,8 @@ export const supportedServiceTypesAndVersions = [ recommendedVersion: 'latest', ports: { main: 2368 - } + }, + labels: ['ghost', 'blog', 'cms'] }, { name: 'meilisearch', @@ -132,7 +142,8 @@ export const supportedServiceTypesAndVersions = [ recommendedVersion: 'latest', ports: { main: 7700 - } + }, + labels: ['meilisearch', 'search', 'search-engine'] }, { name: 'umami', @@ -143,7 +154,8 @@ export const supportedServiceTypesAndVersions = [ recommendedVersion: 'postgresql-latest', ports: { main: 3000 - } + }, + labels: ['umami', 'analytics', 'gdpr', 'no-cookie'] }, { name: 'hasura', @@ -154,7 +166,8 @@ export const supportedServiceTypesAndVersions = [ recommendedVersion: 'v2.10.0', ports: { main: 8080 - } + }, + labels: ['hasura', 'graphql', 'database'] }, { name: 'fider', @@ -165,7 +178,8 @@ export const supportedServiceTypesAndVersions = [ recommendedVersion: 'stable', ports: { main: 3000 - } + }, + labels: ['fider', 'feedback', 'suggestions'] }, { name: 'appwrite', @@ -176,7 +190,8 @@ export const supportedServiceTypesAndVersions = [ recommendedVersion: '1.0', ports: { main: 80 - } + }, + labels: ['appwrite', 'database', 'storage', 'api', 'serverless'] }, // { // name: 'moodle', @@ -198,7 +213,8 @@ export const supportedServiceTypesAndVersions = [ recommendedVersion: 'latest', ports: { main: 8000 - } + }, + labels: ['glitchtip', 'error-reporting', 'error', 'sentry', 'bugsnag'] }, { name: 'searxng', @@ -209,7 +225,8 @@ export const supportedServiceTypesAndVersions = [ recommendedVersion: 'latest', ports: { main: 8080 - } + }, + labels: ['searxng', 'search', 'search-engine'] }, { name: 'weblate', @@ -220,7 +237,8 @@ export const supportedServiceTypesAndVersions = [ recommendedVersion: 'latest', ports: { main: 8080 - } + }, + labels: ['weblate', 'translation', 'localization'] }, // { // name: 'taiga', @@ -242,7 +260,8 @@ export const supportedServiceTypesAndVersions = [ recommendedVersion: 'latest', ports: { main: 3000 - } + }, + labels: ['grafana', 'monitoring', 'metrics', 'dashboard'] }, { name: 'trilium', @@ -253,6 +272,7 @@ export const supportedServiceTypesAndVersions = [ recommendedVersion: 'latest', ports: { main: 8080 - } + }, + labels: ['trilium', 'notes', 'note-taking', 'wiki'] }, ]; diff --git a/apps/ui/src/routes/services/[id]/configuration/type.svelte b/apps/ui/src/routes/services/[id]/configuration/type.svelte index bd4bbd74e..dcc6aee03 100644 --- a/apps/ui/src/routes/services/[id]/configuration/type.svelte +++ b/apps/ui/src/routes/services/[id]/configuration/type.svelte @@ -27,10 +27,12 @@ -
- {#each types as type} +
+
+
+ + + + + +
+ doSearch()} + /> +
+
+
+
+ {#each filteredTypes as type}
handleSubmit(type.name)}>
{/each}
+
From 6882e83d1e572d27de1e29ecf7328043a4f7e1fa Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Mon, 10 Oct 2022 15:28:46 +0200 Subject: [PATCH 21/34] fix: logs for not running containers --- .../routes/api/v1/applications/handlers.ts | 5 ++- .../src/routes/applications/[id]/_Menu.svelte | 4 +-- .../routes/applications/[id]/__layout.svelte | 26 +++++++++++++-- .../applications/[id]/logs/index.svelte | 32 +++++++------------ 4 files changed, 42 insertions(+), 25 deletions(-) diff --git a/apps/api/src/routes/api/v1/applications/handlers.ts b/apps/api/src/routes/api/v1/applications/handlers.ts index 588ad98be..686eea8cc 100644 --- a/apps/api/src/routes/api/v1/applications/handlers.ts +++ b/apps/api/src/routes/api/v1/applications/handlers.ts @@ -1243,7 +1243,10 @@ export async function getApplicationLogs(request: FastifyRequestLogs
  • handleDeploySubmit(false)} > + {#if $status.application.overallStatus !== 'degraded'} - {$status.application.overallStatus === 'degraded' ? 'Restart Degraded Services' : 'Deploy'} + {:else} + + + + + + {/if} + {$status.application.overallStatus === 'degraded' ? 'Restart Degraded Stack' : 'Deploy'} {/if} {#if $location && $status.application.overallStatus === 'healthy'} diff --git a/apps/ui/src/routes/applications/[id]/logs/index.svelte b/apps/ui/src/routes/applications/[id]/logs/index.svelte index a0ef249b2..a716f0788 100644 --- a/apps/ui/src/routes/applications/[id]/logs/index.svelte +++ b/apps/ui/src/routes/applications/[id]/logs/index.svelte @@ -5,8 +5,6 @@ import { errorNotification } from '$lib/common'; import { onMount, onDestroy } from 'svelte'; import Tooltip from '$lib/components/Tooltip.svelte'; - import { status } from '$lib/store'; - import { goto } from '$app/navigation'; let application: any = {}; let logsLoading = false; @@ -19,6 +17,8 @@ let position = 0; let services: any = []; let selectedService: any = null; + let noContainer = false; + const { id } = $page.params; onMount(async () => { const response = await get(`/applications/${id}`); @@ -33,7 +33,7 @@ name: '' } ]; - await selectService('') + await selectService(''); } }); onDestroy(() => { @@ -50,27 +50,17 @@ } return tempdockerComposeServices; } - async function loadAllLogs() { - try { - logsLoading = true; - const data: any = await get(`/applications/${id}/logs`); - if (data?.logs) { - lastLog = data.logs[data.logs.length - 1]; - logs = data.logs; - } - } catch (error) { - return errorNotification(error); - } finally { - logsLoading = false; - } - } async function loadLogs() { if (logsLoading) return; try { const newLogs: any = await get( `/applications/${id}/logs/${selectedService}?since=${lastLog?.split(' ')[0] || 0}` ); - + if (newLogs.noContainer) { + noContainer = true; + } else { + noContainer = false; + } if (newLogs?.logs && newLogs.logs[newLogs.logs.length - 1] !== logs[logs.length - 1]) { logs = logs.concat(newLogs.logs); lastLog = newLogs.logs[newLogs.logs.length - 1]; @@ -103,7 +93,7 @@ } } async function selectService(service: any, init: boolean = false) { - if (services.length === 1 && init) return + if (services.length === 1 && init) return; if (loadLogsInterval) clearInterval(loadLogsInterval); if (followingInterval) clearInterval(followingInterval); @@ -143,7 +133,9 @@ {#if selectedService}
    {#if logs.length === 0} -
    {$t('application.build.waiting_logs')}
    + {#if noContainer} +
    Container not found / exited.
    + {/if} {:else}
    From 42415a81c1d87e35c0a7a0e001973bd0eb973143 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 11 Oct 2022 10:54:13 +0200 Subject: [PATCH 22/34] fix: update docker binaries --- Dockerfile | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index b7bd56212..2f14248dc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,12 @@ ARG PNPM_VERSION=7.11.0 ARG NPM_VERSION=8.19.1 +# https://download.docker.com/linux/static/stable/ +ARG DOCKER_VERSION=20.10.18 +# https://github.com/docker/compose/releases +# Reverted to 2.6.1 because of this https://github.com/docker/compose/issues/9704. 2.9.0 still has a bug. +ARG DOCKER_COMPOSE_VERSION=2.6.1 +# https://github.com/buildpacks/pack/releases +ARG PACK_VERSION=v0.27.0 FROM node:18-slim as build WORKDIR /app @@ -23,14 +30,12 @@ RUN npm --no-update-notifier --no-fund --global install pnpm@${PNPM_VERSION} RUN npm install -g npm@${PNPM_VERSION} RUN mkdir -p ~/.docker/cli-plugins/ -# https://download.docker.com/linux/static/stable/ -RUN curl -SL https://cdn.coollabs.io/bin/$TARGETPLATFORM/docker-20.10.9 -o /usr/bin/docker -# https://github.com/docker/compose/releases -# Reverted to 2.6.1 because of this https://github.com/docker/compose/issues/9704. 2.9.0 still has a bug. -RUN curl -SL https://cdn.coollabs.io/bin/$TARGETPLATFORM/docker-compose-linux-2.6.1 -o ~/.docker/cli-plugins/docker-compose -RUN chmod +x ~/.docker/cli-plugins/docker-compose /usr/bin/docker -RUN (curl -sSL "https://github.com/buildpacks/pack/releases/download/v0.27.0/pack-v0.27.0-linux.tgz" | tar -C /usr/local/bin/ --no-same-owner -xzv pack) +RUN curl -SL https://cdn.coollabs.io/bin/$TARGETPLATFORM/docker-$DOCKER_VERSION -o /usr/bin/docker +RUN curl -SL https://cdn.coollabs.io/bin/$TARGETPLATFORM/docker-compose-linux-$DOCKER_COMPOSE_VERSION -o ~/.docker/cli-plugins/docker-compose +RUN curl -SL https://cdn.coollabs.io/bin/$TARGETPLATFORM/pack-$PACK_VERSION -o /usr/local/bin/pack + +RUN chmod +x ~/.docker/cli-plugins/docker-compose /usr/bin/docker /usr/local/bin/pack COPY --from=build /app/apps/api/build/ . COPY --from=build /app/others/fluentbit/ ./fluentbit From 45b0f791bbf47f5c23a398623d0683b5c863689a Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 11 Oct 2022 10:54:22 +0200 Subject: [PATCH 23/34] ci: update staging release --- .github/workflows/staging-release.yml | 38 +++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/.github/workflows/staging-release.yml b/.github/workflows/staging-release.yml index 7a5e5474f..b8b05da44 100644 --- a/.github/workflows/staging-release.yml +++ b/.github/workflows/staging-release.yml @@ -6,7 +6,7 @@ on: - next jobs: - arm64-making-something-cool: + arm64: runs-on: [self-hosted, arm64] steps: - name: Checkout @@ -34,7 +34,7 @@ jobs: tags: coollabsio/coolify:next-arm64 cache-from: type=registry,ref=coollabsio/coolify:buildcache-next-arm64 cache-to: type=registry,ref=coollabsio/coolify:buildcache-next-arm64,mode=max - amd64-making-something-cool: + amd64: runs-on: ubuntu-latest steps: - name: Checkout @@ -62,9 +62,37 @@ jobs: tags: coollabsio/coolify:next-amd64,coollabsio/coolify:next-test cache-from: type=registry,ref=coollabsio/coolify:buildcache-next-amd64 cache-to: type=registry,ref=coollabsio/coolify:buildcache-next-amd64,mode=max - merge-manifest-to-be-cool: + aarch64: + runs-on: [self-hosted, arm64] + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: "next" + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Get current package version + uses: martinbeentjes/npm-get-version-action@v1.2.3 + id: package-version + - name: Build and push + uses: docker/build-push-action@v3 + with: + context: . + platforms: linux/aarch64 + push: true + tags: coollabsio/coolify:next-aarch64,coollabsio/coolify:next-test + cache-from: type=registry,ref=coollabsio/coolify:buildcache-next-aarch64 + cache-to: type=registry,ref=coollabsio/coolify:buildcache-next-aarch64,mode=max + merge-manifest: runs-on: ubuntu-latest - needs: [arm64-making-something-cool, amd64-making-something-cool] + needs: [arm64, amd64, aarch64] steps: - name: Checkout uses: actions/checkout@v3 @@ -79,7 +107,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Create & publish manifest run: | - docker manifest create coollabsio/coolify:next --amend coollabsio/coolify:next-amd64 --amend coollabsio/coolify:next-arm64 + docker manifest create coollabsio/coolify:next --amend coollabsio/coolify:next-amd64 --amend coollabsio/coolify:next-arm64 --amend coollabsio/coolify:next-aarch64 docker manifest push coollabsio/coolify:next - uses: sarisia/actions-status-discord@v1 if: always() From 4eca05bbbab4237fdc587196d39074c24c4ada8f Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 12 Oct 2022 10:07:18 +0200 Subject: [PATCH 24/34] fix: gh release --- .github/workflows/production-release.yml | 34 +++++++++++++++++++++--- .github/workflows/staging-release.yml | 4 +-- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/.github/workflows/production-release.yml b/.github/workflows/production-release.yml index e3e6a0750..44b5f7003 100644 --- a/.github/workflows/production-release.yml +++ b/.github/workflows/production-release.yml @@ -5,7 +5,7 @@ on: types: [released] jobs: - arm64-build: + arm64: runs-on: [self-hosted, arm64] steps: - name: Checkout @@ -31,7 +31,7 @@ jobs: tags: coollabsio/coolify:${{steps.package-version.outputs.current-version}}-arm64 cache-from: type=registry,ref=coollabsio/coolify:buildcache-arm64 cache-to: type=registry,ref=coollabsio/coolify:buildcache-arm64,mode=max - amd64-build: + amd64: runs-on: ubuntu-latest steps: - name: Checkout @@ -57,9 +57,35 @@ jobs: tags: coollabsio/coolify:${{steps.package-version.outputs.current-version}}-amd64 cache-from: type=registry,ref=coollabsio/coolify:buildcache-amd64 cache-to: type=registry,ref=coollabsio/coolify:buildcache-amd64,mode=max + aarch64: + runs-on: [self-hosted, arm64] + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Get current package version + uses: martinbeentjes/npm-get-version-action@v1.2.3 + id: package-version + - name: Build and push + uses: docker/build-push-action@v2 + with: + context: . + platforms: linux/aarch64 + push: true + tags: coollabsio/coolify:${{steps.package-version.outputs.current-version}}-aarch64 + cache-from: type=registry,ref=coollabsio/coolify:buildcache-aarch64 + cache-to: type=registry,ref=coollabsio/coolify:buildcache-aarch64,mode=max merge-manifest: runs-on: ubuntu-latest - needs: [amd64-build, arm64-build] + needs: [amd64, arm64, aarch64] steps: - name: Checkout uses: actions/checkout@v3 @@ -77,7 +103,7 @@ jobs: id: package-version - name: Create & publish manifest run: | - docker manifest create coollabsio/coolify:${{steps.package-version.outputs.current-version}} --amend coollabsio/coolify:${{steps.package-version.outputs.current-version}}-amd64 --amend coollabsio/coolify:${{steps.package-version.outputs.current-version}}-arm64 + docker manifest create coollabsio/coolify:${{steps.package-version.outputs.current-version}} --amend coollabsio/coolify:${{steps.package-version.outputs.current-version}}-amd64 --amend coollabsio/coolify:${{steps.package-version.outputs.current-version}}-arm64 --amend coollabsio/coolify:${{steps.package-version.outputs.current-version}}-aarch64 docker manifest push coollabsio/coolify:${{steps.package-version.outputs.current-version}} - uses: sarisia/actions-status-discord@v1 if: always() diff --git a/.github/workflows/staging-release.yml b/.github/workflows/staging-release.yml index b8b05da44..42e8f26dc 100644 --- a/.github/workflows/staging-release.yml +++ b/.github/workflows/staging-release.yml @@ -59,7 +59,7 @@ jobs: context: . platforms: linux/amd64 push: true - tags: coollabsio/coolify:next-amd64,coollabsio/coolify:next-test + tags: coollabsio/coolify:next-amd64 cache-from: type=registry,ref=coollabsio/coolify:buildcache-next-amd64 cache-to: type=registry,ref=coollabsio/coolify:buildcache-next-amd64,mode=max aarch64: @@ -87,7 +87,7 @@ jobs: context: . platforms: linux/aarch64 push: true - tags: coollabsio/coolify:next-aarch64,coollabsio/coolify:next-test + tags: coollabsio/coolify:next-aarch64 cache-from: type=registry,ref=coollabsio/coolify:buildcache-next-aarch64 cache-to: type=registry,ref=coollabsio/coolify:buildcache-next-aarch64,mode=max merge-manifest: From 25f250310ed6c5f578083e683fa8b87ccee5dc02 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 12 Oct 2022 10:18:28 +0200 Subject: [PATCH 25/34] fix: dev container --- Dockerfile-dev | 24 ++++++++++++++---------- package.json | 2 +- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/Dockerfile-dev b/Dockerfile-dev index 1ea476959..198307656 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -1,9 +1,15 @@ -ARG PNPM_VERSION=7.11.0 -ARG NPM_VERSION=8.19.1 - FROM node:18-slim ENV NODE_ENV development ARG TARGETPLATFORM +ARG PNPM_VERSION=7.11.0 +ARG NPM_VERSION=8.19.1 +# https://download.docker.com/linux/static/stable/ +ARG DOCKER_VERSION=20.10.18 +# https://github.com/docker/compose/releases +# Reverted to 2.6.1 because of this https://github.com/docker/compose/issues/9704. 2.9.0 still has a bug. +ARG DOCKER_COMPOSE_VERSION=2.6.1 +# https://github.com/buildpacks/pack/releases +ARG PACK_VERSION=v0.27.0 WORKDIR /app RUN npm --no-update-notifier --no-fund --global install pnpm@${PNPM_VERSION} @@ -14,14 +20,12 @@ RUN npm --no-update-notifier --no-fund --global install pnpm@${PNPM_VERSION} RUN npm install -g npm@${PNPM_VERSION} RUN mkdir -p ~/.docker/cli-plugins/ -# https://download.docker.com/linux/static/stable/ -RUN curl -SL https://cdn.coollabs.io/bin/$TARGETPLATFORM/docker-20.10.9 -o /usr/bin/docker -# https://github.com/docker/compose/releases -# Reverted to 2.6.1 because of this https://github.com/docker/compose/issues/9704. 2.9.0 still has a bug. -RUN curl -SL https://cdn.coollabs.io/bin/$TARGETPLATFORM/docker-compose-linux-2.6.1 -o ~/.docker/cli-plugins/docker-compose -RUN chmod +x ~/.docker/cli-plugins/docker-compose /usr/bin/docker -RUN (curl -sSL "https://github.com/buildpacks/pack/releases/download/v0.27.0/pack-v0.27.0-linux.tgz" | tar -C /usr/local/bin/ --no-same-owner -xzv pack) +RUN curl -SL https://cdn.coollabs.io/bin/$TARGETPLATFORM/docker-$DOCKER_VERSION -o /usr/bin/docker +RUN curl -SL https://cdn.coollabs.io/bin/$TARGETPLATFORM/docker-compose-linux-$DOCKER_COMPOSE_VERSION -o ~/.docker/cli-plugins/docker-compose +RUN curl -SL https://cdn.coollabs.io/bin/$TARGETPLATFORM/pack-$PACK_VERSION -o /usr/local/bin/pack + +RUN chmod +x ~/.docker/cli-plugins/docker-compose /usr/bin/docker /usr/local/bin/pack EXPOSE 3000 ENV CHECKPOINT_DISABLE=1 \ No newline at end of file diff --git a/package.json b/package.json index e692b61f6..6b17b3b66 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "format:api": "NODE_ENV=development pnpm run --filter api format", "lint": "run-p -l -n lint:*", "lint:api": "NODE_ENV=development pnpm run --filter api lint", - "dev:container": "docker-compose -f docker-compose-dev.yaml up || docker compose -f docker-compose-dev.yaml up", + "dev:container": "docker-compose -f docker-compose-dev.yaml up --build || docker compose -f docker-compose-dev.yaml up --build", "dev": "run-p -l -n dev:api dev:ui", "dev:api": "NODE_ENV=development pnpm run --filter api dev", "dev:ui": "NODE_ENV=development pnpm run --filter ui dev", From 7a053ce697751a7449bfff2b4fa1eb3926dee394 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 12 Oct 2022 11:11:18 +0200 Subject: [PATCH 26/34] fix: gitlab auth and compose reload --- apps/ui/src/lib/api.ts | 2 +- .../configuration/_GitlabRepositories.svelte | 2 +- .../[id]/configuration/buildpack.svelte | 103 +++++++++++++---- .../src/routes/applications/[id]/index.svelte | 105 ++++++++++++++---- 4 files changed, 167 insertions(+), 45 deletions(-) diff --git a/apps/ui/src/lib/api.ts b/apps/ui/src/lib/api.ts index ed781b8a7..7341d580b 100644 --- a/apps/ui/src/lib/api.ts +++ b/apps/ui/src/lib/api.ts @@ -110,7 +110,7 @@ async function send({ if ( response.status === 401 && !path.startsWith('https://api.github') && - !path.includes('/v4/user') + !path.includes('/v4/') ) { Cookies.remove('token'); } diff --git a/apps/ui/src/routes/applications/[id]/configuration/_GitlabRepositories.svelte b/apps/ui/src/routes/applications/[id]/configuration/_GitlabRepositories.svelte index 43c31fd01..94fc11885 100644 --- a/apps/ui/src/routes/applications/[id]/configuration/_GitlabRepositories.svelte +++ b/apps/ui/src/routes/applications/[id]/configuration/_GitlabRepositories.svelte @@ -95,7 +95,7 @@ if (newWindow?.closed) { clearInterval(timer); $appSession.tokens.gitlab = localStorage.getItem('gitLabToken'); - localStorage.removeItem('gitLabToken'); + // localStorage.removeItem('gitLabToken'); resolve(); } }, 100); diff --git a/apps/ui/src/routes/applications/[id]/configuration/buildpack.svelte b/apps/ui/src/routes/applications/[id]/configuration/buildpack.svelte index 30ec09fe9..da0b15974 100644 --- a/apps/ui/src/routes/applications/[id]/configuration/buildpack.svelte +++ b/apps/ui/src/routes/applications/[id]/configuration/buildpack.svelte @@ -37,7 +37,7 @@ import { onMount } from 'svelte'; import { page } from '$app/stores'; - import { get } from '$lib/api'; + import { get, getAPIUrl } from '$lib/api'; import { appSession } from '$lib/store'; import { t } from '$lib/translations'; import { buildPacks, findBuildPack, scanningTemplates } from '$lib/templates'; @@ -47,6 +47,8 @@ const { id } = $page.params; + let htmlUrl = application.gitSource.htmlUrl; + let scanning: boolean = true; let foundConfig: any = null; let packageManager: string = 'npm'; @@ -65,15 +67,47 @@ } } } + async function getGitlabToken() { + return await new Promise((resolve, reject) => { + const left = screen.width / 2 - 1020 / 2; + const top = screen.height / 2 - 618 / 2; + const newWindow = open( + `${htmlUrl}/oauth/authorize?client_id=${ + application.gitSource.gitlabApp.appId + }&redirect_uri=${getAPIUrl()}/webhooks/gitlab&response_type=code&scope=api+email+read_repository&state=${ + $page.params.id + }`, + 'GitLab', + 'resizable=1, scrollbars=1, fullscreen=0, height=618, width=1020,top=' + + top + + ', left=' + + left + + ', toolbar=0, menubar=0, status=0' + ); + const timer = setInterval(() => { + if (newWindow?.closed) { + clearInterval(timer); + $appSession.tokens.gitlab = localStorage.getItem('gitLabToken'); + localStorage.removeItem('gitLabToken'); + resolve(); + } + }, 100); + }); + } async function scanRepository(isPublicRepository: boolean): Promise { try { if (type === 'gitlab') { + const headers = isPublicRepository + ? {} + : { + Authorization: `Bearer ${$appSession.tokens.gitlab}` + }; if (isPublicRepository) { return; } const url = isPublicRepository ? `` : `/v4/projects/${projectId}/repository/tree`; const files = await get(`${apiUrl}${url}`, { - Authorization: `Bearer ${$appSession.tokens.gitlab}` + ...headers }); const packageJson = files.find( (file: { name: string; type: string }) => @@ -126,7 +160,19 @@ if (pnpmLock) packageManager = 'pnpm'; if (dockerComposeFileYml || dockerComposeFileYaml) { - foundConfig = findBuildPack('dockercompose', packageManager); + foundConfig = findBuildPack('compose', packageManager); + const id = dockerComposeFileYml.id || dockerComposeFileYaml.id; + const data = await get(`${apiUrl}/v4/projects/${projectId}/repository/blobs/${id}`, { + ...headers + }); + if (data?.content) { + const content = atob(data.content); + const dockerComposeJson = yaml.load(content) || null; + dockerComposeFile = JSON.stringify(dockerComposeJson); + dockerComposeFileLocation = dockerComposeFileYml + ? 'docker-compose.yml' + : 'docker-compose.yaml'; + } } else if (dockerfile) { foundConfig = findBuildPack('docker', packageManager); } else if (packageJson && !laravel) { @@ -236,7 +282,7 @@ foundConfig = findBuildPack('docker', packageManager); } else if (packageJson && !laravel) { const data: any = await get(`${packageJson.git_url}`, { - Authorization: `Bearer ${$appSession.tokens.github}`, + ...headers, Accept: 'application/vnd.github.v2.raw' }); const json = JSON.parse(data) || {}; @@ -264,27 +310,36 @@ error.message === '401 Unauthorized' ) { if (application.gitSource.gitlabAppId) { - let htmlUrl = application.gitSource.htmlUrl; - const left = screen.width / 2 - 1020 / 2; - const top = screen.height / 2 - 618 / 2; - const newWindow = open( - `${htmlUrl}/oauth/authorize?client_id=${application.gitSource.gitlabApp.appId}&redirect_uri=${window.location.origin}/webhooks/gitlab&response_type=code&scope=api+email+read_repository&state=${$page.params.id}`, - 'GitLab', - 'resizable=1, scrollbars=1, fullscreen=0, height=618, width=1020,top=' + - top + - ', left=' + - left + - ', toolbar=0, menubar=0, status=0' - ); - const timer = setInterval(() => { - if (newWindow?.closed) { - clearInterval(timer); - window.location.reload(); - } - }, 100); + if (!$appSession.tokens.gitlab) { + await getGitlabToken(); + } + scanRepository(isPublicRepository); + // let htmlUrl = application.gitSource.htmlUrl; + // const left = screen.width / 2 - 1020 / 2; + // const top = screen.height / 2 - 618 / 2; + // const newWindow = open( + // `${htmlUrl}/oauth/authorize?client_id=${ + // application.gitSource.gitlabApp.appId + // }&redirect_uri=${getAPIUrl()}/webhooks/gitlab&response_type=code&scope=api+email+read_repository&state=${ + // $page.params.id + // }`, + // 'GitLab', + // 'resizable=1, scrollbars=1, fullscreen=0, height=618, width=1020,top=' + + // top + + // ', left=' + + // left + + // ', toolbar=0, menubar=0, status=0' + // ); + // const timer = setInterval(() => { + // if (newWindow?.closed) { + // clearInterval(timer); + // $appSession.tokens.gitlab = localStorage.getItem('gitLabToken'); + // // localStorage.removeItem('gitLabToken' ); + + // } + // }, 100); } - } - if (error.message === 'Bad credentials') { + } else if (error.message === 'Bad credentials') { const { token } = await get(`/applications/${id}/configuration/githubToken`); $appSession.tokens.github = token; return await scanRepository(isPublicRepository); diff --git a/apps/ui/src/routes/applications/[id]/index.svelte b/apps/ui/src/routes/applications/[id]/index.svelte index 49edd971f..869c1488a 100644 --- a/apps/ui/src/routes/applications/[id]/index.svelte +++ b/apps/ui/src/routes/applications/[id]/index.svelte @@ -33,7 +33,7 @@ import { page } from '$app/stores'; import { onMount } from 'svelte'; import Select from 'svelte-select'; - import { get, post } from '$lib/api'; + import { get, getAPIUrl, post } from '$lib/api'; import cuid from 'cuid'; import { addToast, @@ -75,6 +75,7 @@ let autodeploy = application.settings.autodeploy; let isBot = application.settings.isBot; let isDBBranching = application.settings.isDBBranching; + let htmlUrl = application.gitSource.htmlUrl; let dockerComposeFile = JSON.parse(application.dockerComposeFile) || null; let dockerComposeServices: any[] = []; @@ -321,28 +322,94 @@ return false; } } + async function getGitlabToken() { + return await new Promise((resolve, reject) => { + const left = screen.width / 2 - 1020 / 2; + const top = screen.height / 2 - 618 / 2; + const newWindow = open( + `${htmlUrl}/oauth/authorize?client_id=${ + application.gitSource.gitlabApp.appId + }&redirect_uri=${getAPIUrl()}/webhooks/gitlab&response_type=code&scope=api+email+read_repository&state=${ + $page.params.id + }`, + 'GitLab', + 'resizable=1, scrollbars=1, fullscreen=0, height=618, width=1020,top=' + + top + + ', left=' + + left + + ', toolbar=0, menubar=0, status=0' + ); + const timer = setInterval(() => { + if (newWindow?.closed) { + clearInterval(timer); + $appSession.tokens.gitlab = localStorage.getItem('gitLabToken'); + localStorage.removeItem('gitLabToken'); + resolve(); + } + }, 100); + }); + } async function reloadCompose() { try { - const headers = isPublicRepository - ? {} - : { - Authorization: `token ${$appSession.tokens.github}` - }; - const data = await get( - `${apiUrl}/repos/${repository}/contents/${dockerComposeFileLocation}?ref=${branch}`, - { - ...headers, - Accept: 'application/vnd.github.v2.json' + if (application.gitSource.type === 'github') { + const headers = isPublicRepository + ? {} + : { + Authorization: `token ${$appSession.tokens.github}` + }; + const data = await get( + `${apiUrl}/repos/${repository}/contents/${dockerComposeFileLocation}?ref=${branch}`, + { + ...headers, + Accept: 'application/vnd.github.v2.json' + } + ); + if (data?.content) { + const content = atob(data.content); + let dockerComposeFileContent = JSON.stringify(yaml.load(content) || null); + let dockerComposeFileContentJSON = JSON.parse(dockerComposeFileContent); + dockerComposeServices = normalizeDockerServices(dockerComposeFileContentJSON?.services); + application.dockerComposeFile = dockerComposeFileContent; + await handleSubmit(false); } - ); - if (data?.content) { - const content = atob(data.content); - let dockerComposeFileContent = JSON.stringify(yaml.load(content) || null); - let dockerComposeFileContentJSON = JSON.parse(dockerComposeFileContent); - dockerComposeServices = normalizeDockerServices(dockerComposeFileContentJSON?.services); - application.dockerComposeFile = dockerComposeFileContent; - await handleSubmit(false); } + if (application.gitSource.type === 'gitlab') { + if (!$appSession.tokens.gitlab) { + await getGitlabToken(); + } + const headers = isPublicRepository + ? {} + : { + Authorization: `Bearer ${$appSession.tokens.gitlab}` + }; + const url = isPublicRepository + ? `` + : `/v4/projects/${application.projectId}/repository/tree`; + const files = await get(`${apiUrl}${url}`, { + ...headers + }); + const dockerComposeFileYml = files.find( + (file: { name: string; type: string }) => + file.name === dockerComposeFileLocation && file.type === 'blob' + ); + const id = dockerComposeFileYml.id; + + const data = await get( + `${apiUrl}/v4/projects/${application.projectId}/repository/blobs/${id}`, + { + ...headers + } + ); + if (data?.content) { + const content = atob(data.content); + let dockerComposeFileContent = JSON.stringify(yaml.load(content) || null); + let dockerComposeFileContentJSON = JSON.parse(dockerComposeFileContent); + dockerComposeServices = normalizeDockerServices(dockerComposeFileContentJSON?.services); + application.dockerComposeFile = dockerComposeFileContent; + await handleSubmit(false); + } + } + addToast({ message: 'Compose file reloaded.', type: 'success' From f734154da89db7c49e70fdff4b8c1b6067f77a21 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 12 Oct 2022 11:17:02 +0200 Subject: [PATCH 27/34] fix: check compose domains in general --- apps/api/src/lib/common.ts | 6 ++++-- .../src/routes/applications/[id]/index.svelte | 17 ++++++++++++++--- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/apps/api/src/lib/common.ts b/apps/api/src/lib/common.ts index bb52979ec..d4d2e5d53 100644 --- a/apps/api/src/lib/common.ts +++ b/apps/api/src/lib/common.ts @@ -264,7 +264,9 @@ export async function isDomainConfigured({ where: { OR: [ { fqdn: { endsWith: `//${nakedDomain}` } }, - { fqdn: { endsWith: `//www.${nakedDomain}` } } + { fqdn: { endsWith: `//www.${nakedDomain}` } }, + { dockerComposeConfiguration: { contains: `//${nakedDomain}` } }, + { dockerComposeConfiguration: { contains: `//www.${nakedDomain}` } } ], id: { not: id }, destinationDocker: { @@ -598,7 +600,7 @@ export async function executeDockerCmd({ debug, buildId, applicationId, dockerId command = command.replace(/docker compose/gi, 'docker-compose'); } } - if (command.startsWith(`docker build`) || command.startsWith(`pack build`)|| command.startsWith(`docker compose build`)) { + if (command.startsWith(`docker build`) || command.startsWith(`pack build`) || command.startsWith(`docker compose build`)) { return await asyncExecShellStream({ debug, buildId, applicationId, command, engine }); } return await execaCommand(command, { env: { DOCKER_BUILDKIT: "1", DOCKER_HOST: engine }, shell: true }) diff --git a/apps/ui/src/routes/applications/[id]/index.svelte b/apps/ui/src/routes/applications/[id]/index.svelte index 869c1488a..2679f4418 100644 --- a/apps/ui/src/routes/applications/[id]/index.svelte +++ b/apps/ui/src/routes/applications/[id]/index.svelte @@ -243,13 +243,24 @@ nonWWWDomain = application.fqdn && getDomain(application.fqdn).replace(/^www\./, ''); if (application.deploymentType) application.deploymentType = application.deploymentType.toLowerCase(); - !isBot && - (await post(`/applications/${id}/check`, { + if (!isBot) { + await post(`/applications/${id}/check`, { fqdn: application.fqdn, forceSave, dualCerts, exposePort: application.exposePort - })); + }); + for (const service of dockerComposeServices) { + if (dockerComposeConfiguration[service.name].fqdn) { + await post(`/applications/${id}/check`, { + fqdn: dockerComposeConfiguration[service.name].fqdn, + forceSave, + dualCerts, + exposePort: application.exposePort + }); + } + } + } await post(`/applications/${id}`, { ...application, baseDatabaseBranch, From 3ee3ab0ad14e24d68ff6dcd6c6053959e1d536ba Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 12 Oct 2022 11:27:13 +0200 Subject: [PATCH 28/34] fix: port required if fqdn is set --- apps/ui/src/routes/applications/[id]/index.svelte | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/ui/src/routes/applications/[id]/index.svelte b/apps/ui/src/routes/applications/[id]/index.svelte index 2679f4418..acfe3c686 100644 --- a/apps/ui/src/routes/applications/[id]/index.svelte +++ b/apps/ui/src/routes/applications/[id]/index.svelte @@ -1074,6 +1074,7 @@ readonly={!$appSession.isAdmin} name="port" id="port" + required={!!dockerComposeConfiguration[service.name].fqdn} bind:value={dockerComposeConfiguration[service.name].port} />
    From 24e7f547fad7d84d7602d3586817a28d6b9aab78 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 12 Oct 2022 11:42:45 +0200 Subject: [PATCH 29/34] feat: monitoring by container --- .../routes/api/v1/applications/handlers.ts | 18 +++ .../src/routes/api/v1/applications/index.ts | 3 +- .../applications/[id]/logs/index.svelte | 2 +- .../src/routes/applications/[id]/usage.svelte | 104 ++++++++++++++---- 4 files changed, 104 insertions(+), 23 deletions(-) diff --git a/apps/api/src/routes/api/v1/applications/handlers.ts b/apps/api/src/routes/api/v1/applications/handlers.ts index 686eea8cc..d179aabfa 100644 --- a/apps/api/src/routes/api/v1/applications/handlers.ts +++ b/apps/api/src/routes/api/v1/applications/handlers.ts @@ -677,6 +677,24 @@ export async function getUsage(request) { return errorHandler({ status, message }) } } + +export async function getUsageByContainer(request) { + try { + const { id, containerId } = request.params + const teamId = request.user?.teamId; + let usage = {}; + + const application: any = await getApplicationFromDB(id, teamId); + if (application.destinationDockerId) { + [usage] = await Promise.all([getContainerUsage(application.destinationDocker.id, containerId)]); + } + return { + usage + } + } catch ({ status, message }) { + return errorHandler({ status, message }) + } +} export async function deployApplication(request: FastifyRequest) { try { const { id } = request.params diff --git a/apps/api/src/routes/api/v1/applications/index.ts b/apps/api/src/routes/api/v1/applications/index.ts index 1c230210f..4ac98d895 100644 --- a/apps/api/src/routes/api/v1/applications/index.ts +++ b/apps/api/src/routes/api/v1/applications/index.ts @@ -1,6 +1,6 @@ import { FastifyPluginAsync } from 'fastify'; import { OnlyId } from '../../../../types'; -import { cancelDeployment, checkDNS, checkDomain, checkRepository, cleanupUnconfiguredApplications, deleteApplication, deleteSecret, deleteStorage, deployApplication, getApplication, getApplicationLogs, getApplicationStatus, getBuildIdLogs, getBuildPack, getBuilds, getGitHubToken, getGitLabSSHKey, getImages, getPreviews, getPreviewStatus, getSecrets, getStorages, getUsage, listApplications, loadPreviews, newApplication, restartApplication, restartPreview, saveApplication, saveApplicationSettings, saveApplicationSource, saveBuildPack, saveConnectedDatabase, saveDeployKey, saveDestination, saveGitLabSSHKey, saveRepository, saveSecret, saveStorage, stopApplication, stopPreviewApplication, updatePreviewSecret, updateSecret } from './handlers'; +import { cancelDeployment, checkDNS, checkDomain, checkRepository, cleanupUnconfiguredApplications, deleteApplication, deleteSecret, deleteStorage, deployApplication, getApplication, getApplicationLogs, getApplicationStatus, getBuildIdLogs, getBuildPack, getBuilds, getGitHubToken, getGitLabSSHKey, getImages, getPreviews, getPreviewStatus, getSecrets, getStorages, getUsage, getUsageByContainer, listApplications, loadPreviews, newApplication, restartApplication, restartPreview, saveApplication, saveApplicationSettings, saveApplicationSource, saveBuildPack, saveConnectedDatabase, saveDeployKey, saveDestination, saveGitLabSSHKey, saveRepository, saveSecret, saveStorage, stopApplication, stopPreviewApplication, updatePreviewSecret, updateSecret } from './handlers'; import type { CancelDeployment, CheckDNS, CheckDomain, CheckRepository, DeleteApplication, DeleteSecret, DeleteStorage, DeployApplication, GetApplicationLogs, GetBuildIdLogs, GetBuilds, GetImages, RestartPreviewApplication, SaveApplication, SaveApplicationSettings, SaveApplicationSource, SaveDeployKey, SaveDestination, SaveSecret, SaveStorage, StopPreviewApplication } from './types'; @@ -51,6 +51,7 @@ const root: FastifyPluginAsync = async (fastify): Promise => { fastify.get('/:id/logs/build/:buildId', async (request) => await getBuildIdLogs(request)); fastify.get('/:id/usage', async (request) => await getUsage(request)) + fastify.get('/:id/usage/:containerId', async (request) => await getUsageByContainer(request)) fastify.post('/:id/deploy', async (request) => await deployApplication(request)) fastify.post('/:id/cancel', async (request, reply) => await cancelDeployment(request, reply)); diff --git a/apps/ui/src/routes/applications/[id]/logs/index.svelte b/apps/ui/src/routes/applications/[id]/logs/index.svelte index a716f0788..2de662d12 100644 --- a/apps/ui/src/routes/applications/[id]/logs/index.svelte +++ b/apps/ui/src/routes/applications/[id]/logs/index.svelte @@ -125,7 +125,7 @@ `${application.id}${service.name ? `-${service.name}` : ''}`} class="w-full rounded p-5 hover:bg-primary font-bold" > - Container: {application.id}{service.name ? `-${service.name}` : ''} {/each}
    diff --git a/apps/ui/src/routes/applications/[id]/usage.svelte b/apps/ui/src/routes/applications/[id]/usage.svelte index f9919ecee..7067c677e 100644 --- a/apps/ui/src/routes/applications/[id]/usage.svelte +++ b/apps/ui/src/routes/applications/[id]/usage.svelte @@ -1,11 +1,14 @@ @@ -38,21 +76,45 @@
    Monitoring
    • -
    -
    -
    -
    Used Memory / Memory Limit
    -
    {usage?.MemUsage}
    -
    +
    + {#each services as service} + + {/each} +
    +{#if selectedService} +
    + {#if usageLoading} + + Streaming logs + {/if} +
    +
    +
    Used Memory / Memory Limit
    +
    {usage?.MemUsage}
    +
    -
    -
    Used CPU
    -
    {usage?.CPUPerc}
    -
    +
    +
    Used CPU
    +
    {usage?.CPUPerc}
    +
    -
    -
    Network IO
    -
    {usage?.NetIO}
    +
    +
    Network IO
    +
    {usage?.NetIO}
    +
    -
    +{/if} From bef5da49cf678f1de0387e66bd0ca717fabfc6be Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 12 Oct 2022 11:43:47 +0200 Subject: [PATCH 30/34] remove text --- apps/ui/src/routes/applications/[id]/usage.svelte | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/ui/src/routes/applications/[id]/usage.svelte b/apps/ui/src/routes/applications/[id]/usage.svelte index 7067c677e..395fd0bf1 100644 --- a/apps/ui/src/routes/applications/[id]/usage.svelte +++ b/apps/ui/src/routes/applications/[id]/usage.svelte @@ -96,8 +96,7 @@ + /> Streaming logs {/if}
    From 324038486f7268403f05f74ddb1aff9ce81a97ef Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 12 Oct 2022 12:02:47 +0200 Subject: [PATCH 31/34] fixes --- apps/api/src/lib/docker.ts | 3 + .../routes/applications/[id]/__layout.svelte | 188 ++++++++++-------- .../[id]/configuration/buildpack.svelte | 6 +- apps/ui/src/routes/index.svelte | 1 + 4 files changed, 110 insertions(+), 88 deletions(-) diff --git a/apps/api/src/lib/docker.ts b/apps/api/src/lib/docker.ts index dff59b7db..a4d36b7fc 100644 --- a/apps/api/src/lib/docker.ts +++ b/apps/api/src/lib/docker.ts @@ -87,6 +87,9 @@ export async function removeContainer({ await executeDockerCmd({ dockerId, command: `docker stop -t 0 ${id}` }) await executeDockerCmd({ dockerId, command: `docker rm ${id}` }) } + if (JSON.parse(stdout).Status === 'exited') { + await executeDockerCmd({ dockerId, command: `docker rm ${id}` }) + } } catch (error) { throw error; } diff --git a/apps/ui/src/routes/applications/[id]/__layout.svelte b/apps/ui/src/routes/applications/[id]/__layout.svelte index 64f8a4ea6..dc51e0185 100644 --- a/apps/ui/src/routes/applications/[id]/__layout.svelte +++ b/apps/ui/src/routes/applications/[id]/__layout.svelte @@ -272,14 +272,13 @@ > {#if $status.application.overallStatus === 'degraded' && application.buildPack !== 'compose'} Application Error - Application exited with an error! {/if} {#if $status.application.initialLoading} {:else if $status.application.overallStatus === 'healthy'} - - {#if application.buildPack !== 'compose'} - -{/if} + + {#if application.buildPack !== 'compose'} + + {/if} + {/if} {/if} {#if $location && $status.application.overallStatus === 'healthy'} diff --git a/apps/ui/src/routes/applications/[id]/configuration/buildpack.svelte b/apps/ui/src/routes/applications/[id]/configuration/buildpack.svelte index da0b15974..37e18a24d 100644 --- a/apps/ui/src/routes/applications/[id]/configuration/buildpack.svelte +++ b/apps/ui/src/routes/applications/[id]/configuration/buildpack.svelte @@ -102,10 +102,8 @@ : { Authorization: `Bearer ${$appSession.tokens.gitlab}` }; - if (isPublicRepository) { - return; - } - const url = isPublicRepository ? `` : `/v4/projects/${projectId}/repository/tree`; + + const url = isPublicRepository ? `/projects/${projectId}/repository/tree` : `/v4/projects/${projectId}/repository/tree`; const files = await get(`${apiUrl}${url}`, { ...headers }); diff --git a/apps/ui/src/routes/index.svelte b/apps/ui/src/routes/index.svelte index f23e74f48..1cd83bfe0 100644 --- a/apps/ui/src/routes/index.svelte +++ b/apps/ui/src/routes/index.svelte @@ -237,6 +237,7 @@ (application.id && application.id.toLowerCase().includes($search.toLowerCase())) || (application.name && application.name.toLowerCase().includes($search.toLowerCase())) || (application.fqdn && application.fqdn.toLowerCase().includes($search.toLowerCase())) || + (application.dockerComposeConfiguration && application.dockerComposeConfiguration.toLowerCase().includes($search.toLowerCase())) || (application.repository && application.repository.toLowerCase().includes($search.toLowerCase())) || (application.buildpack && From bf252f7f2039a55a1ae5cc0cfb7c146e789f9701 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 12 Oct 2022 13:50:28 +0200 Subject: [PATCH 32/34] fix: appwrite v1 missing containers --- apps/api/src/lib/services/handlers.ts | 118 ++++++++++++++++---------- 1 file changed, 71 insertions(+), 47 deletions(-) diff --git a/apps/api/src/lib/services/handlers.ts b/apps/api/src/lib/services/handlers.ts index 3dcf0d5e8..524ebeb86 100644 --- a/apps/api/src/lib/services/handlers.ts +++ b/apps/api/src/lib/services/handlers.ts @@ -1410,6 +1410,7 @@ async function startAppWriteService(request: FastifyRequest) { depends_on: [ `${id}-mariadb`, `${id}-redis`, + `${id}-influxdb`, ], environment: [ "_APP_ENV=production", @@ -1772,54 +1773,77 @@ async function startAppWriteService(request: FastifyRequest) { ], ...defaultComposeConfiguration(network), }, - + [`${id}-usage-timeseries`]: { + image: `${image}:${version}`, + container_name: `${id}-usage`, + labels: makeLabelForServices('appwrite'), + entrypoint: "usage --type=timeseries", + depends_on: [ + `${id}-mariadb`, + `${id}-influxdb`, + ], + environment: [ + "_APP_ENV=production", + `_APP_OPENSSL_KEY_V1=${opensslKeyV1}`, + `_APP_DB_HOST=${mariadbHost}`, + `_APP_DB_PORT=${mariadbPort}`, + `_APP_DB_SCHEMA=${mariadbDatabase}`, + `_APP_DB_USER=${mariadbUser}`, + `_APP_DB_PASS=${mariadbPassword}`, + `_APP_INFLUXDB_HOST=${id}-influxdb`, + "_APP_INFLUXDB_PORT=8086", + `_APP_REDIS_HOST=${id}-redis`, + "_APP_REDIS_PORT=6379", + `OPEN_RUNTIMES_NETWORK=${network}`, + ...secrets + ], + ...defaultComposeConfiguration(network), + }, + [`${id}-usage-database`]: { + image: `${image}:${version}`, + container_name: `${id}-usage-database`, + labels: makeLabelForServices('appwrite'), + entrypoint: "usage --type=database", + depends_on: [ + `${id}-mariadb`, + `${id}-influxdb`, + ], + environment: [ + "_APP_ENV=production", + `_APP_OPENSSL_KEY_V1=${opensslKeyV1}`, + `_APP_DB_HOST=${mariadbHost}`, + `_APP_DB_PORT=${mariadbPort}`, + `_APP_DB_SCHEMA=${mariadbDatabase}`, + `_APP_DB_USER=${mariadbUser}`, + `_APP_DB_PASS=${mariadbPassword}`, + `_APP_INFLUXDB_HOST=${id}-influxdb`, + "_APP_INFLUXDB_PORT=8086", + `_APP_REDIS_HOST=${id}-redis`, + "_APP_REDIS_PORT=6379", + `OPEN_RUNTIMES_NETWORK=${network}`, + ...secrets + ], + ...defaultComposeConfiguration(network), + }, + [`${id}-influxdb`]: { + image: "appwrite/influxdb:1.5.0", + container_name: `${id}-influxdb`, + volumes: [ + `${id}-influxdb:/var/lib/influxdb:rw` + ], + ...defaultComposeConfiguration(network), + }, + [`${id}-telegraf`]: { + image: "appwrite/telegraf:1.4.0", + container_name: `${id}-telegraf`, + environment: [ + `_APP_INFLUXDB_HOST=${id}-influxdb`, + "_APP_INFLUXDB_PORT=8086", + `OPEN_RUNTIMES_NETWORK=${network}`, + ], + ...defaultComposeConfiguration(network), + } }; - dockerCompose[id].depends_on.push(`${id}-influxdb`); - dockerCompose[`${id}-usage`] = { - image: `${image}:${version}`, - container_name: `${id}-usage`, - labels: makeLabelForServices('appwrite'), - entrypoint: "usage", - depends_on: [ - `${id}-mariadb`, - `${id}-influxdb`, - ], - environment: [ - "_APP_ENV=production", - `_APP_OPENSSL_KEY_V1=${opensslKeyV1}`, - `_APP_DB_HOST=${mariadbHost}`, - `_APP_DB_PORT=${mariadbPort}`, - `_APP_DB_SCHEMA=${mariadbDatabase}`, - `_APP_DB_USER=${mariadbUser}`, - `_APP_DB_PASS=${mariadbPassword}`, - `_APP_INFLUXDB_HOST=${id}-influxdb`, - "_APP_INFLUXDB_PORT=8086", - `_APP_REDIS_HOST=${id}-redis`, - "_APP_REDIS_PORT=6379", - `OPEN_RUNTIMES_NETWORK=${network}`, - ...secrets - ], - ...defaultComposeConfiguration(network), - } - dockerCompose[`${id}-influxdb`] = { - image: "appwrite/influxdb:1.5.0", - container_name: `${id}-influxdb`, - volumes: [ - `${id}-influxdb:/var/lib/influxdb:rw` - ], - ...defaultComposeConfiguration(network), - } - dockerCompose[`${id}-telegraf`] = { - image: "appwrite/telegraf:1.4.0", - container_name: `${id}-telegraf`, - environment: [ - `_APP_INFLUXDB_HOST=${id}-influxdb`, - "_APP_INFLUXDB_PORT=8086", - `OPEN_RUNTIMES_NETWORK=${network}`, - ], - ...defaultComposeConfiguration(network), - } - const composeFile: any = { version: '3.8', services: dockerCompose, From 68c983923eeb0011d0a061ab3133a0a690f47ee3 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 12 Oct 2022 14:08:00 +0200 Subject: [PATCH 33/34] fix: dockerfile --- .github/workflows/staging-release.yml | 34 +++------------------------ Dockerfile | 16 ++++++------- 2 files changed, 11 insertions(+), 39 deletions(-) diff --git a/.github/workflows/staging-release.yml b/.github/workflows/staging-release.yml index 42e8f26dc..2280167d8 100644 --- a/.github/workflows/staging-release.yml +++ b/.github/workflows/staging-release.yml @@ -59,40 +59,12 @@ jobs: context: . platforms: linux/amd64 push: true - tags: coollabsio/coolify:next-amd64 + tags: coollabsio/coolify:next-amd64 cache-from: type=registry,ref=coollabsio/coolify:buildcache-next-amd64 cache-to: type=registry,ref=coollabsio/coolify:buildcache-next-amd64,mode=max - aarch64: - runs-on: [self-hosted, arm64] - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - ref: "next" - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Get current package version - uses: martinbeentjes/npm-get-version-action@v1.2.3 - id: package-version - - name: Build and push - uses: docker/build-push-action@v3 - with: - context: . - platforms: linux/aarch64 - push: true - tags: coollabsio/coolify:next-aarch64 - cache-from: type=registry,ref=coollabsio/coolify:buildcache-next-aarch64 - cache-to: type=registry,ref=coollabsio/coolify:buildcache-next-aarch64,mode=max merge-manifest: runs-on: ubuntu-latest - needs: [arm64, amd64, aarch64] + needs: [arm64, amd64] steps: - name: Checkout uses: actions/checkout@v3 @@ -107,7 +79,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Create & publish manifest run: | - docker manifest create coollabsio/coolify:next --amend coollabsio/coolify:next-amd64 --amend coollabsio/coolify:next-arm64 --amend coollabsio/coolify:next-aarch64 + docker manifest create coollabsio/coolify:next --amend coollabsio/coolify:next-amd64 --amend coollabsio/coolify:next-arm64 docker manifest push coollabsio/coolify:next - uses: sarisia/actions-status-discord@v1 if: always() diff --git a/Dockerfile b/Dockerfile index 2f14248dc..8cbc68037 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,4 @@ ARG PNPM_VERSION=7.11.0 -ARG NPM_VERSION=8.19.1 -# https://download.docker.com/linux/static/stable/ -ARG DOCKER_VERSION=20.10.18 -# https://github.com/docker/compose/releases -# Reverted to 2.6.1 because of this https://github.com/docker/compose/issues/9704. 2.9.0 still has a bug. -ARG DOCKER_COMPOSE_VERSION=2.6.1 -# https://github.com/buildpacks/pack/releases -ARG PACK_VERSION=v0.27.0 FROM node:18-slim as build WORKDIR /app @@ -24,6 +16,14 @@ WORKDIR /app ENV NODE_ENV production ARG TARGETPLATFORM +# https://download.docker.com/linux/static/stable/ +ARG DOCKER_VERSION=20.10.18 +# https://github.com/docker/compose/releases +# Reverted to 2.6.1 because of this https://github.com/docker/compose/issues/9704. 2.9.0 still has a bug. +ARG DOCKER_COMPOSE_VERSION=2.6.1 +# https://github.com/buildpacks/pack/releases +ARG PACK_VERSION=v0.27.0 + RUN apt update && apt -y install --no-install-recommends ca-certificates git git-lfs openssh-client curl jq cmake sqlite3 openssl psmisc python3 RUN apt-get clean autoclean && apt-get autoremove --yes && rm -rf /var/lib/{apt,dpkg,cache,log}/ RUN npm --no-update-notifier --no-fund --global install pnpm@${PNPM_VERSION} From ae2d141f0d22d1a8db163626ee2e69106963043b Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Wed, 12 Oct 2022 14:27:13 +0200 Subject: [PATCH 34/34] fix: pull does not work remotely on huge compose file --- apps/api/src/lib/services/handlers.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/api/src/lib/services/handlers.ts b/apps/api/src/lib/services/handlers.ts index 524ebeb86..bcaf07d36 100644 --- a/apps/api/src/lib/services/handlers.ts +++ b/apps/api/src/lib/services/handlers.ts @@ -1892,7 +1892,9 @@ async function startAppWriteService(request: FastifyRequest) { } } async function startServiceContainers(dockerId, composeFileDestination) { - await executeDockerCmd({ dockerId, command: `docker compose -f ${composeFileDestination} pull` }) + try { + await executeDockerCmd({ dockerId, command: `docker compose -f ${composeFileDestination} pull` }) + } catch (error) { } await executeDockerCmd({ dockerId, command: `docker compose -f ${composeFileDestination} build --no-cache` }) await executeDockerCmd({ dockerId, command: `docker compose -f ${composeFileDestination} create` }) await executeDockerCmd({ dockerId, command: `docker compose -f ${composeFileDestination} start` })