Shiloh/lasthourcloud
Shiloh/lasthourcloud
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 4 Wiki Activity

feat: public repo deployment

Browse Source
This commit is contained in:
Andras Bacsai 2022-08-18 15:29:59 +02:00
parent 0c24134ac2
commit 4e7e9b2cfc
12 changed files with 159 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
apps/api/src/jobs/deployApplication.ts
View File

@ -69,6 +69,30 @@ import * as buildpacks from '../lib/buildPacks';
dockerFileLocation,
denoMainFile
} = message
const currentHash = crypto
.createHash('sha256')
.update(
JSON.stringify({
pythonWSGI,
pythonModule,
pythonVariable,
deploymentType,
denoOptions,
baseImage,
baseBuildImage,
buildPack,
port,
exposePort,
installCommand,
buildCommand,
startCommand,
secrets,
branch,
repository,
fqdn
})
)
.digest('hex');
try {
const { debug } = settings;
if (concurrency === 1) {
@ -131,7 +155,8 @@ import * as buildpacks from '../lib/buildPacks';
htmlUrl: gitSource.htmlUrl,
projectId,
deployKeyId: gitSource.gitlabApp?.deployKeyId || null,
privateSshKey: decrypt(gitSource.gitlabApp?.privateSshKey) || null
privateSshKey: decrypt(gitSource.gitlabApp?.privateSshKey) || null,
forPublic: gitSource.forPublic
});
if (!commit) {
throw new Error('No commit found?');
@ -146,38 +171,11 @@ import * as buildpacks from '../lib/buildPacks';
} catch (err) {
console.log(err);
}
if (!pullmergeRequestId) {
const currentHash = crypto
//@ts-ignore
.createHash('sha256')
.update(
JSON.stringify({
pythonWSGI,
pythonModule,
pythonVariable,
deploymentType,
denoOptions,
baseImage,
baseBuildImage,
buildPack,
port,
exposePort,
installCommand,
buildCommand,
startCommand,
secrets,
branch,
repository,
fqdn
})
)
.digest('hex');
if (configHash !== currentHash) {
await prisma.application.update({
where: { id: applicationId },
data: { configHash: currentHash }
});
deployNeeded = true;
if (configHash) {
await saveBuildLog({ line: 'Configuration changed.', buildId, applicationId });
@ -201,7 +199,7 @@ import * as buildpacks from '../lib/buildPacks';
}
await copyBaseConfigurationFiles(buildPack, workdir, buildId, applicationId, baseImage);
if (!imageFound || deployNeeded) {
// if (true) {
// if (true) {
if (buildpacks[buildPack])
await buildpacks[buildPack]({
dockerId: destinationDocker.id,
@ -336,6 +334,10 @@ import * as buildpacks from '../lib/buildPacks';
}
await saveBuildLog({ line: 'Proxy will be updated shortly.', buildId, applicationId });
await prisma.build.update({ where: { id: message.build_id }, data: { status: 'success' } });
if (!pullmergeRequestId) await prisma.application.update({
where: { id: applicationId },
data: { configHash: currentHash }
});
}
}

21
apps/api/src/lib/common.ts
View File

@ -1176,6 +1176,27 @@ export async function updatePasswordInDb(database, user, newPassword, isRoot) {
}
}
}
export async function checkExposedPort({ id, configuredPort, exposePort, dockerId, remoteIpAddress }: { id: string, configuredPort?: number, exposePort: number, dockerId: string, remoteIpAddress?: string }) {
if (exposePort < 1024 || exposePort > 65535) {
throw { status: 500, message: `Exposed Port needs to be between 1024 and 65535.` }
}
if (configuredPort) {
if (configuredPort !== exposePort) {
const availablePort = await getFreeExposedPort(id, exposePort, dockerId, remoteIpAddress);
if (availablePort.toString() !== exposePort.toString()) {
throw { status: 500, message: `Port ${exposePort} is already in use.` }
}
}
} else {
const availablePort = await getFreeExposedPort(id, exposePort, dockerId, remoteIpAddress);
console.log(availablePort, exposePort)
if (availablePort.toString() !== exposePort.toString()) {
throw { status: 500, message: `Port ${exposePort} is already in use.` }
}
}
}
export async function getFreeExposedPort(id, exposePort, dockerId, remoteIpAddress) {
const { default: getPort } = await import('get-port');
const applicationUsed = await (

74
apps/api/src/lib/importers/github.ts
View File

@ -12,7 +12,8 @@ export default async function ({
htmlUrl,
branch,
buildId,
customPort
customPort,
forPublic
}: {
applicationId: string;
workdir: string;
@ -23,40 +24,55 @@ export default async function ({
branch: string;
buildId: string;
customPort: number;
forPublic?: boolean;
}): Promise<string> {
const { default: got } = await import('got')
const url = htmlUrl.replace('https://', '').replace('http://', '');
await saveBuildLog({ line: 'GitHub importer started.', buildId, applicationId });
if (forPublic) {
await saveBuildLog({
line: `Cloning ${repository}:${branch} branch.`,
buildId,
applicationId
});
await asyncExecShell(
`git clone -q -b ${branch} https://${url}/${repository}.git ${workdir}/ && cd ${workdir} && git submodule update --init --recursive && git lfs pull && cd .. `
);
const body = await prisma.githubApp.findUnique({ where: { id: githubAppId } });
if (body.privateKey) body.privateKey = decrypt(body.privateKey);
const { privateKey, appId, installationId } = body
const githubPrivateKey = privateKey.replace(/\\n/g, '\n').replace(/"/g, '');
} else {
const body = await prisma.githubApp.findUnique({ where: { id: githubAppId } });
if (body.privateKey) body.privateKey = decrypt(body.privateKey);
const { privateKey, appId, installationId } = body
const githubPrivateKey = privateKey.replace(/\\n/g, '\n').replace(/"/g, '');
const payload = {
iat: Math.round(new Date().getTime() / 1000),
exp: Math.round(new Date().getTime() / 1000 + 60),
iss: appId
};
const jwtToken = jsonwebtoken.sign(payload, githubPrivateKey, {
algorithm: 'RS256'
});
const { token } = await got
.post(`${apiUrl}/app/installations/${installationId}/access_tokens`, {
headers: {
Authorization: `Bearer ${jwtToken}`,
Accept: 'application/vnd.github.machine-man-preview+json'
}
})
.json();
await saveBuildLog({
line: `Cloning ${repository}:${branch} branch.`,
buildId,
applicationId
});
await asyncExecShell(
`git clone -q -b ${branch} https://x-access-token:${token}@${url}/${repository}.git --config core.sshCommand="ssh -p ${customPort}" ${workdir}/ && cd ${workdir} && git submodule update --init --recursive && git lfs pull && cd .. `
);
const payload = {
iat: Math.round(new Date().getTime() / 1000),
exp: Math.round(new Date().getTime() / 1000 + 60),
iss: appId
};
const jwtToken = jsonwebtoken.sign(payload, githubPrivateKey, {
algorithm: 'RS256'
});
const { token } = await got
.post(`${apiUrl}/app/installations/${installationId}/access_tokens`, {
headers: {
Authorization: `Bearer ${jwtToken}`,
Accept: 'application/vnd.github.machine-man-preview+json'
}
})
.json();
await saveBuildLog({
line: `Cloning ${repository}:${branch} branch.`,
buildId,
applicationId
});
await asyncExecShell(
`git clone -q -b ${branch} https://x-access-token:${token}@${url}/${repository}.git --config core.sshCommand="ssh -p ${customPort}" ${workdir}/ && cd ${workdir} && git submodule update --init --recursive && git lfs pull && cd .. `
);
}
const { stdout: commit } = await asyncExecShell(`cd ${workdir}/ && git rev-parse HEAD`);
return commit.replace('\n', '');
}

20
apps/api/src/routes/api/v1/applications/handlers.ts
View File

@ -5,7 +5,7 @@ import axios from 'axios';
import { FastifyReply } from 'fastify';
import { day } from '../../../../lib/dayjs';
import { setDefaultBaseImage, setDefaultConfiguration } from '../../../../lib/buildPacks/common';
import { checkDomainsIsValidInDNS, checkDoubleBranch, decrypt, encrypt, errorHandler, executeDockerCmd, generateSshKeyPair, getContainerUsage, getDomain, getFreeExposedPort, isDev, isDomainConfigured, listSettings, prisma, stopBuild, uniqueName } from '../../../../lib/common';
import { checkDomainsIsValidInDNS, checkDoubleBranch, checkExposedPort, decrypt, encrypt, errorHandler, executeDockerCmd, generateSshKeyPair, getContainerUsage, getDomain, getFreeExposedPort, isDev, isDomainConfigured, listSettings, prisma, stopBuild, uniqueName } from '../../../../lib/common';
import { checkContainer, formatLabelsOnDocker, isContainerExited, removeContainer } from '../../../../lib/docker';
import { scheduler } from '../../../../lib/scheduler';
@ -238,6 +238,9 @@ export async function saveApplication(request: FastifyRequest<SaveApplication>,
if (exposePort) {
exposePort = Number(exposePort);
}
const { destinationDockerId } = await prisma.application.findUnique({ where: { id } })
if (exposePort) await checkExposedPort({ id, exposePort, dockerId: destinationDockerId })
if (denoOptions) denoOptions = denoOptions.trim();
const defaultConfiguration = await setDefaultConfiguration({
buildPack,
@ -392,18 +395,7 @@ export async function checkDNS(request: FastifyRequest<CheckDNS>) {
if (found) {
throw { status: 500, message: `Domain ${getDomain(fqdn).replace('www.', '')} is already in use!` }
}
if (exposePort) {
if (exposePort < 1024 || exposePort > 65535) {
throw { status: 500, message: `Exposed Port needs to be between 1024 and 65535.` }
}
if (configuredPort !== exposePort) {
const availablePort = await getFreeExposedPort(id, exposePort, dockerId, remoteIpAddress);
if (availablePort.toString() !== exposePort.toString()) {
throw { status: 500, message: `Port ${exposePort} is already in use.` }
}
}
}
await checkExposedPort({ id, configuredPort, exposePort, dockerId, remoteIpAddress })
if (isDNSCheckEnabled && !isDev && !forceSave) {
let hostname = request.hostname.split(':')[0];
if (remoteEngine) hostname = remoteIpAddress;
@ -500,7 +492,6 @@ export async function saveApplicationSource(request: FastifyRequest<SaveApplicat
try {
const { id } = request.params
const { gitSourceId, forPublic, type } = request.body
console.log({ id, gitSourceId, forPublic, type })
if (forPublic) {
const publicGit = await prisma.gitSource.findFirst({ where: { type, forPublic } });
await prisma.application.update({
@ -671,7 +662,6 @@ export async function saveSecret(request: FastifyRequest<SaveSecret>, reply: Fas
throw { status: 500, message: `Secret ${name} already exists.` }
} else {
value = encrypt(value.trim());
console.log({ value })
await prisma.secret.create({
data: { name, value, isBuildSecret, isPRMRSecret, application: { connect: { id } } }
});

1
apps/api/src/routes/api/v1/destinations/handlers.ts
View File

@ -79,7 +79,6 @@ export async function newDestination(request: FastifyRequest<NewDestination>, re
let { name, network, engine, isCoolifyProxyUsed, remoteIpAddress, remoteUser, remotePort } = request.body
if (id === 'new') {
console.log(engine)
if (engine) {
const { stdout } = await asyncExecShell(`DOCKER_HOST=unix:///var/run/docker.sock docker network ls --filter 'name=^${network}$' --format '{{json .}}'`);
if (stdout === '') {

15
apps/api/src/routes/api/v1/services/handlers.ts
View File

@ -2,7 +2,7 @@ import type { FastifyReply, FastifyRequest } from 'fastify';
import fs from 'fs/promises';
import yaml from 'js-yaml';
import bcrypt from 'bcryptjs';
import { prisma, uniqueName, asyncExecShell, getServiceImage, configureServiceType, getServiceFromDB, getContainerUsage, removeService, isDomainConfigured, saveUpdateableFields, fixType, decrypt, encrypt, getServiceMainPort, createDirectories, ComposeFile, makeLabelForServices, getFreePublicPort, getDomain, errorHandler, generatePassword, isDev, stopTcpHttpProxy, supportedServiceTypesAndVersions, executeDockerCmd, listSettings, getFreeExposedPort, checkDomainsIsValidInDNS, persistentVolumes, asyncSleep, isARM, defaultComposeConfiguration } from '../../../../lib/common';
import { prisma, uniqueName, asyncExecShell, getServiceImage, configureServiceType, getServiceFromDB, getContainerUsage, removeService, isDomainConfigured, saveUpdateableFields, fixType, decrypt, encrypt, getServiceMainPort, createDirectories, ComposeFile, makeLabelForServices, getFreePublicPort, getDomain, errorHandler, generatePassword, isDev, stopTcpHttpProxy, supportedServiceTypesAndVersions, executeDockerCmd, listSettings, getFreeExposedPort, checkDomainsIsValidInDNS, persistentVolumes, asyncSleep, isARM, defaultComposeConfiguration, checkExposedPort } from '../../../../lib/common';
import { day } from '../../../../lib/dayjs';
import { checkContainer, isContainerExited, removeContainer } from '../../../../lib/docker';
import cuid from 'cuid';
@ -378,18 +378,7 @@ export async function checkService(request: FastifyRequest<CheckService>) {
}
}
}
if (exposePort) {
if (exposePort < 1024 || exposePort > 65535) {
throw { status: 500, message: `Exposed Port needs to be between 1024 and 65535.` }
}
if (configuredPort !== exposePort) {
const availablePort = await getFreeExposedPort(id, exposePort, dockerId, remoteIpAddress);
if (availablePort.toString() !== exposePort.toString()) {
throw { status: 500, message: `Port ${exposePort} is already in use.` }
}
}
}
await checkExposedPort({ id, configuredPort, exposePort, dockerId, remoteIpAddress })
if (isDNSCheckEnabled && !isDev && !forceSave) {
let hostname = request.hostname.split(':')[0];
if (remoteEngine) hostname = remoteIpAddress;

1
apps/api/src/routes/webhooks/traefik/handlers.ts
View File

@ -484,7 +484,6 @@ export async function traefikOtherConfiguration(request: FastifyRequest<TraefikO
}
throw { status: 500 }
} catch ({ status, message }) {
console.log(status, message);
return errorHandler({ status, message })
}
}

2
apps/ui/src/lib/locales/en.json
View File

@ -159,7 +159,7 @@
"storage_saved": "Storage saved.",
"storage_updated": "Storage updated.",
"storage_deleted": "Storage deleted.",
"persistent_storage_explainer": "You can specify any folder that you want to be persistent across deployments. <br>This is useful for storing data such as a database (SQLite) or a cache."
"persistent_storage_explainer": "You can specify any folder that you want to be persistent across deployments.<br><span class='text-green-500 font-bold'>/example</span> means it will preserve <span class='text-green-500 font-bold'>/app/example</span> in the container as <span class='text-green-500 font-bold'>/app</span> is <span class='text-green-500 font-bold'>the root directory</span> for your application.<br><br>This is useful for storing data such as a <span class='text-green-500 font-bold'>database (SQLite)</span> or a <span class='text-green-500 font-bold'>cache</span>."
},
"deployment_queued": "Deployment queued.",
"confirm_to_delete": "Are you sure you would like to delete '{{name}}'?",

11
apps/ui/src/routes/applications/[id]/configuration/_PublicRepository.svelte
View File

@ -25,8 +25,10 @@
const gitUrl = publicRepositoryLink.replace('http://', '').replace('https://', '');
let [host, ...path] = gitUrl.split('/');
const [owner, repository, ...branch] = path;
ownerName = owner;
repositoryName = repository;
if (branch[0] === 'tree') {
branchName = branch[1];
await saveRepository();
@ -42,20 +44,20 @@
}
const apiUrl = `${protocol}://${host}`;
const repositoryDetails = await get(`${apiUrl}/repos/${owner}/${repository}`);
const repositoryDetails = await get(`${apiUrl}/repos/${ownerName}/${repositoryName}`);
projectId = repositoryDetails.id.toString();
let branches: any[] = [];
let page = 1;
let branchCount = 0;
loading.branches = true;
const loadedBranches = await loadBranchesByPage(apiUrl, owner, repository, page);
const loadedBranches = await loadBranchesByPage(apiUrl, ownerName, repositoryName, page);
branches = branches.concat(loadedBranches);
branchCount = branches.length;
if (branchCount === 100) {
while (branchCount === 100) {
page = page + 1;
const nextBranches = await loadBranchesByPage(apiUrl, owner, repository, page);
const nextBranches = await loadBranchesByPage(apiUrl, ownerName, repositoryName, page);
branches = branches.concat(nextBranches);
branchCount = nextBranches.length;
}
@ -68,7 +70,6 @@
}
async function loadBranchesByPage(apiUrl: string, owner: string, repository: string, page = 1) {
return await get(`${apiUrl}/repos/${owner}/${repository}/branches?per_page=100&page=${page}`);
// console.log(publicRepositoryLink);
}
async function saveRepository(event?: any) {
try {
@ -81,7 +82,7 @@
type
});
await post(`/applications/${id}/configuration/repository`, {
repository: `${repositoryName}/${branchName}`,
repository: `${ownerName}/${repositoryName}`,
branch: branchName,
projectId,
autodeploy: false,

35
apps/ui/src/routes/applications/[id]/configuration/buildpack.svelte
View File

@ -48,7 +48,7 @@
export let type: any;
export let application: any;
export let isPublicRepository: boolean;
console.log(isPublicRepository)
function checkPackageJSONContents({ key, json }: { key: any; json: any }) {
return json?.dependencies?.hasOwnProperty(key) || json?.devDependencies?.hasOwnProperty(key);
}
@ -237,7 +237,7 @@
if (error.message === 'Bad credentials') {
const { token } = await get(`/applications/${id}/configuration/githubToken`);
$appSession.tokens.github = token;
return await scanRepository()
return await scanRepository();
}
return errorNotification(error);
} finally {
@ -246,7 +246,12 @@
}
}
onMount(async () => {
await scanRepository();
if (!isPublicRepository) {
await scanRepository();
} else {
foundConfig = findBuildPack('node', packageManager);
scanning = false;
}
});
</script>
@ -263,27 +268,25 @@
</div>
</div>
{:else}
<div class="max-w-7xl mx-auto ">
<div class="title pb-2">Coolify Buildpacks</div>
<div class="flex flex-wrap justify-center">
{#each buildPacks.filter(bp => bp.isCoolifyBuildPack === true) as buildPack}
<div class="p-2">
<BuildPack {packageManager} {buildPack} {scanning} bind:foundConfig />
</div>
{/each}
{#each buildPacks.filter((bp) => bp.isCoolifyBuildPack === true) as buildPack}
<div class="p-2">
<BuildPack {packageManager} {buildPack} {scanning} bind:foundConfig />
</div>
{/each}
</div>
</div>
<div class="max-w-7xl mx-auto ">
<div class="title pb-2">Heroku</div>
<div class="flex flex-wrap justify-center">
{#each buildPacks.filter(bp => bp.isHerokuBuildPack === true) as buildPack}
<div class="p-2">
<BuildPack {packageManager} {buildPack} {scanning} bind:foundConfig />
</div>
{/each}
</div>
{#each buildPacks.filter((bp) => bp.isHerokuBuildPack === true) as buildPack}
<div class="p-2">
<BuildPack {packageManager} {buildPack} {scanning} bind:foundConfig />
</div>
{/each}
</div>
</div>
{/if}

36
apps/ui/src/routes/applications/[id]/index.svelte
View File

@ -133,6 +133,7 @@
autodeploy = !autodeploy;
}
if (name === 'isBot') {
if ($status.application.isRunning) return;
isBot = !isBot;
application.settings.isBot = isBot;
setLocation(application, settings);
@ -345,8 +346,11 @@
<label for="gitSource" class="text-base font-bold text-stone-100"
>{$t('application.git_source')}</label
>
{#if isDisabled}
<input disabled={isDisabled} value={application.gitSource.name} />
{#if isDisabled || application.settings.isPublicRepository}
<input
disabled={isDisabled || application.settings.isPublicRepository}
value={application.gitSource.name}
/>
{:else}
<a
href={`/applications/${id}/configuration/source?from=/applications/${id}`}
@ -363,8 +367,11 @@
<label for="repository" class="text-base font-bold text-stone-100"
>{$t('application.git_repository')}</label
>
{#if isDisabled}
<input disabled={isDisabled} value="{application.repository}/{application.branch}" />
{#if isDisabled || application.settings.isPublicRepository}
<input
disabled={isDisabled || application.settings.isPublicRepository}
value="{application.repository}/{application.branch}"
/>
{:else}
<a
href={`/applications/${id}/configuration/repository?from=/applications/${id}&to=/applications/${id}/configuration/buildpack`}
@ -488,6 +495,7 @@
on:click={() => changeSettings('isBot')}
title="Is your application a bot?"
description="You can deploy applications without domains. <br>They will listen on <span class='text-green-500 font-bold'>IP:EXPOSEDPORT</span> instead.<br></Setting><br>Useful to host <span class='text-green-500 font-bold'>Twitch bots.</span>"
disabled={$status.application.isRunning}
/>
</div>
{#if !isBot}
@ -770,15 +778,17 @@
<div class="title">{$t('application.features')}</div>
</div>
<div class="px-10 pb-10">
<div class="grid grid-cols-2 items-center">
<Setting
isCenter={false}
bind:setting={autodeploy}
on:click={() => changeSettings('autodeploy')}
title={$t('application.enable_automatic_deployment')}
description={$t('application.enable_auto_deploy_webhooks')}
/>
</div>
{#if !application.settings.isPublicRepository}
<div class="grid grid-cols-2 items-center">
<Setting
isCenter={false}
bind:setting={autodeploy}
on:click={() => changeSettings('autodeploy')}
title={$t('application.enable_automatic_deployment')}
description={$t('application.enable_auto_deploy_webhooks')}
/>
</div>
{/if}
{#if !application.settings.isBot}
<div class="grid grid-cols-2 items-center">
<Setting

7
apps/ui/src/routes/applications/[id]/storages.svelte
View File

@ -87,9 +87,7 @@
</div>
<div class="mx-auto max-w-6xl rounded-xl px-6 pt-4">
<div class="flex justify-center py-4 text-center">
<Explainer customClass="w-full" text={$t('application.storage.persistent_storage_explainer')} />
</div>
<table class="mx-auto border-separate text-left">
<thead>
<tr class="h-12">
@ -109,4 +107,7 @@
</tr>
</tbody>
</table>
<div class="flex justify-center py-4 text-center">
<Explainer customClass="w-full" text={$t('application.storage.persistent_storage_explainer')} />
</div>
</div>