Shiloh/lasthourcloud
Shiloh/lasthourcloud
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 4 Wiki Activity

Merge branch 'v4' into ijpatricio-wip-9

Browse Source
This commit is contained in:
Andras Bacsai 2023-04-12 13:18:33 +02:00
commit 01ec1ed563
11 changed files with 155 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
app/Actions/RemoteProcess/RunRemoteProcess.php
View File

@ -67,6 +67,9 @@ public function __invoke(): ProcessResult
$this->activity->save();
if ($processResult->exitCode() != 0 && $processResult->errorOutput()) {
throw new \RuntimeException('Remote command failed');
}
return $processResult;
}

103
app/Jobs/DeployApplicationJob.php
View File

@ -14,6 +14,7 @@
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Http;
use Lcobucci\JWT\Encoding\ChainedFormatter;
use Lcobucci\JWT\Encoding\JoseEncoder;
@ -49,7 +50,7 @@ public function __construct(
$server = $this->destination->server;
$private_key_location = savePrivateKey($server);
$private_key_location = savePrivateKeyForServer($server);
$remoteProcessArgs = new RemoteProcessArgs(
server_ip: $server->ip,
@ -98,8 +99,14 @@ public function handle(): void
// Import git repository
$this->executeNow([
"echo -n 'Importing {$this->application->git_repository}:{$this->application->git_branch} to {$this->workdir}... '",
$this->gitImport(),
"echo -n 'Importing {$this->application->git_repository}:{$this->application->git_branch} to {$this->workdir}... '"
]);
$this->executeNow([
...$this->gitImport(),
], 'importing_git_repository');
$this->executeNow([
"echo 'Done.'"
]);
@ -135,10 +142,8 @@ public function handle(): void
]);
$this->executeNow([
"echo -n 'Starting new container... '",
$this->execute_in_builder("docker compose --project-directory {$this->workdir} up -d >/dev/null 2>&1"),
$this->execute_in_builder("docker compose --project-directory {$this->workdir} up -d >/dev/null"),
"echo 'Done. 🎉'",
], setStatus: true);
$this->executeNow([
"docker stop -t 0 {$this->deployment_uuid} >/dev/null"
], setStatus: true);
}
@ -150,7 +155,8 @@ private function execute_in_builder(string $command)
private function generate_docker_compose()
{
$persistentStorages = $this->generate_local_persistent_volumes();
$volume_names = $this->generate_local_persistent_volumes_only_volume_names();
$docker_compose = [
'version' => '3.8',
'services' => [
@ -158,6 +164,9 @@ private function generate_docker_compose()
'image' => "{$this->application->uuid}:$this->git_commit",
'container_name' => $this->application->uuid,
'restart' => 'always',
'environment' => [
'PORT' => $this->application->ports_exposes[0]
],
'labels' => $this->set_labels_for_applications(),
'expose' => $this->application->ports_exposes,
'networks' => [
@ -186,15 +195,36 @@ private function generate_docker_compose()
if (count($this->application->ports_mappings) > 0) {
$docker_compose['services'][$this->application->uuid]['ports'] = $this->application->ports_mappings;
}
// if (count($volumes) > 0) {
// $docker_compose['services'][$this->application->uuid]['volumes'] = $volumes;
// }
// if (count($volume_names) > 0) {
// $docker_compose['volumes'] = $volume_names;
// }
if (count($persistentStorages) > 0) {
$docker_compose['services'][$this->application->uuid]['volumes'] = $persistentStorages;
}
if (count($volume_names) > 0) {
$docker_compose['volumes'] = $volume_names;
}
return Yaml::dump($docker_compose);
}
private function generate_local_persistent_volumes()
{
foreach ($this->application->persistentStorages as $persistentStorage) {
$volume_name = $persistentStorage->host_path ?? $persistentStorage->name;
$local_persistent_volumes[] = $volume_name . ':' . $persistentStorage->mount_path;
}
return $local_persistent_volumes ?? [];
}
private function generate_local_persistent_volumes_only_volume_names()
{
foreach ($this->application->persistentStorages as $persistentStorage) {
if ($persistentStorage->host_path) {
continue;
}
$local_persistent_volumes_names[$persistentStorage->name] = [
'name' => $persistentStorage->name,
'external' => false,
];
}
return $local_persistent_volumes_names ?? [];
}
private function generate_healthcheck_commands()
{
if (!$this->application->health_check_port) {
@ -202,18 +232,12 @@ private function generate_healthcheck_commands()
}
if ($this->application->health_check_path) {
$generated_healthchecks_commands = [
"curl -X {$this->application->health_check_method} -f {$this->application->health_check_scheme}://{$this->application->health_check_host}:{$this->application->health_check_port}{$this->application->health_check_path}"
"curl -s -X {$this->application->health_check_method} -f {$this->application->health_check_scheme}://{$this->application->health_check_host}:{$this->application->health_check_port}{$this->application->health_check_path} > /dev/null"
];
} else {
$generated_healthchecks_commands = [];
foreach ($this->application->ports_exposes as $key => $port) {
$generated_healthchecks_commands = [
"curl -X {$this->application->health_check_method} -f {$this->application->health_check_scheme}://{$this->application->health_check_host}:{$port}/"
];
if (count($this->application->ports_exposes) != $key + 1) {
$generated_healthchecks_commands[] = '&&';
}
}
$generated_healthchecks_commands = [
"curl -s -X {$this->application->health_check_method} -f {$this->application->health_check_scheme}://{$this->application->health_check_host}:{$this->application->health_check_port}/"
];
}
return implode(' ', $generated_healthchecks_commands);
}
@ -255,10 +279,14 @@ private function set_labels_for_applications()
return $labels;
}
private function executeNow(array $command, string $propertyName = null, bool $hideFromOutput = false, $setStatus = false)
private function executeNow(array|Collection $command, string $propertyName = null, bool $hideFromOutput = false, $setStatus = false)
{
static::$batch_counter++;
$commandText = collect($command)->implode("\n");
if ($command instanceof Collection) {
$commandText = $command->implode("\n");
} else {
$commandText = collect($command)->implode("\n");
}
$this->activity->properties = $this->activity->properties->merge([
'command' => $commandText,
@ -270,15 +298,13 @@ private function executeNow(array $command, string $propertyName = null, bool $h
'hideFromOutput' => $hideFromOutput,
'setStatus' => $setStatus,
]);
$result = $remoteProcess();
if ($propertyName) {
$result = $remoteProcess();
$this->activity->properties = $this->activity->properties->merge([
$propertyName => trim($result->output()),
]);
$this->activity->save();
} else {
$remoteProcess();
}
}
private function gitImport()
@ -287,12 +313,27 @@ private function gitImport()
$url = parse_url(filter_var($source_html_url, FILTER_SANITIZE_URL));
$source_html_url_host = $url['host'];
$source_html_url_scheme = $url['scheme'];
if ($this->application->source->getMorphClass() == 'App\Models\GithubApp') {
if ($this->source->is_public) {
return $this->execute_in_builder("git clone -q -b {$this->application->git_branch} {$this->source->html_url}/{$this->application->git_repository}.git {$this->workdir}");
return [
$this->execute_in_builder("git clone -q -b {$this->application->git_branch} {$this->source->html_url}/{$this->application->git_repository}.git {$this->workdir}")
];
} else {
$github_access_token = $this->generate_jwt_token_for_github();
return $this->execute_in_builder("git clone -q -b {$this->application->git_branch} $source_html_url_scheme://x-access-token:$github_access_token@$source_html_url_host/{$this->application->git_repository}.git {$this->workdir}");
if (!$this->application->source->app_id) {
$private_key = base64_encode($this->application->source->privateKey->private_key);
return [
$this->execute_in_builder("mkdir -p /root/.ssh"),
$this->execute_in_builder("echo '{$private_key}' | base64 -d > /root/.ssh/id_rsa"),
$this->execute_in_builder("chmod 600 /root/.ssh/id_rsa"),
$this->execute_in_builder("GIT_SSH_COMMAND=\"ssh -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa\" git clone -q -b {$this->application->git_branch} git@$source_html_url_host:{$this->application->git_repository}.git {$this->workdir}")
];
} else {
$github_access_token = $this->generate_jwt_token_for_github();
return [
$this->execute_in_builder("git clone -q -b {$this->application->git_branch} $source_html_url_scheme://x-access-token:$github_access_token@$source_html_url_host/{$this->application->git_repository}.git {$this->workdir}")
];
}
}
}
}

11
bootstrap/helpers.php
View File

@ -29,7 +29,7 @@ function remoteProcess(
// @TODO: Check if the user has access to this server
// checkTeam($server->team_id);
$private_key_location = savePrivateKey($server);
$private_key_location = savePrivateKeyForServer($server);
return resolve(DispatchRemoteProcess::class, [
'remoteProcessArgs' => new RemoteProcessArgs(
@ -56,8 +56,8 @@ function remoteProcess(
// }
// }
if (!function_exists('savePrivateKey')) {
function savePrivateKey(Server $server)
if (!function_exists('savePrivateKeyForServer')) {
function savePrivateKeyForServer(Server $server)
{
$temp_file = 'id.rsa_' . 'root' . '@' . $server->ip;
Storage::disk('local')->put($temp_file, $server->privateKey->private_key, 'private');
@ -118,9 +118,10 @@ function formatDockerLabelsToJson($rawOutput): Collection
}
}
if (!function_exists('runRemoteCommandSync')) {
function runRemoteCommandSync($server, array $command) {
function runRemoteCommandSync($server, array $command)
{
$command_string = implode("\n", $command);
$private_key_location = savePrivateKey($server);
$private_key_location = savePrivateKeyForServer($server);
$ssh_command = generateSshCommand($private_key_location, $server->ip, $server->user, $server->port, $command_string);
$process = Process::run($ssh_command);
$output = trim($process->output());

2
database/migrations/2023_03_27_081716_create_applications_table.php
View File

@ -40,7 +40,7 @@ public function up(): void
$table->string('base_directory')->default('/');
$table->string('publish_directory')->nullable();
$table->string('health_check_path')->nullable();
$table->string('health_check_path')->default('/');
$table->string('health_check_port')->nullable();
$table->string('health_check_host')->default('localhost');
$table->string('health_check_method')->default('GET');

5
database/migrations/2023_03_28_083723_create_github_apps_table.php
View File

@ -21,8 +21,6 @@ public function up(): void
$table->string('html_url');
$table->integer('custom_port')->default(22);
$table->string('custom_user')->default('git');
$table->boolean('is_system_wide')->default(false);
$table->boolean('is_public')->default(false);
$table->integer('app_id')->nullable();
$table->integer('installation_id')->nullable();
@ -30,6 +28,9 @@ public function up(): void
$table->longText('client_secret')->nullable();
$table->longText('webhook_secret')->nullable();
$table->boolean('is_system_wide')->default(false);
$table->boolean('is_public')->default(false);
$table->foreignId('private_key_id')->nullable();
$table->foreignId('team_id');
$table->timestamps();

46
database/seeders/ApplicationSeeder.php
View File

@ -22,12 +22,12 @@ public function run(): void
$standalone_docker_1 = StandaloneDocker::find(1);
$swarm_docker_1 = SwarmDocker::find(1);
$github_public_source = GithubApp::find(1);
$github_private_source = GithubApp::find(2);
$github_public_source = GithubApp::where('name', 'Public GitHub')->first();
$github_private_source = GithubApp::where('name', 'coolify-laravel-development-private-github')->first();
$github_private_source_with_deploy_key = GithubApp::where('name', 'Private GitHub (deployment key)')->first();
$pv_storage = LocalPersistentVolume::find(1);
Application::create([
'id' => 1,
'name' => 'Public application (from GitHub)',
'git_repository' => 'coollabsio/coolify-examples',
'git_branch' => 'nodejs-fastify',
@ -40,19 +40,31 @@ public function run(): void
'source_id' => $github_public_source->id,
'source_type' => GithubApp::class,
]);
Application::create([
'id' => 2,
'name' => 'Private application (through GitHub App)',
'git_repository' => 'coollabsio/nodejs-example',
'git_branch' => 'main',
'build_pack' => 'nixpacks',
'ports_exposes' => '3000',
'ports_mappings' => '3001:3000',
'environment_id' => $environment_1->id,
'destination_id' => $standalone_docker_1->id,
'destination_type' => StandaloneDocker::class,
'source_id' => $github_private_source->id,
'source_type' => GithubApp::class,
]);
// Application::create([
// 'name' => 'Private application (through GitHub App)',
// 'git_repository' => 'coollabsio/nodejs-example',
// 'git_branch' => 'main',
// 'build_pack' => 'nixpacks',
// 'ports_exposes' => '3000',
// 'ports_mappings' => '3001:3000',
// 'environment_id' => $environment_1->id,
// 'destination_id' => $standalone_docker_1->id,
// 'destination_type' => StandaloneDocker::class,
// 'source_id' => $github_private_source->id,
// 'source_type' => GithubApp::class,
// ]);
// Application::create([
// 'name' => 'Public application (from GitHub through Deploy Key)',
// 'git_repository' => 'coollabsio/php',
// 'git_branch' => 'main',
// 'build_pack' => 'nixpacks',
// 'ports_exposes' => '80,3000',
// 'ports_mappings' => '3002:80',
// 'environment_id' => $environment_1->id,
// 'destination_id' => $standalone_docker_1->id,
// 'destination_type' => StandaloneDocker::class,
// 'source_id' => $github_private_source_with_deploy_key->id,
// 'source_type' => GithubApp::class,
// ]);
}
}

2
database/seeders/DatabaseSeeder.php
View File

@ -18,7 +18,6 @@ public function run(): void
ProjectSeeder::class,
ProjectSettingSeeder::class,
EnvironmentSeeder::class,
LocalPersistentVolumeSeeder::class,
StandaloneDockerSeeder::class,
SwarmDockerSeeder::class,
KubernetesSeeder::class,
@ -28,6 +27,7 @@ public function run(): void
ApplicationSettingsSeeder::class,
DBSeeder::class,
ServiceSeeder::class,
LocalPersistentVolumeSeeder::class,
]);
}
}

11
database/seeders/GithubAppSeeder.php
View File

@ -16,9 +16,9 @@ class GithubAppSeeder extends Seeder
public function run(): void
{
$root_team = Team::find(1);
$private_key_1 = PrivateKey::find(1);
$private_key_2 = PrivateKey::find(2);
GithubApp::create([
'id' => 1,
'name' => 'Public GitHub',
'api_url' => 'https://api.github.com',
'html_url' => 'https://github.com',
@ -26,7 +26,6 @@ public function run(): void
'team_id' => $root_team->id,
]);
GithubApp::create([
'id' => 2,
'name' => 'coolify-laravel-development-private-github',
'api_url' => 'https://api.github.com',
'html_url' => 'https://github.com',
@ -39,5 +38,13 @@ public function run(): void
'private_key_id' => $private_key_2->id,
'team_id' => $root_team->id,
]);
GithubApp::create([
'name' => 'Private GitHub (deployment key)',
'api_url' => 'https://api.github.com',
'html_url' => 'https://github.com',
'is_public' => false,
'private_key_id' => $private_key_1->id,
'team_id' => $root_team->id,
]);
}
}

3
database/seeders/LocalPersistentVolumeSeeder.php
View File

@ -13,10 +13,11 @@ class LocalPersistentVolumeSeeder extends Seeder
*/
public function run(): void
{
$application = Application::where('name', 'Public application (from GitHub)')->first();
LocalPersistentVolume::create([
'name' => 'test-pv',
'mount_path' => '/data',
'resource_id' => 1,
'resource_id' => $application->id,
'resource_type' => Application::class,
]);
}

51
run
View File

@ -1,51 +0,0 @@
#!/usr/bin/env bash
# Inspired on https://github.com/adriancooney/Taskfile
#
# Install an alias, to be able to simply execute `run`
# echo 'alias run=./run' >> ~/.aliases
#
# Define Docker Compose command prefix...
set -e
docker compose &> /dev/null
if [ $? == 0 ]; then
DOCKER_COMPOSE="docker compose"
else
DOCKER_COMPOSE="docker-compose"
fi
SAIL=./vendor/bin/sail
export WWWUSER=${WWWUSER:-$UID}
export WWWGROUP=${WWWGROUP:-$(id -g)}
function help {
echo "$0 <task> <args>"
echo "Tasks:"
compgen -A function | cat -n
}
function default {
help
}
function wait_db {
TRIES=0
MAX_TRIES=15
WAIT=4
until $DOCKER_COMPOSE exec postgres bash -c "psql -U coolify -d coolify -t -q -c \"SELECT datname FROM pg_database;\" " | grep coolify
do
((TRIES++))
if [ $TRIES -gt $MAX_TRIES ]; then
echo "Database is not ready after $MAX_TRIES tries. Exiting."
exit 1
fi
echo "Database is not ready yet. Attempt $TRIES/$MAX_TRIES. Waiting $WAIT seconds before next try..."
sleep $WAIT
done
}
TIMEFORMAT="Task completed in %3lR"
time "${@:-default}"

46
scripts/run
View File

@ -1,11 +1,25 @@
#!/usr/bin/env bash
# Inspired on https://github.com/adriancooney/Taskfile
# Install an alias, to be able to simply execute `run` => echo 'alias run=./run' >> ~/.aliases
#
# Install an alias, to be able to simply execute `run`
# echo 'alias run=./run' >> ~/.aliases
#
# Define Docker Compose command prefix...
set -e
docker compose &> /dev/null
if [ $? == 0 ]; then
DOCKER_COMPOSE="docker compose"
else
DOCKER_COMPOSE="docker-compose"
fi
SAIL=./vendor/bin/sail
export WWWUSER=${WWWUSER:-$UID}
export WWWGROUP=${WWWGROUP:-$(id -g)}
function help {
echo "$0 <task> <args>"
echo "Tasks:"
@ -16,23 +30,21 @@ function default {
help
}
function bash {
docker-compose exec -u $(id -u) php bash
}
function wait_db {
TRIES=0
MAX_TRIES=15
WAIT=4
# The user with native SSH capability
function coolify-bash {
docker-compose exec -u coolify php bash
}
function root-bash {
docker-compose exec php bash
}
# Usage: ./Taskfile envFile:set FOOBAR abc
# This will set the FOOBAR variable to "abc" in the .env file
function envFile:set {
sed -i "s#^$1=.*#$1=$2#g" .env
until $DOCKER_COMPOSE exec postgres bash -c "psql -U coolify -d coolify -t -q -c \"SELECT datname FROM pg_database;\" " | grep coolify
do
((TRIES++))
if [ $TRIES -gt $MAX_TRIES ]; then
echo "Database is not ready after $MAX_TRIES tries. Exiting."
exit 1
fi
echo "Database is not ready yet. Attempt $TRIES/$MAX_TRIES. Waiting $WAIT seconds before next try..."
sleep $WAIT
done
}
TIMEFORMAT="Task completed in %3lR"