From 3b191fa73e193ece6217389f0a4f5b21810d3af2 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 30 Mar 2023 11:09:39 +0200 Subject: [PATCH 1/2] fix: settings for apps and projects add: coolify version config fix: private key for private git based apps --- app/Models/Application.php | 9 +++++++ app/Models/ApplicationSetting.php | 4 ++++ app/Models/GithubApp.php | 4 ++++ app/Models/GitlabApp.php | 4 ++++ app/Models/Project.php | 8 +++++++ app/Providers/AppServiceProvider.php | 10 ++++---- config/coolify.php | 5 ++++ ...03_27_081716_create_applications_table.php | 1 - database/seeders/ApplicationSeeder.php | 24 ++++++++++++------- .../seeders/ApplicationSettingsSeeder.php | 10 ++++---- database/seeders/ProjectSettingSeeder.php | 8 ++----- 11 files changed, 62 insertions(+), 25 deletions(-) create mode 100644 config/coolify.php diff --git a/app/Models/Application.php b/app/Models/Application.php index c0fa88067..a139bc6a8 100644 --- a/app/Models/Application.php +++ b/app/Models/Application.php @@ -7,6 +7,15 @@ use Spatie\Activitylog\Models\Activity; class Application extends BaseModel { + protected static function booted() + { + static::created(function ($application) { + ApplicationSetting::create([ + 'application_id' => $application->id, + ]); + }); + } + public function environment() { return $this->belongsTo(Environment::class); diff --git a/app/Models/ApplicationSetting.php b/app/Models/ApplicationSetting.php index d5f9b6f4e..9acad2ae9 100644 --- a/app/Models/ApplicationSetting.php +++ b/app/Models/ApplicationSetting.php @@ -6,4 +6,8 @@ use Illuminate\Database\Eloquent\Model; class ApplicationSetting extends Model { + public function application() + { + return $this->belongsTo(Application::class); + } } diff --git a/app/Models/GithubApp.php b/app/Models/GithubApp.php index 4d461fdc4..fd7dd0775 100644 --- a/app/Models/GithubApp.php +++ b/app/Models/GithubApp.php @@ -8,4 +8,8 @@ class GithubApp extends BaseModel { return $this->morphMany(Application::class, 'source'); } + public function privateKey() + { + return $this->belongsTo(PrivateKey::class); + } } diff --git a/app/Models/GitlabApp.php b/app/Models/GitlabApp.php index 65024b3f7..ad5f658e2 100644 --- a/app/Models/GitlabApp.php +++ b/app/Models/GitlabApp.php @@ -4,4 +4,8 @@ namespace App\Models; class GitlabApp extends BaseModel { + public function privateKey() + { + return $this->belongsTo(PrivateKey::class); + } } diff --git a/app/Models/Project.php b/app/Models/Project.php index 1efe88548..1b0a71049 100644 --- a/app/Models/Project.php +++ b/app/Models/Project.php @@ -4,6 +4,14 @@ namespace App\Models; class Project extends BaseModel { + protected static function booted() + { + static::created(function ($project) { + ProjectSetting::create([ + 'project_id' => $project->id, + ]); + }); + } public function environments() { return $this->hasMany(Environment::class); } diff --git a/app/Providers/AppServiceProvider.php b/app/Providers/AppServiceProvider.php index 3487238e9..984d714cd 100644 --- a/app/Providers/AppServiceProvider.php +++ b/app/Providers/AppServiceProvider.php @@ -21,9 +21,11 @@ class AppServiceProvider extends ServiceProvider */ public function boot(): void { - // @TODO: Should remove builder container here - // Queue::after(function (JobProcessed $event) { - // dd($event->job->resolveName()); - // }); + Queue::after(function (JobProcessed $event) { + // @TODO: Remove `coolify-builder` container after the remoteProcess job is finishged and remoteProcess->type == `deployment`. + if ($event->job->resolveName() === 'App\Jobs\ExecuteRemoteProcess') { + + } + }); } } diff --git a/config/coolify.php b/config/coolify.php new file mode 100644 index 000000000..4c6af61d4 --- /dev/null +++ b/config/coolify.php @@ -0,0 +1,5 @@ + '4.0.0-rc.1', +]; diff --git a/database/migrations/2023_03_27_081716_create_applications_table.php b/database/migrations/2023_03_27_081716_create_applications_table.php index 57515e4fc..701a70c17 100644 --- a/database/migrations/2023_03_27_081716_create_applications_table.php +++ b/database/migrations/2023_03_27_081716_create_applications_table.php @@ -58,7 +58,6 @@ return new class extends Migration $table->morphs('source'); $table->foreignId('environment_id'); - $table->timestamps(); }); } diff --git a/database/seeders/ApplicationSeeder.php b/database/seeders/ApplicationSeeder.php index 91b954834..ccfa74045 100644 --- a/database/seeders/ApplicationSeeder.php +++ b/database/seeders/ApplicationSeeder.php @@ -22,6 +22,7 @@ class ApplicationSeeder extends Seeder $swarm_docker_1 = SwarmDocker::find(1); $github_public_source = GithubApp::find(1); + $github_private_source = GithubApp::find(2); Application::create([ 'id' => 1, 'name' => 'My first application', @@ -36,14 +37,19 @@ class ApplicationSeeder extends Seeder 'source_id' => $github_public_source->id, 'source_type' => GithubApp::class, ]); - // Application::create([ - // 'id' => 2, - // 'name' => 'My second application (Swarm)', - // 'environment_id' => $environment_1->id, - // 'destination_id' => $swarm_docker_1->id, - // 'destination_type' => SwarmDocker::class, - // 'source_id' => $github_public_source->id, - // 'source_type' => GithubApp::class, - // ]); + Application::create([ + 'id' => 2, + 'name' => 'My second application', + 'git_repository' => 'coollabsio/nodejs-example', + 'git_branch' => 'main', + 'build_pack' => 'nixpacks', + 'ports_exposes' => '3000', + 'ports_mappings' => '3001:3000', + 'environment_id' => $environment_1->id, + 'destination_id' => $standalone_docker_1->id, + 'destination_type' => StandaloneDocker::class, + 'source_id' => $github_private_source->id, + 'source_type' => GithubApp::class, + ]); } } diff --git a/database/seeders/ApplicationSettingsSeeder.php b/database/seeders/ApplicationSettingsSeeder.php index 760306b07..fceb32954 100644 --- a/database/seeders/ApplicationSettingsSeeder.php +++ b/database/seeders/ApplicationSettingsSeeder.php @@ -17,10 +17,10 @@ class ApplicationSettingsSeeder extends Seeder */ public function run(): void { - $application_1 = Application::find(1); - ApplicationSetting::create([ - 'id' => 1, - 'application_id' => $application_1->id, - ]); + // $application_1 = Application::find(1); + // ApplicationSetting::create([ + // 'id' => 1, + // 'application_id' => $application_1->id, + // ]); } } diff --git a/database/seeders/ProjectSettingSeeder.php b/database/seeders/ProjectSettingSeeder.php index 050982bee..8a2eddcc1 100644 --- a/database/seeders/ProjectSettingSeeder.php +++ b/database/seeders/ProjectSettingSeeder.php @@ -3,7 +3,6 @@ namespace Database\Seeders; use App\Models\Project; -use App\Models\ProjectSetting; use Illuminate\Database\Seeder; class ProjectSettingSeeder extends Seeder @@ -11,10 +10,7 @@ class ProjectSettingSeeder extends Seeder public function run(): void { $first_project = Project::find(1); - ProjectSetting::create([ - 'id' => 1, - 'wildcard_domain' => 'testing-host.localhost', - 'project_id' => $first_project->id, - ]); + $first_project->settings->wildcard_domain = 'wildcard.example.com'; + $first_project->settings->save(); } } From b9ffe3b204100d17390908c59fcc7d8d35ae9a88 Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Thu, 30 Mar 2023 11:10:31 +0200 Subject: [PATCH 2/2] feat: private github deployments feat: expose, labels for compose feat: use application debug settings for debug logs --- app/Http/Livewire/DeployApplication.php | 111 ++++++++++++++++++--- composer.json | 1 + composer.lock | 125 +++++++++++++++++++++++- 3 files changed, 224 insertions(+), 13 deletions(-) diff --git a/app/Http/Livewire/DeployApplication.php b/app/Http/Livewire/DeployApplication.php index ff11c0e25..461fe0765 100644 --- a/app/Http/Livewire/DeployApplication.php +++ b/app/Http/Livewire/DeployApplication.php @@ -4,22 +4,35 @@ namespace App\Http\Livewire; use App\Models\Application; use App\Models\CoolifyInstanceSettings; +use DateTimeImmutable; +use Illuminate\Support\Facades\Http; use Livewire\Component; use Symfony\Component\Yaml\Yaml; use Visus\Cuid2\Cuid2; +use Lcobucci\JWT\Encoding\ChainedFormatter; +use Lcobucci\JWT\Encoding\JoseEncoder; +use Lcobucci\JWT\Signer\Key\InMemory; +use Lcobucci\JWT\Signer\Rsa\Sha256; +use Lcobucci\JWT\Token\Builder; class DeployApplication extends Component { public string $application_uuid; public $activity; + protected string $deployment_uuid; protected array $command = []; protected Application $application; protected $destination; + protected $source; private function execute_in_builder(string $command) { - return $this->command[] = "docker exec {$this->deployment_uuid} bash -c '{$command}'"; + if ($this->application->settings->is_debug) { + return $this->command[] = "docker exec {$this->deployment_uuid} bash -c '{$command}'"; + } else { + return $this->command[] = "docker exec {$this->deployment_uuid} bash -c '{$command}' > /dev/null 2>&1"; + } } private function start_builder_container() { @@ -27,7 +40,7 @@ class DeployApplication extends Component } private function generate_docker_compose() { - return Yaml::dump([ + $docker_compose = [ 'version' => '3.8', 'services' => [ $this->application->uuid => [ @@ -35,6 +48,8 @@ class DeployApplication extends Component 'expose' => $this->application->ports_exposes, 'container_name' => $this->application->uuid, 'restart' => 'always', + 'labels' => $this->set_labels_for_applications(), + 'expose' => $this->application->ports_exposes, 'networks' => [ $this->destination->network, ], @@ -57,7 +72,30 @@ class DeployApplication extends Component 'attachable' => true, ] ] - ]); + ]; + if (count($this->application->ports_mappings) > 0) { + $docker_compose['services'][$this->application->uuid]['ports'] = $this->application->ports_mappings; + } + // if (count($volumes) > 0) { + // $docker_compose['services'][$this->application->uuid]['volumes'] = $volumes; + // } + // if (count($volume_names) > 0) { + // $docker_compose['volumes'] = $volume_names; + // } + return Yaml::dump($docker_compose); + } + private function set_labels_for_applications() + { + $labels = []; + $labels[] = 'coolify.managed=true'; + $labels[] = 'coolify.version=' . config('coolify.version'); + $labels[] = 'coolify.applicationId=' . $this->application->id; + $labels[] = 'coolify.type=application'; + $labels[] = 'coolify.name=' . $this->application->name; + if ($this->application->fqdn) { + $labels[] = "traefik.http.routers.container.rule=Host(`{$this->application->fqdn}`)"; + } + return $labels; } private function generate_healthcheck_commands() { @@ -81,12 +119,40 @@ class DeployApplication extends Component } return implode(' ', $generated_healthchecks_commands); } + private function generate_jwt_token_for_github() + { + $signingKey = InMemory::plainText($this->source->privateKey->private_key); + $algorithm = new Sha256(); + $tokenBuilder = (new Builder(new JoseEncoder(), ChainedFormatter::default())); + $now = new DateTimeImmutable(); + $now = $now->setTime($now->format('H'), $now->format('i')); + $issuedToken = $tokenBuilder + ->issuedBy($this->source->app_id) + ->issuedAt($now) + ->expiresAt($now->modify('+10 minutes')) + ->getToken($algorithm, $signingKey) + ->toString(); + $token = Http::withHeaders([ + 'Authorization' => "Bearer $issuedToken", + 'Accept' => 'application/vnd.github.machine-man-preview+json' + ])->post("{$this->source->api_url}/app/installations/{$this->source->installation_id}/access_tokens"); + if ($token->failed()) { + throw new \Exception("Failed to get access token for $this->application_name from " . $this->source_name . " with error: " . $token->json()['message']); + } + return $token->json()['token']; + } public function deploy() { $coolify_instance_settings = CoolifyInstanceSettings::find(1); $this->application = Application::where('uuid', $this->application_uuid)->first(); $this->destination = $this->application->destination->getMorphClass()::where('id', $this->application->destination->id)->first(); - $source = $this->application->source->getMorphClass()::where('id', $this->application->source->id)->first(); + $this->source = $this->application->source->getMorphClass()::where('id', $this->application->source->id)->first(); + + $source_html_url = data_get($this->application, 'source.html_url'); + $url = parse_url(filter_var($source_html_url, FILTER_SANITIZE_URL)); + $source_html_url_host = $url['host']; + $source_html_url_scheme = $url['scheme']; + // Get Wildcard Domain $project_wildcard_domain = data_get($this->application, 'environment.project.settings.wildcard_domain'); $global_wildcard_domain = data_get($coolify_instance_settings, 'wildcard_domain'); @@ -103,20 +169,34 @@ class DeployApplication extends Component $workdir = "/artifacts/{$this->deployment_uuid}"; // Start build process - $docker_compose_base64 = base64_encode($this->generate_docker_compose($this->application)); - $this->command[] = "echo 'Starting deployment of {$this->application->name} ({$this->application->uuid})'"; + $this->command[] = "echo 'Starting deployment of {$this->application->git_repository}:{$this->application->git_branch}...'"; + $this->command[] = "echo -n 'Pulling latest version of the builder image (ghcr.io/coollabsio/coolify-builder)... '"; $this->start_builder_container(); - $this->execute_in_builder("git clone -b {$this->application->git_branch} {$source->html_url}/{$this->application->git_repository}.git {$workdir}"); - + $this->command[] = "echo 'Done.'"; + $this->command[] = "echo -n 'Importing {$this->application->git_repository}:{$this->application->git_branch} to {$workdir}... '"; + if ($this->application->source->getMorphClass() == 'App\Models\GithubApp') { + if ($this->source->is_public) { + $this->execute_in_builder("git clone -q -b {$this->application->git_branch} {$this->source->html_url}/{$this->application->git_repository}.git {$workdir}"); + } else { + $github_access_token = $this->generate_jwt_token_for_github(); + $this->execute_in_builder("git clone -q -b {$this->application->git_branch} $source_html_url_scheme://x-access-token:$github_access_token@$source_html_url_host/{$this->application->git_repository}.git {$workdir}"); + } + } + $this->command[] = "echo 'Done.'"; // Export git commit to a file + $this->command[] = "echo -n 'Checking commit sha... '"; $this->execute_in_builder("cd {$workdir} && git rev-parse HEAD > {$workdir}/.git-commit"); + $this->command[] = "echo 'Done.'"; + // Remove .git folder + $this->command[] = "echo -n 'Removing .git folder... '"; $this->execute_in_builder("rm -fr {$workdir}/.git"); - - // Create docker-compose.yml + $this->command[] = "echo 'Done.'"; + // Create docker-compose.yml && replace TAG with git commit + $docker_compose_base64 = base64_encode($this->generate_docker_compose($this->application)); $this->execute_in_builder("echo '{$docker_compose_base64}' | base64 -d > {$workdir}/docker-compose.yml"); - // Set TAG in docker-compose.yml $this->execute_in_builder("sed -i \"s/TAG/$(cat {$workdir}/.git-commit)/g\" {$workdir}/docker-compose.yml"); + $this->command[] = "echo -n 'Generating nixpacks configuration... '"; if (str_starts_with($this->application->base_image, 'apache') || str_starts_with($this->application->base_image, 'nginx')) { // @TODO: Add static site builds } else { @@ -135,10 +215,17 @@ class DeployApplication extends Component $this->execute_in_builder("cp {$workdir}/.nixpacks/Dockerfile {$workdir}/Dockerfile"); $this->execute_in_builder("rm -f {$workdir}/.nixpacks/Dockerfile"); } + $this->command[] = "echo 'Done.'"; + $this->command[] = "echo -n 'Building image... '"; $this->execute_in_builder("docker build -f {$workdir}/Dockerfile --build-arg SOURCE_COMMIT=$(cat {$workdir}/.git-commit) --progress plain -t {$this->application->uuid}:$(cat {$workdir}/.git-commit) {$workdir}"); - $this->execute_in_builder("test -z \"$(docker ps --format '{{.State}}' --filter 'name={$this->application->uuid}')\" && docker rm -f {$this->application->uuid}"); + $this->command[] = "echo 'Done.'"; + $this->execute_in_builder("test ! -z \"$(docker ps --format '{{.State}}' --filter 'name={$this->application->uuid}')\" && docker rm -f {$this->application->uuid} >/dev/null 2>&1"); + + $this->command[] = "echo -n 'Deploying... '"; + $this->execute_in_builder("docker compose --project-directory {$workdir} up -d"); + $this->command[] = "echo 'Done. 🎉'"; $this->command[] = "docker stop -t 0 {$this->deployment_uuid} >/dev/null"; $this->activity = remoteProcess($this->command, $this->destination->server, $this->deployment_uuid, $this->application); diff --git a/composer.json b/composer.json index 64d74332a..ab21d01e1 100644 --- a/composer.json +++ b/composer.json @@ -11,6 +11,7 @@ "laravel/framework": "^10.0", "laravel/sanctum": "^3.2", "laravel/tinker": "^2.8", + "lcobucci/jwt": "^5.0", "livewire/livewire": "^2.12", "spatie/laravel-activitylog": "^4.7", "spatie/laravel-data": "^3.2", diff --git a/composer.lock b/composer.lock index a880bdcd0..28478c6fd 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "82e138615877e8bffa63f91428b555d2", + "content-hash": "03c1963c1ac555dff749aba587e631c8", "packages": [ { "name": "bacon/bacon-qr-code", @@ -1584,6 +1584,81 @@ }, "time": "2023-02-15T16:40:09+00:00" }, + { + "name": "lcobucci/jwt", + "version": "5.0.0", + "source": { + "type": "git", + "url": "https://github.com/lcobucci/jwt.git", + "reference": "47bdb0e0b5d00c2f89ebe33e7e384c77e84e7c34" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/lcobucci/jwt/zipball/47bdb0e0b5d00c2f89ebe33e7e384c77e84e7c34", + "reference": "47bdb0e0b5d00c2f89ebe33e7e384c77e84e7c34", + "shasum": "" + }, + "require": { + "ext-hash": "*", + "ext-json": "*", + "ext-openssl": "*", + "ext-sodium": "*", + "php": "~8.1.0 || ~8.2.0", + "psr/clock": "^1.0" + }, + "require-dev": { + "infection/infection": "^0.26.19", + "lcobucci/clock": "^3.0", + "lcobucci/coding-standard": "^9.0", + "phpbench/phpbench": "^1.2.8", + "phpstan/extension-installer": "^1.2", + "phpstan/phpstan": "^1.10.3", + "phpstan/phpstan-deprecation-rules": "^1.1.2", + "phpstan/phpstan-phpunit": "^1.3.8", + "phpstan/phpstan-strict-rules": "^1.5.0", + "phpunit/phpunit": "^10.0.12" + }, + "suggest": { + "lcobucci/clock": ">= 3.0" + }, + "type": "library", + "autoload": { + "psr-4": { + "Lcobucci\\JWT\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Luís Cobucci", + "email": "lcobucci@gmail.com", + "role": "Developer" + } + ], + "description": "A simple library to work with JSON Web Token and JSON Web Signature", + "keywords": [ + "JWS", + "jwt" + ], + "support": { + "issues": "https://github.com/lcobucci/jwt/issues", + "source": "https://github.com/lcobucci/jwt/tree/5.0.0" + }, + "funding": [ + { + "url": "https://github.com/lcobucci", + "type": "github" + }, + { + "url": "https://www.patreon.com/lcobucci", + "type": "patreon" + } + ], + "time": "2023-02-25T21:35:16+00:00" + }, { "name": "league/commonmark", "version": "2.4.0", @@ -2889,6 +2964,54 @@ }, "time": "2022-06-13T21:57:56+00:00" }, + { + "name": "psr/clock", + "version": "1.0.0", + "source": { + "type": "git", + "url": "https://github.com/php-fig/clock.git", + "reference": "e41a24703d4560fd0acb709162f73b8adfc3aa0d" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/clock/zipball/e41a24703d4560fd0acb709162f73b8adfc3aa0d", + "reference": "e41a24703d4560fd0acb709162f73b8adfc3aa0d", + "shasum": "" + }, + "require": { + "php": "^7.0 || ^8.0" + }, + "type": "library", + "autoload": { + "psr-4": { + "Psr\\Clock\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "PHP-FIG", + "homepage": "https://www.php-fig.org/" + } + ], + "description": "Common interface for reading the clock.", + "homepage": "https://github.com/php-fig/clock", + "keywords": [ + "clock", + "now", + "psr", + "psr-20", + "time" + ], + "support": { + "issues": "https://github.com/php-fig/clock/issues", + "source": "https://github.com/php-fig/clock/tree/1.0.0" + }, + "time": "2022-11-25T14:36:26+00:00" + }, { "name": "psr/container", "version": "2.0.2",