Merge branch 'v4' into ijpatricio-wip-9

app/Actions/RemoteProcess/RunRemoteProcess.php

@@ -67,6 +67,9 @@ class RunRemoteProcess
 
         $this->activity->save();
 
+        if ($processResult->exitCode() != 0 && $processResult->errorOutput()) {
+            throw new \RuntimeException('Remote command failed');
+        }
         return $processResult;
     }
 

app/Jobs/DeployApplicationJob.php

@@ -14,6 +14,7 @@ use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Collection;
 use Illuminate\Support\Facades\Http;
 use Lcobucci\JWT\Encoding\ChainedFormatter;
 use Lcobucci\JWT\Encoding\JoseEncoder;
@@ -49,7 +50,7 @@ class DeployApplicationJob implements ShouldQueue
 
         $server = $this->destination->server;
 
-        $private_key_location = savePrivateKey($server);
+        $private_key_location = savePrivateKeyForServer($server);
 
         $remoteProcessArgs = new RemoteProcessArgs(
             server_ip: $server->ip,
@@ -98,8 +99,14 @@ class DeployApplicationJob implements ShouldQueue
 
         // Import git repository
         $this->executeNow([
-            "echo -n 'Importing {$this->application->git_repository}:{$this->application->git_branch} to {$this->workdir}... '",
+            "echo -n 'Importing {$this->application->git_repository}:{$this->application->git_branch} to {$this->workdir}... '"
-            $this->gitImport(),
+        ]);
+
+        $this->executeNow([
+            ...$this->gitImport(),
+        ], 'importing_git_repository');
+
+        $this->executeNow([
             "echo 'Done.'"
         ]);
 
@@ -135,10 +142,8 @@ class DeployApplicationJob implements ShouldQueue
         ]);
         $this->executeNow([
             "echo -n 'Starting new container... '",
-            $this->execute_in_builder("docker compose --project-directory {$this->workdir} up -d >/dev/null 2>&1"),
+            $this->execute_in_builder("docker compose --project-directory {$this->workdir} up -d >/dev/null"),
             "echo 'Done. 🎉'",
-        ], setStatus: true);
-        $this->executeNow([
             "docker stop -t 0 {$this->deployment_uuid} >/dev/null"
         ], setStatus: true);
     }
@@ -150,7 +155,8 @@ class DeployApplicationJob implements ShouldQueue
 
     private function generate_docker_compose()
     {
-
+        $persistentStorages = $this->generate_local_persistent_volumes();
+        $volume_names = $this->generate_local_persistent_volumes_only_volume_names();
         $docker_compose = [
             'version' => '3.8',
             'services' => [
@@ -158,6 +164,9 @@ class DeployApplicationJob implements ShouldQueue
                     'image' => "{$this->application->uuid}:$this->git_commit",
                     'container_name' => $this->application->uuid,
                     'restart' => 'always',
+                    'environment' => [
+                        'PORT' => $this->application->ports_exposes[0]
+                    ],
                     'labels' => $this->set_labels_for_applications(),
                     'expose' => $this->application->ports_exposes,
                     'networks' => [
@@ -186,15 +195,36 @@ class DeployApplicationJob implements ShouldQueue
         if (count($this->application->ports_mappings) > 0) {
             $docker_compose['services'][$this->application->uuid]['ports'] = $this->application->ports_mappings;
         }
-        // if (count($volumes) > 0) {
+        if (count($persistentStorages) > 0) {
-        //     $docker_compose['services'][$this->application->uuid]['volumes'] = $volumes;
+            $docker_compose['services'][$this->application->uuid]['volumes'] = $persistentStorages;
-        // }
+        }
-        // if (count($volume_names) > 0) {
+        if (count($volume_names) > 0) {
-        //     $docker_compose['volumes'] = $volume_names;
+            $docker_compose['volumes'] = $volume_names;
-        // }
+        }
         return Yaml::dump($docker_compose);
     }
+    private function generate_local_persistent_volumes()
+    {
+        foreach ($this->application->persistentStorages as $persistentStorage) {
+            $volume_name = $persistentStorage->host_path ?? $persistentStorage->name;
+            $local_persistent_volumes[] = $volume_name . ':' . $persistentStorage->mount_path;
+        }
+        return $local_persistent_volumes ?? [];
+    }
 
+    private function generate_local_persistent_volumes_only_volume_names()
+    {
+        foreach ($this->application->persistentStorages as $persistentStorage) {
+            if ($persistentStorage->host_path) {
+                continue;
+            }
+            $local_persistent_volumes_names[$persistentStorage->name] = [
+                'name' => $persistentStorage->name,
+                'external' => false,
+            ];
+        }
+        return $local_persistent_volumes_names ?? [];
+    }
     private function generate_healthcheck_commands()
     {
         if (!$this->application->health_check_port) {
@@ -202,18 +232,12 @@ class DeployApplicationJob implements ShouldQueue
         }
         if ($this->application->health_check_path) {
             $generated_healthchecks_commands = [
-                "curl -X {$this->application->health_check_method} -f {$this->application->health_check_scheme}://{$this->application->health_check_host}:{$this->application->health_check_port}{$this->application->health_check_path}"
+                "curl -s -X {$this->application->health_check_method} -f {$this->application->health_check_scheme}://{$this->application->health_check_host}:{$this->application->health_check_port}{$this->application->health_check_path} > /dev/null"
             ];
         } else {
-            $generated_healthchecks_commands = [];
+            $generated_healthchecks_commands = [
-            foreach ($this->application->ports_exposes as $key => $port) {
+                "curl -s -X {$this->application->health_check_method} -f {$this->application->health_check_scheme}://{$this->application->health_check_host}:{$this->application->health_check_port}/"
-                $generated_healthchecks_commands = [
+            ];
-                    "curl -X {$this->application->health_check_method} -f {$this->application->health_check_scheme}://{$this->application->health_check_host}:{$port}/"
-                ];
-                if (count($this->application->ports_exposes) != $key + 1) {
-                    $generated_healthchecks_commands[] = '&&';
-                }
-            }
         }
         return implode(' ', $generated_healthchecks_commands);
     }
@@ -255,10 +279,14 @@ class DeployApplicationJob implements ShouldQueue
         return $labels;
     }
 
-    private function executeNow(array $command, string $propertyName = null, bool $hideFromOutput = false, $setStatus = false)
+    private function executeNow(array|Collection $command, string $propertyName = null, bool $hideFromOutput = false, $setStatus = false)
     {
         static::$batch_counter++;
-        $commandText = collect($command)->implode("\n");
+        if ($command instanceof Collection) {
+            $commandText = $command->implode("\n");
+        } else {
+            $commandText = collect($command)->implode("\n");
+        }
 
         $this->activity->properties = $this->activity->properties->merge([
             'command' => $commandText,
@@ -270,15 +298,13 @@ class DeployApplicationJob implements ShouldQueue
             'hideFromOutput' => $hideFromOutput,
             'setStatus' => $setStatus,
         ]);
+        $result = $remoteProcess();
 
         if ($propertyName) {
-            $result = $remoteProcess();
             $this->activity->properties = $this->activity->properties->merge([
                 $propertyName => trim($result->output()),
             ]);
             $this->activity->save();
-        } else {
-            $remoteProcess();
         }
     }
     private function gitImport()
@@ -287,12 +313,27 @@ class DeployApplicationJob implements ShouldQueue
         $url = parse_url(filter_var($source_html_url, FILTER_SANITIZE_URL));
         $source_html_url_host = $url['host'];
         $source_html_url_scheme = $url['scheme'];
+
         if ($this->application->source->getMorphClass() == 'App\Models\GithubApp') {
             if ($this->source->is_public) {
-                return $this->execute_in_builder("git clone -q -b {$this->application->git_branch} {$this->source->html_url}/{$this->application->git_repository}.git {$this->workdir}");
+                return [
+                    $this->execute_in_builder("git clone -q -b {$this->application->git_branch} {$this->source->html_url}/{$this->application->git_repository}.git {$this->workdir}")
+                ];
             } else {
-                $github_access_token = $this->generate_jwt_token_for_github();
+                if (!$this->application->source->app_id) {
-                return $this->execute_in_builder("git clone -q -b {$this->application->git_branch} $source_html_url_scheme://x-access-token:$github_access_token@$source_html_url_host/{$this->application->git_repository}.git {$this->workdir}");
+                    $private_key = base64_encode($this->application->source->privateKey->private_key);
+                    return [
+                        $this->execute_in_builder("mkdir -p /root/.ssh"),
+                        $this->execute_in_builder("echo '{$private_key}' | base64 -d > /root/.ssh/id_rsa"),
+                        $this->execute_in_builder("chmod 600 /root/.ssh/id_rsa"),
+                        $this->execute_in_builder("GIT_SSH_COMMAND=\"ssh -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa\" git clone -q -b {$this->application->git_branch} git@$source_html_url_host:{$this->application->git_repository}.git {$this->workdir}")
+                    ];
+                } else {
+                    $github_access_token = $this->generate_jwt_token_for_github();
+                    return [
+                        $this->execute_in_builder("git clone -q -b {$this->application->git_branch} $source_html_url_scheme://x-access-token:$github_access_token@$source_html_url_host/{$this->application->git_repository}.git {$this->workdir}")
+                    ];
+                }
             }
         }
     }

bootstrap/helpers.php

@@ -29,7 +29,7 @@ if (!function_exists('remoteProcess')) {
         // @TODO: Check if the user has access to this server
         // checkTeam($server->team_id);
 
-        $private_key_location = savePrivateKey($server);
+        $private_key_location = savePrivateKeyForServer($server);
 
         return resolve(DispatchRemoteProcess::class, [
             'remoteProcessArgs' => new RemoteProcessArgs(
@@ -56,8 +56,8 @@ if (!function_exists('remoteProcess')) {
 //     }
 // }
 
-if (!function_exists('savePrivateKey')) {
+if (!function_exists('savePrivateKeyForServer')) {
-    function savePrivateKey(Server $server)
+    function savePrivateKeyForServer(Server $server)
     {
         $temp_file = 'id.rsa_' . 'root' . '@' . $server->ip;
         Storage::disk('local')->put($temp_file, $server->privateKey->private_key, 'private');
@@ -118,9 +118,10 @@ if (!function_exists('formatDockerLabelsToJson')) {
     }
 }
 if (!function_exists('runRemoteCommandSync')) {
-    function runRemoteCommandSync($server, array $command) {
+    function runRemoteCommandSync($server, array $command)
+    {
         $command_string = implode("\n", $command);
-        $private_key_location = savePrivateKey($server);
+        $private_key_location = savePrivateKeyForServer($server);
         $ssh_command = generateSshCommand($private_key_location, $server->ip, $server->user, $server->port, $command_string);
         $process = Process::run($ssh_command);
         $output = trim($process->output());

database/migrations/2023_03_27_081716_create_applications_table.php

@@ -40,7 +40,7 @@ return new class extends Migration
             $table->string('base_directory')->default('/');
             $table->string('publish_directory')->nullable();
 
-            $table->string('health_check_path')->nullable();
+            $table->string('health_check_path')->default('/');
             $table->string('health_check_port')->nullable();
             $table->string('health_check_host')->default('localhost');
             $table->string('health_check_method')->default('GET');

database/migrations/2023_03_28_083723_create_github_apps_table.php

@@ -21,8 +21,6 @@ return new class extends Migration
             $table->string('html_url');
             $table->integer('custom_port')->default(22);
             $table->string('custom_user')->default('git');
-            $table->boolean('is_system_wide')->default(false);
-            $table->boolean('is_public')->default(false);
 
             $table->integer('app_id')->nullable();
             $table->integer('installation_id')->nullable();
@@ -30,6 +28,9 @@ return new class extends Migration
             $table->longText('client_secret')->nullable();
             $table->longText('webhook_secret')->nullable();
 
+            $table->boolean('is_system_wide')->default(false);
+            $table->boolean('is_public')->default(false);
+
             $table->foreignId('private_key_id')->nullable();
             $table->foreignId('team_id');
             $table->timestamps();

database/seeders/ApplicationSeeder.php

@@ -22,12 +22,12 @@ class ApplicationSeeder extends Seeder
         $standalone_docker_1 = StandaloneDocker::find(1);
         $swarm_docker_1 = SwarmDocker::find(1);
 
-        $github_public_source = GithubApp::find(1);
+        $github_public_source = GithubApp::where('name', 'Public GitHub')->first();
-        $github_private_source = GithubApp::find(2);
+        $github_private_source = GithubApp::where('name', 'coolify-laravel-development-private-github')->first();
+        $github_private_source_with_deploy_key = GithubApp::where('name', 'Private GitHub (deployment key)')->first();
 
         $pv_storage = LocalPersistentVolume::find(1);
         Application::create([
-            'id' => 1,
             'name' => 'Public application (from GitHub)',
             'git_repository' => 'coollabsio/coolify-examples',
             'git_branch' => 'nodejs-fastify',
@@ -40,19 +40,31 @@ class ApplicationSeeder extends Seeder
             'source_id' => $github_public_source->id,
             'source_type' => GithubApp::class,
         ]);
-        Application::create([
+        // Application::create([
-            'id' => 2,
+        //     'name' => 'Private application (through GitHub App)',
-            'name' => 'Private application (through GitHub App)',
+        //     'git_repository' => 'coollabsio/nodejs-example',
-            'git_repository' => 'coollabsio/nodejs-example',
+        //     'git_branch' => 'main',
-            'git_branch' => 'main',
+        //     'build_pack' => 'nixpacks',
-            'build_pack' => 'nixpacks',
+        //     'ports_exposes' => '3000',
-            'ports_exposes' => '3000',
+        //     'ports_mappings' => '3001:3000',
-            'ports_mappings' => '3001:3000',
+        //     'environment_id' => $environment_1->id,
-            'environment_id' => $environment_1->id,
+        //     'destination_id' => $standalone_docker_1->id,
-            'destination_id' => $standalone_docker_1->id,
+        //     'destination_type' => StandaloneDocker::class,
-            'destination_type' => StandaloneDocker::class,
+        //     'source_id' => $github_private_source->id,
-            'source_id' => $github_private_source->id,
+        //     'source_type' => GithubApp::class,
-            'source_type' => GithubApp::class,
+        // ]);
-        ]);
+        // Application::create([
+        //     'name' => 'Public application (from GitHub through Deploy Key)',
+        //     'git_repository' => 'coollabsio/php',
+        //     'git_branch' => 'main',
+        //     'build_pack' => 'nixpacks',
+        //     'ports_exposes' => '80,3000',
+        //     'ports_mappings' => '3002:80',
+        //     'environment_id' => $environment_1->id,
+        //     'destination_id' => $standalone_docker_1->id,
+        //     'destination_type' => StandaloneDocker::class,
+        //     'source_id' => $github_private_source_with_deploy_key->id,
+        //     'source_type' => GithubApp::class,
+        // ]);
     }
 }

database/seeders/DatabaseSeeder.php

@@ -18,7 +18,6 @@ class DatabaseSeeder extends Seeder
             ProjectSeeder::class,
             ProjectSettingSeeder::class,
             EnvironmentSeeder::class,
-            LocalPersistentVolumeSeeder::class,
             StandaloneDockerSeeder::class,
             SwarmDockerSeeder::class,
             KubernetesSeeder::class,
@@ -28,6 +27,7 @@ class DatabaseSeeder extends Seeder
             ApplicationSettingsSeeder::class,
             DBSeeder::class,
             ServiceSeeder::class,
+            LocalPersistentVolumeSeeder::class,
         ]);
     }
 }

database/seeders/GithubAppSeeder.php

@@ -16,9 +16,9 @@ class GithubAppSeeder extends Seeder
     public function run(): void
     {
         $root_team = Team::find(1);
+        $private_key_1 = PrivateKey::find(1);
         $private_key_2 = PrivateKey::find(2);
         GithubApp::create([
-            'id' => 1,
             'name' => 'Public GitHub',
             'api_url' => 'https://api.github.com',
             'html_url' => 'https://github.com',
@@ -26,7 +26,6 @@ class GithubAppSeeder extends Seeder
             'team_id' => $root_team->id,
         ]);
         GithubApp::create([
-            'id' => 2,
             'name' => 'coolify-laravel-development-private-github',
             'api_url' => 'https://api.github.com',
             'html_url' => 'https://github.com',
@@ -39,5 +38,13 @@ class GithubAppSeeder extends Seeder
             'private_key_id' => $private_key_2->id,
             'team_id' => $root_team->id,
         ]);
+        GithubApp::create([
+            'name' => 'Private GitHub (deployment key)',
+            'api_url' => 'https://api.github.com',
+            'html_url' => 'https://github.com',
+            'is_public' => false,
+            'private_key_id' => $private_key_1->id,
+            'team_id' => $root_team->id,
+        ]);
     }
 }

database/seeders/LocalPersistentVolumeSeeder.php

@@ -13,10 +13,11 @@ class LocalPersistentVolumeSeeder extends Seeder
      */
     public function run(): void
     {
+        $application = Application::where('name', 'Public application (from GitHub)')->first();
         LocalPersistentVolume::create([
             'name' => 'test-pv',
             'mount_path' => '/data',
-            'resource_id' => 1,
+            'resource_id' => $application->id,
             'resource_type' => Application::class,
         ]);
     }

run

@@ -1,51 +0,0 @@
-#!/usr/bin/env bash
-
-# Inspired on https://github.com/adriancooney/Taskfile
-#
-# Install an alias, to be able to simply execute `run`
-# echo 'alias run=./run' >> ~/.aliases
-#
-
-# Define Docker Compose command prefix...
-set -e
-
-docker compose &> /dev/null
-if [ $? == 0 ]; then
-    DOCKER_COMPOSE="docker compose"
-else
-    DOCKER_COMPOSE="docker-compose"
-fi
-
-SAIL=./vendor/bin/sail
-export WWWUSER=${WWWUSER:-$UID}
-export WWWGROUP=${WWWGROUP:-$(id -g)}
-
-function help {
-    echo "$0 <task> <args>"
-    echo "Tasks:"
-    compgen -A function | cat -n
-}
-
-function default {
-    help
-}
-
-function wait_db {
-    TRIES=0
-    MAX_TRIES=15
-    WAIT=4
-
-    until $DOCKER_COMPOSE exec postgres bash -c "psql -U coolify -d coolify -t -q -c  \"SELECT datname FROM pg_database;\" " | grep coolify
-    do
-        ((TRIES++))
-        if [ $TRIES -gt $MAX_TRIES ]; then
-            echo "Database is not ready after $MAX_TRIES tries. Exiting."
-            exit 1
-        fi
-        echo "Database is not ready yet. Attempt $TRIES/$MAX_TRIES. Waiting $WAIT seconds before next try..."
-        sleep $WAIT
-    done
-}
-
-TIMEFORMAT="Task completed in %3lR"
-time "${@:-default}"

scripts/run

@@ -1,11 +1,25 @@
 #!/usr/bin/env bash
 
 # Inspired on https://github.com/adriancooney/Taskfile
-# Install an alias, to be able to simply execute `run` => echo 'alias run=./run' >> ~/.aliases
+#
+# Install an alias, to be able to simply execute `run`
+# echo 'alias run=./run' >> ~/.aliases
 #
 
+# Define Docker Compose command prefix...
 set -e
 
+docker compose &> /dev/null
+if [ $? == 0 ]; then
+    DOCKER_COMPOSE="docker compose"
+else
+    DOCKER_COMPOSE="docker-compose"
+fi
+
+SAIL=./vendor/bin/sail
+export WWWUSER=${WWWUSER:-$UID}
+export WWWGROUP=${WWWGROUP:-$(id -g)}
+
 function help {
     echo "$0 <task> <args>"
     echo "Tasks:"
@@ -16,23 +30,21 @@ function default {
     help
 }
 
-function bash {
+function wait_db {
-	docker-compose exec -u $(id -u) php bash
+    TRIES=0
-}
+    MAX_TRIES=15
+    WAIT=4
 
-# The user with native SSH capability
+    until $DOCKER_COMPOSE exec postgres bash -c "psql -U coolify -d coolify -t -q -c  \"SELECT datname FROM pg_database;\" " | grep coolify
-function coolify-bash {
+    do
-	docker-compose exec -u coolify php bash
+        ((TRIES++))
-}
+        if [ $TRIES -gt $MAX_TRIES ]; then
-
+            echo "Database is not ready after $MAX_TRIES tries. Exiting."
-function root-bash {
+            exit 1
-	docker-compose exec php bash
+        fi
-}
+        echo "Database is not ready yet. Attempt $TRIES/$MAX_TRIES. Waiting $WAIT seconds before next try..."
-
+        sleep $WAIT
-# Usage: ./Taskfile envFile:set FOOBAR abc
+    done
-# This will set the FOOBAR variable to "abc" in the .env file
-function envFile:set {
-    sed -i "s#^$1=.*#$1=$2#g" .env
 }
 
 TIMEFORMAT="Task completed in %3lR"