value; } if ($command instanceof Collection) { $command = $command->toArray(); } if ($server->isNonRoot()) { $command = parseCommandsByLineForSudo(collect($command), $server); } ray($command); $command_string = implode("\n", $command); if (auth()->user()) { $teams = auth()->user()->teams->pluck('id'); if (!$teams->contains($server->team_id) && !$teams->contains(0)) { throw new \Exception("User is not part of the team that owns this server"); } } return resolve(PrepareCoolifyTask::class, [ 'remoteProcessArgs' => new CoolifyTaskArgs( server_uuid: $server->uuid, command: <<muxFilename(); return [ 'location' => $location, 'mux_filename' => $mux_filename, 'private_key_filename' => $private_key_filename ]; } function savePrivateKeyToFs(Server $server) { if (data_get($server, 'privateKey.private_key') === null) { throw new \Exception("Server {$server->name} does not have a private key"); } ['location' => $location, 'private_key_filename' => $private_key_filename] = server_ssh_configuration($server); Storage::disk('ssh-keys')->makeDirectory('.'); Storage::disk('ssh-mux')->makeDirectory('.'); Storage::disk('ssh-keys')->put($private_key_filename, $server->privateKey->private_key); return $location; } function generateScpCommand(Server $server, string $source, string $dest) { $user = $server->user; $port = $server->port; $privateKeyLocation = savePrivateKeyToFs($server); $timeout = config('constants.ssh.command_timeout'); $connectionTimeout = config('constants.ssh.connection_timeout'); $serverInterval = config('constants.ssh.server_interval'); $scp_command = "timeout $timeout scp "; $scp_command .= "-i {$privateKeyLocation} " . '-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ' . '-o PasswordAuthentication=no ' . "-o ConnectTimeout=$connectionTimeout " . "-o ServerAliveInterval=$serverInterval " . '-o RequestTTY=no ' . '-o LogLevel=ERROR ' . "-P {$port} " . "{$source} " . "{$user}@{$server->ip}:{$dest}"; return $scp_command; } function instant_scp(string $source, string $dest, Server $server, $throwError = true) { $timeout = config('constants.ssh.command_timeout'); $scp_command = generateScpCommand($server, $source, $dest); $process = Process::timeout($timeout)->run($scp_command); $output = trim($process->output()); $exitCode = $process->exitCode(); if ($exitCode !== 0) { if (!$throwError) { return null; } return excludeCertainErrors($process->errorOutput(), $exitCode); } if ($output === 'null') { $output = null; } return $output; } function generateSshCommand(Server $server, string $command) { if ($server->settings->force_disabled) { throw new \RuntimeException('Server is disabled.'); } $user = $server->user; $port = $server->port; $privateKeyLocation = savePrivateKeyToFs($server); $timeout = config('constants.ssh.command_timeout'); $connectionTimeout = config('constants.ssh.connection_timeout'); $serverInterval = config('constants.ssh.server_interval'); $muxPersistTime = config('constants.ssh.mux_persist_time'); $ssh_command = "timeout $timeout ssh "; if (config('coolify.mux_enabled') && config('coolify.is_windows_docker_desktop') == false) { $ssh_command .= "-o ControlMaster=auto -o ControlPersist={$muxPersistTime} -o ControlPath=/var/www/html/storage/app/ssh/mux/%h_%p_%r "; } if (data_get($server, 'settings.is_cloudflare_tunnel')) { $ssh_command .= '-o ProxyCommand="/usr/local/bin/cloudflared access ssh --hostname %h" '; } $command = "PATH=\$PATH:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/host/usr/local/sbin:/host/usr/local/bin:/host/usr/sbin:/host/usr/bin:/host/sbin:/host/bin && $command"; $delimiter = Hash::make($command); $command = str_replace($delimiter, '', $command); $ssh_command .= "-i {$privateKeyLocation} " . '-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ' . '-o PasswordAuthentication=no ' . "-o ConnectTimeout=$connectionTimeout " . "-o ServerAliveInterval=$serverInterval " . '-o RequestTTY=no ' . '-o LogLevel=ERROR ' . "-p {$port} " . "{$user}@{$server->ip} " . " 'bash -se' << \\$delimiter" . PHP_EOL . $command . PHP_EOL . $delimiter; // ray($ssh_command); return $ssh_command; } function instant_remote_process(Collection|array $command, Server $server, bool $throwError = true, bool $no_sudo = false) { $timeout = config('constants.ssh.command_timeout'); if ($command instanceof Collection) { $command = $command->toArray(); } if ($server->isNonRoot() && !$no_sudo) { $command = parseCommandsByLineForSudo(collect($command), $server); } $command_string = implode("\n", $command); $ssh_command = generateSshCommand($server, $command_string, $no_sudo); $process = Process::timeout($timeout)->run($ssh_command); $output = trim($process->output()); $exitCode = $process->exitCode(); if ($exitCode !== 0) { if (!$throwError) { return null; } return excludeCertainErrors($process->errorOutput(), $exitCode); } if ($output === 'null') { $output = null; } return $output; } function excludeCertainErrors(string $errorOutput, ?int $exitCode = null) { $ignoredErrors = collect([ 'Permission denied (publickey', 'Could not resolve hostname', ]); $ignored = false; foreach ($ignoredErrors as $ignoredError) { if (Str::contains($errorOutput, $ignoredError)) { $ignored = true; break; } } if ($ignored) { // TODO: Create new exception and disable in sentry throw new \RuntimeException($errorOutput, $exitCode); } throw new \RuntimeException($errorOutput, $exitCode); } function decode_remote_command_output(?ApplicationDeploymentQueue $application_deployment_queue = null): Collection { $application = Application::find(data_get($application_deployment_queue, 'application_id')); $is_debug_enabled = data_get($application, 'settings.is_debug_enabled'); if (is_null($application_deployment_queue)) { return collect([]); } // ray(data_get($application_deployment_queue, 'logs')); try { $decoded = json_decode( data_get($application_deployment_queue, 'logs'), associative: true, flags: JSON_THROW_ON_ERROR ); } catch (\JsonException $exception) { return collect([]); } // ray($decoded ); $formatted = collect($decoded); if (!$is_debug_enabled) { $formatted = $formatted->filter(fn ($i) => $i['hidden'] === false ?? false); } $formatted = $formatted ->sortBy(fn ($i) => data_get($i, 'order')) ->map(function ($i) { data_set($i, 'timestamp', Carbon::parse(data_get($i, 'timestamp'))->format('Y-M-d H:i:s.u')); return $i; }); return $formatted; } function remove_iip($text) { $text = preg_replace('/x-access-token:.*?(?=@)/', "x-access-token:" . REDACTED, $text); return preg_replace('/\x1b\[[0-9;]*m/', '', $text); } function remove_mux_and_private_key(Server $server) { $muxFilename = $server->muxFilename(); $privateKeyLocation = savePrivateKeyToFs($server); Storage::disk('ssh-mux')->delete($muxFilename); Storage::disk('ssh-keys')->delete($privateKeyLocation); } function refresh_server_connection(?PrivateKey $private_key = null) { if (is_null($private_key)) { return; } foreach ($private_key->servers as $server) { Storage::disk('ssh-mux')->delete($server->muxFilename()); } } function checkRequiredCommands(Server $server) { $commands = collect(["jq", "jc"]); foreach ($commands as $command) { $commandFound = instant_remote_process(["docker run --rm --privileged --net=host --pid=host --ipc=host --volume /:/host busybox chroot /host bash -c 'command -v {$command}'"], $server, false); if ($commandFound) { ray($command . ' found'); continue; } try { instant_remote_process(["docker run --rm --privileged --net=host --pid=host --ipc=host --volume /:/host busybox chroot /host bash -c 'apt update && apt install -y {$command}'"], $server); } catch (\Throwable $e) { ray('could not install ' . $command); ray($e); break; } $commandFound = instant_remote_process(["docker run --rm --privileged --net=host --pid=host --ipc=host --volume /:/host busybox chroot /host bash -c 'command -v {$command}'"], $server, false); if ($commandFound) { ray($command . ' found'); continue; } ray('could not install ' . $command); break; } }