lasthourcloud/bootstrap/helpers/remoteProcess.php

259 lines
9.2 KiB
PHP
Raw Normal View History

2023-05-24 12:26:50 +00:00
<?php
2023-05-24 13:25:08 +00:00
use App\Actions\CoolifyTask\PrepareCoolifyTask;
2023-05-24 12:26:50 +00:00
use App\Data\CoolifyTaskArgs;
2023-05-25 12:40:47 +00:00
use App\Enums\ActivityTypes;
2023-06-30 13:57:40 +00:00
use App\Models\Application;
use App\Models\ApplicationDeploymentQueue;
use App\Models\PrivateKey;
2023-05-24 12:26:50 +00:00
use App\Models\Server;
use App\Notifications\Server\Revived;
use App\Notifications\Server\Unreachable;
2023-06-30 13:57:40 +00:00
use Carbon\Carbon;
2023-05-24 12:26:50 +00:00
use Illuminate\Database\Eloquent\Model;
2023-06-30 13:57:40 +00:00
use Illuminate\Support\Collection;
2023-05-24 12:26:50 +00:00
use Illuminate\Support\Facades\Process;
use Illuminate\Support\Facades\Storage;
2023-09-15 10:43:03 +00:00
use Illuminate\Support\Str;
2023-09-18 09:49:26 +00:00
use Spatie\Activitylog\Contracts\Activity;
2023-05-24 12:26:50 +00:00
2023-05-24 13:25:08 +00:00
function remote_process(
Collection|array $command,
2023-05-24 13:25:08 +00:00
Server $server,
2023-09-18 12:41:31 +00:00
?string $type = null,
2023-05-24 13:25:08 +00:00
?string $type_uuid = null,
?Model $model = null,
2023-06-30 20:24:39 +00:00
bool $ignore_errors = false,
$callEventOnFinish = null
2023-08-11 18:48:52 +00:00
): Activity {
2023-09-18 12:41:31 +00:00
if (is_null($type)) {
$type = ActivityTypes::INLINE->value;
}
if ($command instanceof Collection) {
$command = $command->toArray();
}
2023-05-24 13:25:08 +00:00
$command_string = implode("\n", $command);
2023-06-30 20:24:39 +00:00
if (auth()->user()) {
$teams = auth()->user()->teams->pluck('id');
if (!$teams->contains($server->team_id) && !$teams->contains(0)) {
throw new \Exception("User is not part of the team that owns this server");
}
}
2023-05-24 13:25:08 +00:00
return resolve(PrepareCoolifyTask::class, [
'remoteProcessArgs' => new CoolifyTaskArgs(
server_uuid: $server->uuid,
2023-05-24 13:25:08 +00:00
command: <<<EOT
2023-05-24 12:26:50 +00:00
{$command_string}
EOT,
2023-05-24 13:25:08 +00:00
type: $type,
type_uuid: $type_uuid,
model: $model,
ignore_errors: $ignore_errors,
call_event_on_finish: $callEventOnFinish,
2023-05-24 13:25:08 +00:00
),
])();
2023-05-24 12:26:50 +00:00
}
function savePrivateKeyToFs(Server $server)
2023-05-24 13:25:08 +00:00
{
2023-06-07 08:33:45 +00:00
if (data_get($server, 'privateKey.private_key') === null) {
throw new \Exception("Server {$server->name} does not have a private key");
}
2023-09-15 10:30:25 +00:00
$sshKeyFileLocation = "id.root@{$server->uuid}";
Storage::disk('ssh-keys')->makeDirectory('.');
Storage::disk('ssh-mux')->makeDirectory('.');
Storage::disk('ssh-keys')->put($sshKeyFileLocation, $server->privateKey->private_key);
$location = '/var/www/html/storage/app/ssh/keys/' . $sshKeyFileLocation;
return $location;
2023-05-24 12:26:50 +00:00
}
2024-01-06 05:24:57 +00:00
function generateScpCommand(Server $server, string $source, string $dest)
{
$user = $server->user;
$port = $server->port;
$privateKeyLocation = savePrivateKeyToFs($server);
$timeout = config('constants.ssh.command_timeout');
$connectionTimeout = config('constants.ssh.connection_timeout');
$serverInterval = config('constants.ssh.server_interval');
$scp_command = "timeout $timeout scp ";
$scp_command .= "-i {$privateKeyLocation} "
. '-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null '
. '-o PasswordAuthentication=no '
. "-o ConnectTimeout=$connectionTimeout "
. "-o ServerAliveInterval=$serverInterval "
. '-o RequestTTY=no '
. '-o LogLevel=ERROR '
. "-P {$port} "
. "{$source} "
. "{$user}@{$server->ip}:{$dest}";
return $scp_command;
}
function instant_scp(string $source, string $dest, Server $server, $throwError = true)
{
$timeout = config('constants.ssh.command_timeout');
$scp_command = generateScpCommand($server, $source, $dest);
$process = Process::timeout($timeout)->run($scp_command);
$output = trim($process->output());
$exitCode = $process->exitCode();
if ($exitCode !== 0) {
if (!$throwError) {
return null;
}
return excludeCertainErrors($process->errorOutput(), $exitCode);
}
if ($output === 'null') {
$output = null;
}
return $output;
}
function generateSshCommand(Server $server, string $command)
2023-05-24 13:25:08 +00:00
{
2024-02-26 13:22:24 +00:00
if ($server->settings->force_disabled) {
throw new \RuntimeException('Server is disabled.');
}
$user = $server->user;
$port = $server->port;
$privateKeyLocation = savePrivateKeyToFs($server);
2023-09-13 11:00:16 +00:00
$timeout = config('constants.ssh.command_timeout');
$connectionTimeout = config('constants.ssh.connection_timeout');
$serverInterval = config('constants.ssh.server_interval');
2023-05-24 13:25:08 +00:00
$delimiter = 'EOF-COOLIFY-SSH';
2023-09-13 11:00:16 +00:00
$ssh_command = "timeout $timeout ssh ";
2023-05-24 12:26:50 +00:00
if (config('coolify.mux_enabled') && config('coolify.is_windows_docker_desktop') == false) {
2023-05-25 10:55:34 +00:00
$ssh_command .= '-o ControlMaster=auto -o ControlPersist=1m -o ControlPath=/var/www/html/storage/app/ssh/mux/%h_%p_%r ';
2023-05-24 12:26:50 +00:00
}
if (data_get($server, 'settings.is_cloudflare_tunnel')) {
2023-09-23 11:34:40 +00:00
$ssh_command .= '-o ProxyCommand="/usr/local/bin/cloudflared access ssh --hostname %h" ';
}
$command = "PATH=\$PATH:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/host/usr/local/sbin:/host/usr/local/bin:/host/usr/sbin:/host/usr/bin:/host/sbin:/host/bin && $command";
2023-09-15 10:30:25 +00:00
$ssh_command .= "-i {$privateKeyLocation} "
. '-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null '
2023-05-24 13:25:08 +00:00
. '-o PasswordAuthentication=no '
2023-09-13 11:00:16 +00:00
. "-o ConnectTimeout=$connectionTimeout "
. "-o ServerAliveInterval=$serverInterval "
2023-05-24 13:25:08 +00:00
. '-o RequestTTY=no '
. '-o LogLevel=ERROR '
. "-p {$port} "
. "{$user}@{$server->ip} "
2023-05-24 13:25:08 +00:00
. " 'bash -se' << \\$delimiter" . PHP_EOL
. $command . PHP_EOL
. $delimiter;
2023-09-24 08:55:15 +00:00
// ray($ssh_command);
2023-05-24 13:25:08 +00:00
return $ssh_command;
2023-05-24 12:26:50 +00:00
}
function instant_remote_process(Collection|array $command, Server $server, $throwError = true)
2023-05-24 13:25:08 +00:00
{
$timeout = config('constants.ssh.command_timeout');
if ($command instanceof Collection) {
$command = $command->toArray();
}
2023-05-24 13:25:08 +00:00
$command_string = implode("\n", $command);
$ssh_command = generateSshCommand($server, $command_string);
$process = Process::timeout($timeout)->run($ssh_command);
2023-05-24 13:25:08 +00:00
$output = trim($process->output());
$exitCode = $process->exitCode();
if ($exitCode !== 0) {
if (!$throwError) {
return null;
2023-05-24 12:26:50 +00:00
}
2023-09-18 09:49:26 +00:00
return excludeCertainErrors($process->errorOutput(), $exitCode);
2023-05-24 12:26:50 +00:00
}
2023-11-28 17:31:04 +00:00
if ($output === 'null') {
$output = null;
}
2023-05-24 13:25:08 +00:00
return $output;
2023-05-24 12:26:50 +00:00
}
function excludeCertainErrors(string $errorOutput, ?int $exitCode = null)
{
2023-09-18 09:49:26 +00:00
$ignoredErrors = collect([
'Permission denied (publickey',
'Could not resolve hostname',
]);
$ignored = false;
foreach ($ignoredErrors as $ignoredError) {
2023-09-18 09:49:26 +00:00
if (Str::contains($errorOutput, $ignoredError)) {
$ignored = true;
break;
}
}
if ($ignored) {
// TODO: Create new exception and disable in sentry
throw new \RuntimeException($errorOutput, $exitCode);
}
throw new \RuntimeException($errorOutput, $exitCode);
}
2023-06-30 13:57:40 +00:00
function decode_remote_command_output(?ApplicationDeploymentQueue $application_deployment_queue = null): Collection
{
$application = Application::find(data_get($application_deployment_queue, 'application_id'));
$is_debug_enabled = data_get($application, 'settings.is_debug_enabled');
if (is_null($application_deployment_queue)) {
return collect([]);
}
// ray(data_get($application_deployment_queue, 'logs'));
2023-06-30 13:57:40 +00:00
try {
$decoded = json_decode(
2023-06-30 19:22:14 +00:00
data_get($application_deployment_queue, 'logs'),
2023-06-30 13:57:40 +00:00
associative: true,
flags: JSON_THROW_ON_ERROR
);
} catch (\JsonException $exception) {
return collect([]);
}
// ray($decoded );
2023-06-30 13:57:40 +00:00
$formatted = collect($decoded);
if (!$is_debug_enabled) {
2023-08-11 18:48:52 +00:00
$formatted = $formatted->filter(fn ($i) => $i['hidden'] === false ?? false);
2023-06-30 13:57:40 +00:00
}
2023-06-30 19:22:14 +00:00
$formatted = $formatted
2023-11-23 20:02:30 +00:00
->sortBy(fn ($i) => data_get($i, 'order'))
2023-06-30 13:57:40 +00:00
->map(function ($i) {
2023-11-23 20:02:30 +00:00
data_set($i, 'timestamp', Carbon::parse(data_get($i, 'timestamp'))->format('Y-M-d H:i:s.u'));
2023-06-30 13:57:40 +00:00
return $i;
});
return $formatted;
}
function remove_iip($text)
{
$text = preg_replace('/x-access-token:.*?(?=@)/', "x-access-token:" . REDACTED, $text);
return preg_replace('/\x1b\[[0-9;]*m/', '', $text);
}
2023-11-06 09:53:01 +00:00
function refresh_server_connection(?PrivateKey $private_key = null)
{
2023-11-06 09:53:01 +00:00
if (is_null($private_key)) {
return;
}
foreach ($private_key->servers as $server) {
Storage::disk('ssh-mux')->delete($server->muxFilename());
}
}
function checkRequiredCommands(Server $server)
{
$commands = collect(["jq", "jc"]);
foreach ($commands as $command) {
$commandFound = instant_remote_process(["docker run --rm --privileged --net=host --pid=host --ipc=host --volume /:/host busybox chroot /host bash -c 'command -v {$command}'"], $server, false);
if ($commandFound) {
ray($command . ' found');
continue;
}
try {
instant_remote_process(["docker run --rm --privileged --net=host --pid=host --ipc=host --volume /:/host busybox chroot /host bash -c 'apt update && apt install -y {$command}'"], $server);
2023-09-11 15:36:30 +00:00
} catch (\Throwable $e) {
ray('could not install ' . $command);
ray($e);
break;
}
$commandFound = instant_remote_process(["docker run --rm --privileged --net=host --pid=host --ipc=host --volume /:/host busybox chroot /host bash -c 'command -v {$command}'"], $server, false);
if ($commandFound) {
ray($command . ' found');
continue;
}
ray('could not install ' . $command);
break;
}
}