diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d8d8ffc..c7b9352c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,20 @@ # Changelog + ## 20240628 ## What's Changed * use Salt Repos for salt-master and salt-minion * better salt key handling * !! Attention: doil needs to be reinstalled !! +## 20240617 +## What's Changes +* update of an instance can now be triggered by url + +## 20240604 +## What's Changed +* CSP Rules per instance + ## 20240422 ## What's Changed * fix typo in apache 000-default diff --git a/README.md b/README.md index 2ead4a87..7680a279 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,7 @@ The following commands are available: * `doil delete ` deletes an instance you do not need anymore * `doil status` lists the current running doil instances * `doil exec ` executes a bash command inside the instance +* `doil sut` (alias for `doil instances:set-update-token`) Sets an update token as an environment variable. This is cross-checked during instance updates via the browser See `doil instances: --help` for more information @@ -180,6 +181,8 @@ Following commands come with the `--global` flag: * `doil instances:path` * `doil instances:login` * `doil instances:exec` +* `doil instances:csp` +* `doil instance:set-update-token` **`doil repo`** * `doil repo:add` diff --git a/app/src/Commands/Instances/ApplyCommand.php b/app/src/Commands/Instances/ApplyCommand.php index b3454cd1..1b58862f 100644 --- a/app/src/Commands/Instances/ApplyCommand.php +++ b/app/src/Commands/Instances/ApplyCommand.php @@ -33,7 +33,9 @@ class ApplyCommand extends Command "reactor", "change-roundcube-password", "nodejs", - "proxy-enable-https" + "proxy-enable-https", + "ilias-update-hook", + "set-update-token" ]; protected static $defaultName = "instances:apply"; diff --git a/app/src/Commands/Instances/CreateCommand.php b/app/src/Commands/Instances/CreateCommand.php index 34119394..8a1fc69a 100644 --- a/app/src/Commands/Instances/CreateCommand.php +++ b/app/src/Commands/Instances/CreateCommand.php @@ -291,10 +291,14 @@ public function execute(InputInterface $input, OutputInterface $output) : int $cron_password = $this->generatePassword(16); } $host = explode("=", $this->filesystem->getLineInFile("/etc/doil/doil.conf", "host")); + $this->docker->setGrain($instance_salt_name, "mpass", "$mysql_password"); sleep(1); $this->docker->setGrain($instance_salt_name, "cpass", "$cron_password"); sleep(1); + $update_token = explode("=", $this->filesystem->getLineInFile("/etc/doil/doil.conf", "update_token")); + $this->docker->setGrain($instance_salt_name, "update_token", "${update_token[1]}"); + sleep(1); $this->docker->setGrain($instance_salt_name, "doil_domain", "http://" . $host[1] . "/" . $options["name"]); sleep(1); $this->docker->setGrain($instance_salt_name, "doil_project_name", $options["name"]); @@ -358,6 +362,16 @@ public function execute(InputInterface $input, OutputInterface $output) : int $this->writer->endBlock(); } + // apply set-update-token state + $this->writer->beginBlock($output, "Apply set-update-token state"); + $this->docker->applyState($instance_salt_name, "set-update-token"); + $this->writer->endBlock(); + + // apply ilias-update-hook state + $this->writer->beginBlock($output, "Apply ilias-update-hook state"); + $this->docker->applyState($instance_salt_name, "ilias-update-hook"); + $this->writer->endBlock(); + // apply access state $this->writer->beginBlock($output, "Apply access state"); $this->docker->applyState($instance_salt_name, "access"); diff --git a/app/src/Commands/Instances/SetUpdateTokenCommand.php b/app/src/Commands/Instances/SetUpdateTokenCommand.php new file mode 100644 index 00000000..06850335 --- /dev/null +++ b/app/src/Commands/Instances/SetUpdateTokenCommand.php @@ -0,0 +1,156 @@ + - Extended GPL, see LICENSE */ + +namespace CaT\Doil\Commands\Instances; + +use CaT\Doil\Lib\Posix\Posix; +use CaT\Doil\Lib\Docker\Docker; +use CaT\Doil\Lib\ConsoleOutput\Writer; +use CaT\Doil\Lib\FileSystem\Filesystem; +use Symfony\Component\Console\Command\Command; +use Symfony\Component\Console\Input\InputOption; +use Symfony\Component\Console\Input\InputArgument; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Output\OutputInterface; +use Symfony\Component\Console\Question\ConfirmationQuestion; +use Symfony\Component\Console\Exception\InvalidArgumentException; + +class SetUpdateTokenCommand extends Command +{ + protected static $defaultName = "instances:set-update-token"; + protected static $defaultDescription = + "!NEEDS SUDO PRIVILEGES! This command sets a update token for all instances" + ; + + protected Docker $docker; + protected Posix $posix; + protected Filesystem $filesystem; + protected Writer $writer; + + public function __construct(Docker $docker, Posix $posix, Filesystem $filesystem, Writer $writer) + { + parent::__construct(); + + $this->docker = $docker; + $this->posix = $posix; + $this->filesystem = $filesystem; + $this->writer = $writer; + } + + public function configure() : void + { + $this + ->setAliases(["sut"]) + ->addArgument("instance", InputArgument::OPTIONAL, "Name of the instance to set update token for") + ->addOption("token", "t", InputOption::VALUE_REQUIRED, "Update token as string") + ->addOption("global", "g", InputOption::VALUE_NONE, "Determines if an instance is global or not") + ; + } + + public function execute(InputInterface $input, OutputInterface $output) : int + { + if (! $this->posix->isSudo()) { + $this->writer->error( + $output, + "Please execute this script as sudo user!" + ); + return Command::FAILURE; + } + + $token = $input->getOption("token"); + + $home_dir = $this->posix->getHomeDirectory($this->posix->getUserId()); + + $path = "/usr/local/share/doil/instances"; + $suffix = "global"; + if (! $input->getOption("global")) { + $path = "$home_dir/.doil/instances"; + $suffix = "local"; + } + + $instances = $this->filesystem->getFilesInPath($path); + if (count($instances) == 0) { + $this->writer->error( + $output, + "No instances found!", + "Use doil instances:ls --help for more information." + ); + return Command::FAILURE; + } + + $question = new ConfirmationQuestion( + "This will also update 'update_token' in your doil config. Want to continue? [yN]: ", + false + ); + + $helper = $this->getHelper("question"); + if (!$helper->ask($input, $output, $question)) { + $output->writeln("Abort by user!"); + return Command::FAILURE; + } + + $this->filesystem->replaceLineInFile("/etc/doil/doil.conf", "/update_token=.*/", "update_token=" . $token); + + foreach ($instances as $i) { + $started = $this->startInstance($output, $path, $i); + sleep(3); + $this->applyUpdateToken($output, $i . "." . $suffix, $token); + $this->docker->commit($i . "_" . $suffix); + if ($started) { + $this->stopInstance($output, $path, $i); + } + } + return Command::SUCCESS; + } + + protected function startInstance(OutputInterface $output, string $path, string $instance) : bool + { + if (! $this->hasDockerComposeFile($path . "/" . $instance, $output)) { + throw new InvalidArgumentException("Can't find a suitable docker-compose.yml file in $path/$instance"); + } + + if (! $this->docker->isInstanceUp($path . "/" . $instance)) { + $this->writer->beginBlock($output, "Start instance $instance"); + $this->docker->startContainerByDockerCompose($path . "/" . $instance); + $this->writer->endBlock(); + return true; + } + + return false; + } + + protected function stopInstance(OutputInterface $output, string $path, string $instance) : string + { + if ($this->docker->isInstanceUp($path . "/" . $instance)) { + $this->writer->beginBlock($output, "Stop instance $instance"); + $this->docker->stopContainerByDockerCompose($path . "/" . $instance); + $this->writer->endBlock(); + } + + return $instance; + } + + protected function hasDockerComposeFile(string $path, OutputInterface $output) : bool + { + if ($this->filesystem->exists($path . "/docker-compose.yml")) { + return true; + } + + $output->writeln("Error:"); + $output->writeln("\tCan't find a suitable docker-compose file in this directory '$path'."); + $output->writeln("\tIs this the right directory?"); + $output->writeln("\tSupported filenames: docker-compose.yml"); + + return false; + } + + protected function applyUpdateToken(OutputInterface $output, string $salt_key, string $token) + { + $this->writer->beginBlock($output, "Apply update token to $salt_key"); + $this->docker->setGrain($salt_key, "update_token", $token); + $this->docker->refreshGrains($salt_key); + $this->docker->applyState($salt_key, "set-update-token"); + $this->writer->endBlock(); + } +} \ No newline at end of file diff --git a/app/src/Commands/Pack/PackCreateCommand.php b/app/src/Commands/Pack/PackCreateCommand.php index c93b3cdc..a00350f4 100644 --- a/app/src/Commands/Pack/PackCreateCommand.php +++ b/app/src/Commands/Pack/PackCreateCommand.php @@ -292,9 +292,13 @@ public function execute(InputInterface $input, OutputInterface $output) : int $cron_password = $this->generatePassword(16); } $host = explode("=", $this->filesystem->getLineInFile("/etc/doil/doil.conf", "host")); - $this->docker->setGrain($instance_salt_name, "mpass", "${mysql_password}"); + + $this->docker->setGrain($instance_salt_name, "mpass", "$mysql_password"); + sleep(1); + $this->docker->setGrain($instance_salt_name, "cpass", "$cron_password"); sleep(1); - $this->docker->setGrain($instance_salt_name, "cpass", "${cron_password}"); + $update_token = explode("=", $this->filesystem->getLineInFile("/etc/doil/doil.conf", "update_token")); + $this->docker->setGrain($instance_salt_name, "update_token", "${update_token[1]}"); sleep(1); $doil_domain = "http://" . $host[1] . "/" . $options["name"]; $this->docker->setGrain($instance_salt_name, "doil_domain", "${doil_domain}"); @@ -341,9 +345,19 @@ public function execute(InputInterface $input, OutputInterface $output) : int $this->writer->endBlock(); } + // apply composer state $this->writer->beginBlock($output, "Apply composer state"); $this->docker->applyState($instance_salt_name, $this->getComposerVersion($ilias_version)); + + // apply set-update-token state + $this->writer->beginBlock($output, "Apply set-update-token state"); + $this->docker->applyState($instance_salt_name, "set-update-token"); + $this->writer->endBlock(); + + // apply ilias-update-hook state + $this->writer->beginBlock($output, "Apply ilias-update-hook state"); + $this->docker->applyState($instance_salt_name, "ilias-update-hook"); $this->writer->endBlock(); // apply access state diff --git a/app/src/cli.php b/app/src/cli.php index f53b5e19..a27053d4 100644 --- a/app/src/cli.php +++ b/app/src/cli.php @@ -44,6 +44,7 @@ function buildContainerForApp() : Container $c["command.instances.login"], $c["command.instances.path"], $c["command.instances.restart"], + $c["command.instances.set.update.token"], $c["command.instances.status"], $c["command.instances.up"], $c["command.mail.change.password"], @@ -234,6 +235,15 @@ function buildContainerForApp() : Container ); }; + $c["command.instances.set.update.token"] = function($c) { + return new Instances\SetUpdateTokenCommand( + $c["docker.shell"], + $c["posix.shell"], + $c["filesystem.shell"], + $c["command.writer"] + ); + }; + $c["command.instances.status"] = function($c) { return new Instances\StatusCommand( $c["docker.shell"] diff --git a/app/tests/Commands/Instances/CreateCommandTest.php b/app/tests/Commands/Instances/CreateCommandTest.php index a381f175..816f20fe 100644 --- a/app/tests/Commands/Instances/CreateCommandTest.php +++ b/app/tests/Commands/Instances/CreateCommandTest.php @@ -335,10 +335,10 @@ public function test_execute() : void ->willReturn(false, true, true) ; $filesystem - ->expects($this->once()) + ->expects($this->exactly(2)) ->method("getLineInFile") - ->with("/etc/doil/doil.conf", "host") - ->willReturnOnConsecutiveCalls("foo=doil", "7.8") + ->withConsecutive(["/etc/doil/doil.conf", "host"], ["/etc/doil/doil.conf", "update_token"]) + ->willReturnOnConsecutiveCalls("foo=doil", "update_token=foobar") ; $filesystem ->expects($this->once()) diff --git a/setup/conf/doil.conf b/setup/conf/doil.conf index dd185cc6..9844ec1f 100755 --- a/setup/conf/doil.conf +++ b/setup/conf/doil.conf @@ -1,4 +1,5 @@ group=doil host=doil mail_password=ilias -global_instances_path=/srv/instances \ No newline at end of file +global_instances_path=/srv/instances +update_token=foobar diff --git a/setup/stack/config/master.cnf b/setup/stack/config/master.cnf index b9bdd3bf..47ca0a2b 100755 --- a/setup/stack/config/master.cnf +++ b/setup/stack/config/master.cnf @@ -676,6 +676,8 @@ file_roots: - /srv/salt/states/php8.2 ilias: - /srv/salt/states/ilias + ilias-update-hook: + - srv/salt/states/ilias-update-hook compile-skins: - /srv/salt/states/compile-skins composer: @@ -706,6 +708,8 @@ file_roots: - /srv/salt/states/enable-https disable-https: - /srv/salt/states/disable-https + set-update-token: + - /srv/salt/states/set-update-token # The master_roots setting configures a master-only copy of the file_roots dictionary, diff --git a/setup/stack/states/ilias-update-hook/description.txt b/setup/stack/states/ilias-update-hook/description.txt new file mode 100644 index 00000000..783219a3 --- /dev/null +++ b/setup/stack/states/ilias-update-hook/description.txt @@ -0,0 +1 @@ +description = Inject an update hook file into docroot \ No newline at end of file diff --git a/setup/stack/states/ilias-update-hook/ilias-update-hook/init.sls b/setup/stack/states/ilias-update-hook/ilias-update-hook/init.sls new file mode 100644 index 00000000..dafe2ba2 --- /dev/null +++ b/setup/stack/states/ilias-update-hook/ilias-update-hook/init.sls @@ -0,0 +1,8 @@ +{% set ilias_version = salt['grains.get']('ilias_version', '8.0') %} + +/var/www/html/.update_hook.php: + file.managed: + - source: salt://ilias-update-hook/update_hook.php.j2 + - template: jinja + - context: + ilias_version: {{ ilias_version }} \ No newline at end of file diff --git a/setup/stack/states/ilias-update-hook/ilias-update-hook/update_hook.php.j2 b/setup/stack/states/ilias-update-hook/ilias-update-hook/update_hook.php.j2 new file mode 100644 index 00000000..2c71f23b --- /dev/null +++ b/setup/stack/states/ilias-update-hook/ilias-update-hook/update_hook.php.j2 @@ -0,0 +1,69 @@ + - Extended GPL, see LICENSE */ + +$request_headers = apache_request_headers(); +if (is_null($request_headers) || !array_key_exists("Authorization", $request_headers)) { + http_response_code(401); + throw new Exception("Missing Authorization header"); +} +$request_token = $request_headers["Authorization"]; + +$environment_token = getenv("UPDATE_TOKEN"); +if (is_null($environment_token)) { + http_response_code(401); + throw new Exception("Missing Environment Token"); +} + +if ($environment_token !== $request_token) { + http_response_code(401); + throw new Exception("Invalid Token"); +} + +// Pull current checked out branch from origin +$pull_output = []; +$result_code = 0; +$branch = exec("git branch --show-current"); +exec("git pull origin " . $branch . " 2>&1", $pull_output, $result_code); +if ($result_code !== 0) { + http_response_code(500); + throw new Exception("Failed to run update hook"); +} + +// Run composer install +$composer_output = []; +$result_code = 0; +exec("COMPOSER_HOME=/usr/local/bin/composer composer install 2>&1", $composer_output, $result_code); +if ($result_code !== 0) { + http_response_code(500); + throw new Exception("Failed to run update hook"); +} + +// Run ilias update +$update_output = []; +$result_code = 0; +{% if ilias_version | int < 10 %} +exec("php " . __DIR__ . "/setup/setup.php update -y 2>&1", $update_output, $result_code); +{% else %} +exec("php " . __DIR__ . "/cli/setup.php update -y 2>&1", $update_output, $result_code); +{% endif %} +if ($result_code !== 0) { + http_response_code(500); + throw new Exception("Failed to run update hook"); +} + +$output = ""; +foreach ($pull_output as $line) { + $output .= $line . "
"; +} + +foreach ($composer_output as $line) { + $output .= $line . "
"; +} + +foreach ($update_output as $line) { + $output .= $line . "
"; +} + +echo $output; + diff --git a/setup/stack/states/ilias-update-hook/top.sls b/setup/stack/states/ilias-update-hook/top.sls new file mode 100644 index 00000000..8a10ad74 --- /dev/null +++ b/setup/stack/states/ilias-update-hook/top.sls @@ -0,0 +1,3 @@ +ilias-update-hook: + '*': + - ilias-update-hook \ No newline at end of file diff --git a/setup/stack/states/proxyservices/proxyservices/service-config.tpl b/setup/stack/states/proxyservices/proxyservices/service-config.tpl index f8a0e04b..ab69b39e 100755 --- a/setup/stack/states/proxyservices/proxyservices/service-config.tpl +++ b/setup/stack/states/proxyservices/proxyservices/service-config.tpl @@ -7,4 +7,15 @@ location /%DOMAIN%/ { proxy_set_header X-Forwarded-Proto https; rewrite ^/%DOMAIN%/(.*) /%DOMAIN%/$1 break; +} + +location /%DOMAIN%/update/ { +proxy_pass http://%IP%/; +proxy_set_header Host $host; +proxy_set_header X-Real-IP $remote_addr; +proxy_set_header X-Forwarded-For $remote_addr; +proxy_pass_request_headers on; +proxy_set_header X-Forwarded-Proto https; + +rewrite ^/%DOMAIN%/(.*) /%DOMAIN%/.update_hook.php break; } \ No newline at end of file diff --git a/setup/stack/states/set-update-token/description.txt b/setup/stack/states/set-update-token/description.txt new file mode 100644 index 00000000..f17a1f48 --- /dev/null +++ b/setup/stack/states/set-update-token/description.txt @@ -0,0 +1 @@ +description = Set an update token for all instances \ No newline at end of file diff --git a/setup/stack/states/set-update-token/set-update-token/init.sls b/setup/stack/states/set-update-token/set-update-token/init.sls new file mode 100644 index 00000000..baca43de --- /dev/null +++ b/setup/stack/states/set-update-token/set-update-token/init.sls @@ -0,0 +1,7 @@ +{% set update_token = salt['grains.get']('update_token', '') %} + +/etc/apache2/envvars: + file.replace: + - pattern: '^export UPDATE_TOKEN=.*$' + - repl: 'export UPDATE_TOKEN={{ update_token }}' + - append_if_not_found: True \ No newline at end of file diff --git a/setup/stack/states/set-update-token/top.sls b/setup/stack/states/set-update-token/top.sls new file mode 100644 index 00000000..ecd86735 --- /dev/null +++ b/setup/stack/states/set-update-token/top.sls @@ -0,0 +1,3 @@ +set-update-token: + '*': + - set-update-token \ No newline at end of file diff --git a/setup/templates/minion/docker-compose.yml b/setup/templates/minion/docker-compose.yml index 6baf7040..deae4884 100755 --- a/setup/templates/minion/docker-compose.yml +++ b/setup/templates/minion/docker-compose.yml @@ -44,6 +44,9 @@ services: - type: bind source: ~/.ssh/ target: /root/.ssh + - type: bind + source: ~/.ssh/ + target: /var/www/.ssh - type: bind source: ./conf/salt-minion.conf target: /etc/supervisor/conf.d/salt-minion.conf diff --git a/setup/updates/update-20240617.sh b/setup/updates/update-20240617.sh new file mode 100644 index 00000000..b40fb763 --- /dev/null +++ b/setup/updates/update-20240617.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash + +doil_update_20240617() { + + if ! (grep -Fq "update_token" /etc/doil/doil.conf) + then + echo "update_token=foobar" >> /etc/doil/doil.conf + fi +exit + cp -r ${SCRIPT_DIR}/../app/src/* /usr/local/lib/doil/app/src/ + cp -r ${SCRIPT_DIR}/../setup/stack/states/* /usr/local/share/doil/stack/states/ + cp -r ${SCRIPT_DIR}/../setup/stack/config/* /usr/local/share/doil/stack/config/ + cp -r ${SCRIPT_DIR}/../setup/templates/minion/* /usr/local/share/doil/templates/minion/ + cp -r ${SCRIPT_DIR}/../setup/templates/proxy/conf/* /usr/local/lib/doil/server/proxy/conf/ + doil salt:restart + doil proxy:restart + sleep 10 + NAME=$(cat /etc/doil/doil.conf | grep "host" | cut -d '=' -f 2-) + sed -i "s/%TPL_SERVER_NAME%/${NAME}/g" "/usr/local/lib/doil/server/proxy/conf/nginx/local.conf" + docker exec -it doil_saltmain /bin/bash -c "salt \"doil.proxy\" state.highstate saltenv=proxyservices" &> /dev/null + docker commit doil_proxy doil_proxy:stable + + doil proxy:reload + + if [ $(docker ps -a --filter "name=_local" --filter "name=_global" --format "{{.Names}}" | wc -l) -gt 0 ] + doil_status_send_message "Applying patch on existing instances" + then + for INSTANCE in $(docker ps -a --filter "name=_local" --filter "name=_global" --format "{{.Names}}") + do + STARTED=0 + NAME=${INSTANCE%_*} + SUFFIX=${INSTANCE##*_} + + if [ -L /usr/local/share/doil/instances/"${NAME}" ] || [ -L /home/"${SUDO_USER}"/.doil/instances/"${NAME}" ] + then + if [ "${SUFFIX}" == "global" ] + then + INSTANCE_PATH=$(/usr/local/bin/doil path "${NAME}" -g -p) + GLOBAL="-g" + else + INSTANCE_PATH=$(su -c "/usr/local/bin/doil path ${NAME} -p" "${SUDO_USER}") + GLOBAL="" + fi + + if [[ ! $(docker ps --filter "name=_local" --filter "name=_global" --format "{{.Names}}") =~ ${INSTANCE} ]] + then + doil up ${NAME} ${GLOBAL} + STARTED=1 + fi + + DOCKER_COMPOSE_PATH="${INSTANCE_PATH}/docker-compose.yml" + + if ! grep -q "target: /var/www/.ssh" "${DOCKER_COMPOSE_PATH}" + then + sed -i 's/volumes:/&\n - type: bind\n source: ~\/.ssh\/\n target: \/var\/www\/.ssh/' "${DOCKER_COMPOSE_PATH}" + fi + sleep 5 + docker exec -it doil_saltmain /bin/bash -c "salt \"${NAME}.${SUFFIX}\" state.highstate saltenv=ilias-update-hook" &> /dev/null + if [ ${STARTED} == 1 ] + then + doil down ${NAME} ${GLOBAL} + else + doil restart ${NAME} ${GLOBAL} + fi + fi + done + + doil_status_okay + fi + + return $? +} \ No newline at end of file