diff --git a/porthos/README.md b/porthos/README.md index 32f5899..8a39bfa 100644 --- a/porthos/README.md +++ b/porthos/README.md @@ -68,11 +68,17 @@ repository mirrors for the given parameters. `true` - `arch` system architecture, like `x86_64` -HTTP repository metadata query (HTTP Basic authentication required): +HTTP repository metadata query (SSL + HTTP Basic authentication required): https://m1.nethserver.com/autoupdate////repodata/repomd.xml https://m1.nethserver.com/stable////repodata/repomd.xml +HTTP repository metadata query (username as path token, SSL not required): + + http://m1.nethserver.com/autoupdate/////repodata/repomd.xml + http://m1.nethserver.com/stable/////repodata/repomd.xml + + * `autoupdate` returns data from the tier associated to the credentials provided * `stable` always returns data from `t0` @@ -80,22 +86,61 @@ HTTP repository metadata query (HTTP Basic authentication required): * 401 - authorization required * 404 - resource not found -* 403 - bad credentials or `tier_id` is false (disabled) +* 403 - bad credentials or access denied conditions * 503 - redis connection failed, see `/var/log/nginx/porthos-php-error.log` * 502 - php-fpm connection failed, see `/var/log/nginx/error.log` * 500 - generic PHP error, see `/var/log/nginx/porthos-php-error.log` -## Redis DB format +## Porthos repository access control + +Access permissions to Porthos repositories are checked by `auth.php`. + +A special `subscription.json` API endpoint used by the Software center clients +is implemented by `subscription.php`. -The `repomd.php` script expects the following storage format in redis DB: +All HTTP requests to access YUM repositories must be authenticated with HTTP +Basic Auth. If the special `$config['legacy_auth']` is enabled, the HTTP +username is considered a valid access token, and can be set as password value too. + +The access token can be passed also as an URL path component, like: + + http://m1.nethserver.com/stable/////repodata/repomd.xml + +The `auth.php` and `subscription.php` scripts expects the following record +format in redis DB: key: - value: hash{ tier_id => , secret => } + value: hash{ tier_id => , secret => , icat => } -If `tier_id` is not set, the access is denied (403 - forbidden). For instance to create a key on athos +For instance to create a key on athos redis-cli -p PORT - redis-cli PORT> HMSET 0ILD29RH-D78A-C444-1F82-EE92-3211-FC47-43AD-DQFD tier_id 2 secret S3Cr3t + redis-cli PORT> HMSET 0ILD29RH-D78A-C444-1F82-EE92-3211-FC47-43AD-DQFD tier_id 2 secret S3Cr3t icat cat1,cat2,cat3 + +### `tier_id` field + +The `tier_id` value should be a number. If the value is negative, the tier +number is calculated by an hash function, based on the system identifier. + +If `tier_id` is not a number, both `auth.php` and `subscription.php` reply with +403 - forbidden. + +### `icat` field + +The `icat` field is a string of a comma separated list of YUM category +identifiers (refer to the repository comps/groups for valid names). Its purpose +is to show the products entitlement on the Software center page. It is used by +`subscription.php` to return the included/excluded YUM categories list to the +client. See also the `$config['categories']` parameter to configure it. This +field is ignored by `auth.php`. + +If `icat` field is not set, the `subscription.php` replies with 403 - forbidden. + +### `secret` field + +If `secret` field is not set, both `auth.php` and `subscription.php` reply with +403 - forbidden, unless `$config['legacy_auth']` is enabled. + ## Repository management commands @@ -104,10 +149,10 @@ from `/etc/porthos.conf`. Upstream YUM rsync URLs are defined there. - `repo-bulk-hinit` runs initial synchronization from upstream repositories (-f disables the check for already existing directories) - `repo-bulk-pull` creates a snapshot date-dir (e.g. `d20180301`) under - `dest_dir` with differences from upstream repositories. Set `t0` to point at + `dest_dir` with differences from upstream repositories. It sets `t0` to point at it. - `repo-bulk-shift [N]` updates `t1` ... `tN` links by shifting tiers of one position - the optional `N` + the optional `N` parameter creates missing links up to N - 1. - `repo-bulk-cleanup` erases stale tier snapshots The following commands should not be invoked directly. They are intended to be diff --git a/porthos/root/etc/nginx/conf.d/mirrorlist.conf b/porthos/root/etc/nginx/conf.d/mirrorlist.conf deleted file mode 100644 index 4545d87..0000000 --- a/porthos/root/etc/nginx/conf.d/mirrorlist.conf +++ /dev/null @@ -1,25 +0,0 @@ -server { - listen 80; - listen [::]:80; - - server_name mirrorlist.nethserver.com; - - # Uncomment the following line when TLS certificate is available - # include porthos-certbot.conf; - - access_log /var/log/nginx/mirrorlist.access.log main; - root /usr/share/nginx/html; - - location /.well-known/acme-challenge { - root /srv/porthos/certbot; - allow all; - } - - location / { - include fastcgi.conf; - fastcgi_pass unix:/var/run/porthos-fpm; - fastcgi_param SCRIPT_FILENAME /srv/porthos/script/mirrorlist.php; - fastcgi_param PORTHOS_REDIS /var/run/redis/athos.sock; - } - -} diff --git a/porthos/root/etc/nginx/conf.d/porthos.conf b/porthos/root/etc/nginx/conf.d/porthos.conf index aae0ca1..b3bb8bb 100644 --- a/porthos/root/etc/nginx/conf.d/porthos.conf +++ b/porthos/root/etc/nginx/conf.d/porthos.conf @@ -2,7 +2,7 @@ server { listen 80; listen [::]:80; - server_name ~^m[0-9]+.nethserver.com; + server_name porthos.nethserver.com; # Uncomment the following line when TLS certificate is available # include porthos-certbot.conf; @@ -38,27 +38,29 @@ server { return 200 "pong\n"; } - location ~ "^/t0/(.*)" { + location ~ "^/T0/(.*)" { internal; - try_files $uri /head/$1; + try_files /t0/$1 /head/$1; } - location ~ "^/t1/(.*)" { + location ~ "^/T1/(.*)" { internal; - try_files $uri /t0/$1; + try_files /t1/$1 /t0/$1 /head/$1; } - location ~ "^/t2/(.*)" { + location ~ "^/T2/(.*)" { internal; - try_files $uri /t1/$1; + try_files /t2/$1 /t1/$1 /t0/$1 /head/$1; } - location ~ "^/t3/(.*)" { + location ~ "^/T3/(.*)" { internal; - try_files $uri /t2/$1; + try_files /t3/$1 /t2/$1 /t1/$1 /t0/$1 /head/$1; } - location ~ "^/(d[0-9]{8}|head)(/.*)" { + # add another location for tier "Tx" here... + + location ~ "^/(t[0-9]|d[0-9]{8}|head)(/.*)" { internal; } @@ -66,8 +68,35 @@ server { include fastcgi.conf; fastcgi_pass unix:/var/run/porthos-fpm; fastcgi_param SCRIPT_FILENAME /srv/porthos/script/auth.php; + fastcgi_param PORTHOS_SITE porthos; fastcgi_param PORTHOS_REDIS /var/run/redis/athos.sock; - fastcgi_param DOCUMENT_URI /$2; + fastcgi_param DOCUMENT_URI $uri; + } + +} + +server { + listen 80; + listen [::]:80; + + server_name porthos-mirrorlist.nethserver.com; + + # Uncomment the following line when TLS certificate is available + # include porthos-certbot.conf; + + access_log /var/log/nginx/mirrorlist.access.log main; + root /usr/share/nginx/html; + + location /.well-known/acme-challenge { + root /srv/porthos/certbot; + allow all; } -} \ No newline at end of file + location / { + include fastcgi.conf; + fastcgi_pass unix:/var/run/porthos-fpm; + fastcgi_param SCRIPT_FILENAME /srv/porthos/script/mirrorlist.php; + fastcgi_param PORTHOS_SITE porthos; + fastcgi_param PORTHOS_REDIS /var/run/redis/athos.sock; + } +} diff --git a/porthos/root/etc/redis-athos.conf b/porthos/root/etc/redis-athos.conf index 171c224..28a8f57 100644 --- a/porthos/root/etc/redis-athos.conf +++ b/porthos/root/etc/redis-athos.conf @@ -5,7 +5,6 @@ # do not listen on TCP sockets port 0 -unixsocket /var/run/redis/athos.sock unixsocketperm 660 # connect athos from local stunnel endpoint diff --git a/porthos/root/etc/systemd/system/redis@.service b/porthos/root/etc/systemd/system/redis@.service index c86a4f1..288f7e2 100644 --- a/porthos/root/etc/systemd/system/redis@.service +++ b/porthos/root/etc/systemd/system/redis@.service @@ -6,12 +6,14 @@ ConditionPathExists=/etc/redis-%i.conf [Service] Type=notify -ExecStart=/usr/bin/redis-server /etc/redis-%i.conf --daemonize no -ExecStop=/usr/bin/redis-cli -s /var/run/redis/%i.sock shutdown +Environment="UNIXSOCK=%t/redis-%i/server.sock" +ExecStartPre=/usr/bin/chcon -L -u system_u -t redis_var_run_t %t/redis-%i +ExecStart=/usr/bin/redis-server /etc/redis-%i.conf --unixsocket $UNIXSOCK --loglevel debug --daemonize no +ExecStop=/usr/bin/redis-cli -s $UNIXSOCK shutdown User=redis Group=nginx -RuntimeDirectory=redis -RuntimeDirectoryMode=0755 +RuntimeDirectory=redis-%i +RuntimeDirectoryMode=0750 LimitNOFILE=10240 [Install] diff --git a/porthos/root/srv/porthos/script/auth.php b/porthos/root/srv/porthos/script/auth.php index 11ab3c6..c35568f 100644 --- a/porthos/root/srv/porthos/script/auth.php +++ b/porthos/root/srv/porthos/script/auth.php @@ -1,27 +1,103 @@ 'porthos-' . $_SERVER['PORTHOS_SITE'], + 'connection' => $_SERVER['CONNECTION'] ?: '', + 'msg_type' => 'repomdxml-auth', + 'msg_severity' => 'notice', + 'server_id' => $_SERVER['PHP_AUTH_USER'], + 'repo' => $uri['repo'], + 'version' => $uri['version'], + 'arch' => $uri['arch'], + 'tier_id' => $uri['prefix'] == 'autoupdate' ? NULL : $tier_id, + 'tier_auto' => isset($hash), + 'tls' => isset($_SERVER['HTTPS']), + ))); +} + +if($uri['prefix'] == 'autoupdate') { + return_file('/T' . $tier_id . $uri['full_path']); +} else { + return_file('/head' . $uri['full_path']); +} diff --git a/porthos/root/srv/porthos/script/config-porthos.php b/porthos/root/srv/porthos/script/config-porthos.php new file mode 100644 index 0000000..abc6a17 --- /dev/null +++ b/porthos/root/srv/porthos/script/config-porthos.php @@ -0,0 +1,54 @@ +connect($_SERVER['PORTHOS_REDIS'])) { - error_log(sprintf('[ERROR] redis connect(%s) failed!', $_SERVER['PORTHOS_REDIS'])); + application_log(json_encode(array( + 'application' => 'porthos-' . $_SERVER['PORTHOS_SITE'], + 'connection' => $_SERVER['CONNECTION'] ?: '', + 'msg_type' => 'redis-connect', + 'msg_severity' => 'alert', + )), LOG_ALERT); exit_http(503); }; - $descriptor = $redis->hMGet($system_id, array('tier_id', 'secret')); + $descriptor = $redis->hMGet($system_id, array('tier_id', 'secret', 'icat')); $redis->close(); return $descriptor; } + +function parse_uri($uri) { + $matches = array(); + $parts = array( + 'uri' => $uri, + 'system_id' => NULL, + 'full_path' => $uri, + 'prefix' => NULL, + 'version' => NULL, + 'repo' => NULL, + 'arch' => NULL, + 'rest' => NULL, + ); + preg_match('#^/(?Pautoupdate|stable)(?:/(?P[\w-]{36,48}))?(?P/(?P[\d\.]+)/(?P[\w-]+)/(?P\w+)/(?P.*))$#', $uri, $matches); + $parts = array_merge($parts, $matches); + return $parts; +} diff --git a/porthos/root/srv/porthos/script/mirrorlist.php b/porthos/root/srv/porthos/script/mirrorlist.php index 8d5a46b..95d36ea 100644 --- a/porthos/root/srv/porthos/script/mirrorlist.php +++ b/porthos/root/srv/porthos/script/mirrorlist.php @@ -1,54 +1,56 @@ 1, + 'include_categories' => $include_categories, + 'exclude_categories' => $exclude_categories, +)); diff --git a/porthos/root/usr/local/bin/repo-bulk-pull b/porthos/root/usr/local/bin/repo-bulk-pull index fbe9c64..3137c03 100755 --- a/porthos/root/usr/local/bin/repo-bulk-pull +++ b/porthos/root/usr/local/bin/repo-bulk-pull @@ -24,8 +24,10 @@ date_dir=d$(date +%Y%m%d) +trap 'kill $(jobs -p)' INT HUP TERM + for repo in ${!repos[@]}; do - /usr/local/bin/repo-tier-pull ${repo} ${date_dir} & + /usr/local/bin/repo-tier-pull $* ${repo} ${date_dir} & done wait diff --git a/porthos/root/usr/local/bin/repo-head-init b/porthos/root/usr/local/bin/repo-head-init index 9228596..1125739 100755 --- a/porthos/root/usr/local/bin/repo-head-init +++ b/porthos/root/usr/local/bin/repo-head-init @@ -57,4 +57,5 @@ if [[ -z ${force} && -d ${head_dir} ]]; then fi mkdir -p ${head_dir} -exec /usr/local/bin/xrsync ${repos[$repo_id]}/ ${head_dir} --delete-after +set -f +exec /usr/local/bin/xrsync ${repos[$repo_id]}/ ${head_dir} --delete-after ${options[${repo_id}]} diff --git a/porthos/root/usr/local/bin/repo-tier-pull b/porthos/root/usr/local/bin/repo-tier-pull index 52896de..d7384f6 100755 --- a/porthos/root/usr/local/bin/repo-tier-pull +++ b/porthos/root/usr/local/bin/repo-tier-pull @@ -24,16 +24,35 @@ set -e . /etc/porthos/repos.conf +function exit_help () +{ + echo -e "Usage:\n $(basename $0) [-f] REPO_ID DATE_DIR" 1>&2 + exit 1 +} + +while getopts ":f" opt; do + case $opt in + f) + force=1 + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit_help + ;; + esac +done + +shift "$((OPTIND-1))" + repo_id=$1 date_dir=$2 + if [[ -z ${repo_id} || -z "${repos[$repo_id]}" || -z ${date_dir} ]]; then - echo -e "Usage:\n $(basename $0) REPO_ID DATE_DIR" 1>&2 - exit 1 + exit_help fi - head_dir=${dest_dir}/head/${repo_id} -backup_dir=${dest_dir}/${date_dir}/$repo_id +backup_dir=${dest_dir}/${date_dir}/${repo_id} repomd_file=${head_dir}/repodata/repomd.xml if [[ ! -f "${repomd_file}" ]]; then @@ -42,10 +61,16 @@ if [[ ! -f "${repomd_file}" ]]; then fi if [[ -d ${backup_dir} ]]; then - echo "[WARNING] the repository backup directory already exists. Quit now ${backup_dir}" - exit 0 + if [[ -n ${force} ]]; then + echo "[WARNING] the already existing repository backup directory ${date_dir}/${repo_id} has been removed" + rm -rf ${backup_dir} + else + echo "[WARNING] the repository backup directory already exists. Quit now ${backup_dir}" + exit 0 + fi fi mkdir -p ${backup_dir} touch ${dest_dir}/head +set -f exec /usr/local/bin/xrsync ${repos[$repo_id]}/ ${head_dir} --delete-after --backup --backup-dir=${backup_dir} ${options[$repo_id]} diff --git a/porthos/root/usr/local/bin/xrsync b/porthos/root/usr/local/bin/xrsync index 8c3b7cd..7a51068 100755 --- a/porthos/root/usr/local/bin/xrsync +++ b/porthos/root/usr/local/bin/xrsync @@ -24,13 +24,20 @@ attempts=3 src=$1 ; shift dest=$1 ; shift - + for ((I=1; I < attempts; I++)); do rsync -aqSH --no-super --no-g --no-o "${@}" "${src}" "${dest}" - if [[ $? == 0 ]]; then + ret_rsync=$? + if [[ $ret_rsync == 0 ]]; then exit 0 + elif [[ $ret_rsync == 1 ]]; then + echo "[ERROR] rsync bad syntax" + exit 1 + elif [[ $ret_rsync == 20 ]]; then + echo "[ERROR] rsync signal caught" + exit 2 fi - echo "[WARNING] rsync from $src failed; attempt $I of ${attempts}..." 1>&2 + echo "[WARNING] rsync from $src failed with code $ret_rsync; attempt $I of ${attempts}..." 1>&2 ((I < attempts)) && sleep $(($RANDOM % 30)) done