diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 6e14a32fb..1fe9340e2 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -8,7 +8,7 @@ on: jobs: tests: - runs-on: ubuntu-18.04 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v1 diff --git a/delivery.yaml b/delivery.yaml index 162c0fada..b26afc2ba 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -8,6 +8,8 @@ pipeline: IMAGE: registry-write.opensource.zalan.do/acid/spilo PGVERSION: 14 type: script + vm_config: + type: linux commands: - desc: Build spilo docker image cmd: | diff --git a/postgres-appliance/launch.sh b/postgres-appliance/launch.sh index 7e7b937ea..a9323cf87 100755 --- a/postgres-appliance/launch.sh +++ b/postgres-appliance/launch.sh @@ -11,7 +11,7 @@ if [ -f /a.tar.xz ]; then fi fi -if [ "x$1" = "xinit" ]; then +if [ "$1" = "init" ]; then exec /usr/bin/dumb-init -c --rewrite 1:0 -- /bin/sh /launch.sh fi diff --git a/postgres-appliance/scripts/basebackup.sh b/postgres-appliance/scripts/basebackup.sh index cf115ba86..7c8fc68dc 100755 --- a/postgres-appliance/scripts/basebackup.sh +++ b/postgres-appliance/scripts/basebackup.sh @@ -21,13 +21,14 @@ done if which pg_receivewal &> /dev/null; then PG_RECEIVEWAL=pg_receivewal - PG_BASEBACKUP_OPTS="-X none" + PG_BASEBACKUP_OPTS=(-X none) else PG_RECEIVEWAL=pg_receivexlog - PG_BASEBACKUP_OPTS="" + PG_BASEBACKUP_OPTS=() fi -readonly WAL_FAST=$(dirname "$DATA_DIR")/wal_fast +WAL_FAST=$(dirname "$DATA_DIR")/wal_fast +readonly WAL_FAST mkdir -p "$WAL_FAST" rm -fr "$DATA_DIR" "${WAL_FAST:?}"/* @@ -96,7 +97,7 @@ fi ATTEMPT=0 while [[ $((ATTEMPT++)) -le $RETRIES ]]; do - pg_basebackup --pgdata="${DATA_DIR}" ${PG_BASEBACKUP_OPTS} --dbname="${CONNSTR}" & + pg_basebackup --pgdata="${DATA_DIR}" "${PG_BASEBACKUP_OPTS[@]}" --dbname="${CONNSTR}" & basebackup_pid=$! wait $basebackup_pid EXITCODE=$? @@ -108,5 +109,5 @@ while [[ $((ATTEMPT++)) -le $RETRIES ]]; do fi done -[[ $EXITCODE != 0 && ! -z $receivewal_pid ]] && kill "$receivewal_pid" +[[ $EXITCODE != 0 && -n $receivewal_pid ]] && kill "$receivewal_pid" exit $EXITCODE diff --git a/postgres-appliance/scripts/patroni_wait.sh b/postgres-appliance/scripts/patroni_wait.sh index 30d2497e7..79a0be650 100755 --- a/postgres-appliance/scripts/patroni_wait.sh +++ b/postgres-appliance/scripts/patroni_wait.sh @@ -60,10 +60,10 @@ do done if [ $# -gt 0 ]; then - [ ! -z "$TIMEOUT" ] && CUTOFF=$(($(date +%s)+TIMEOUT)) + [ -n "$TIMEOUT" ] && CUTOFF=$(($(date +%s)+TIMEOUT)) while [ "$(curl -so /dev/null -w '%{http_code}' "http://localhost:8008/$ROLE")" != "200" ]; do - [ ! -z "$TIMEOUT" ] && [ $CUTOFF -le "$(date +%s)" ] && exit 2 + [ -n "$TIMEOUT" ] && [ $CUTOFF -le "$(date +%s)" ] && exit 2 sleep "$INTERVAL" done diff --git a/postgres-appliance/scripts/post_init.sh b/postgres-appliance/scripts/post_init.sh index e62a541be..119d6eea7 100755 --- a/postgres-appliance/scripts/post_init.sh +++ b/postgres-appliance/scripts/post_init.sh @@ -176,14 +176,14 @@ while IFS= read -r db_name; do # In case if timescaledb binary is missing the first query fails with the error # ERROR: could not access file "$libdir/timescaledb-$OLD_VERSION": No such file or directory UPGRADE_TIMESCALEDB=$(echo -e "SELECT NULL;\nSELECT default_version != installed_version FROM pg_catalog.pg_available_extensions WHERE name = 'timescaledb'" | psql -tAX -d "${db_name}" 2> /dev/null | tail -n 1) - if [ "x$UPGRADE_TIMESCALEDB" = "xt" ]; then + if [ "$UPGRADE_TIMESCALEDB" = "t" ]; then echo "ALTER EXTENSION timescaledb UPDATE;" fi UPGRADE_POSTGIS=$(echo -e "SELECT COUNT(*) FROM pg_catalog.pg_extension WHERE extname = 'postgis'" | psql -tAX -d "${db_name}" 2> /dev/null | tail -n 1) - if [ "x$UPGRADE_POSTGIS" = "x1" ]; then + if [ "$UPGRADE_POSTGIS" = "1" ]; then # public.postgis_lib_version() is available only if postgis extension is created UPGRADE_POSTGIS=$(echo -e "SELECT extversion != public.postgis_lib_version() FROM pg_catalog.pg_extension WHERE extname = 'postgis'" | psql -tAX -d "${db_name}" 2> /dev/null | tail -n 1) - if [ "x$UPGRADE_POSTGIS" = "xt" ]; then + if [ "$UPGRADE_POSTGIS" = "t" ]; then echo "ALTER EXTENSION postgis UPDATE;" echo "SELECT public.postgis_extensions_upgrade();" fi @@ -200,7 +200,7 @@ GRANT EXECUTE ON FUNCTION public.pg_stat_statements_reset($RESET_ARGS) TO admin; else echo "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO admin;" fi - if [ "x$ENABLE_PG_MON" = "xtrue" ] && [ "$PGVER" -ge 11 ]; then echo "CREATE EXTENSION IF NOT EXISTS pg_mon SCHEMA public;"; fi + if [ "$ENABLE_PG_MON" = "true" ] && [ "$PGVER" -ge 11 ]; then echo "CREATE EXTENSION IF NOT EXISTS pg_mon SCHEMA public;"; fi cat metric_helpers.sql done < <(psql -d "$2" -tAc 'select pg_catalog.quote_ident(datname) from pg_catalog.pg_database where datallowconn') ) | PGOPTIONS="-c synchronous_commit=local" psql -Xd "$2" diff --git a/postgres-appliance/scripts/postgres_backup.sh b/postgres-appliance/scripts/postgres_backup.sh index be4be272c..ed72c8252 100755 --- a/postgres-appliance/scripts/postgres_backup.sh +++ b/postgres-appliance/scripts/postgres_backup.sh @@ -13,7 +13,8 @@ log "I was called as: $0 $*" readonly PGDATA=$1 DAYS_TO_RETAIN=$BACKUP_NUM_TO_RETAIN -readonly IN_RECOVERY=$(psql -tXqAc "select pg_is_in_recovery()") +IN_RECOVERY=$(psql -tXqAc "select pg_is_in_recovery()") +readonly IN_RECOVERY if [[ $IN_RECOVERY == "f" ]]; then [[ "$WALG_BACKUP_FROM_REPLICA" == "true" ]] && log "Cluster is not in recovery, not running backup" && exit 0 elif [[ $IN_RECOVERY == "t" ]]; then @@ -41,7 +42,8 @@ fi BEFORE="" LEFT=0 -readonly NOW=$(date +%s -u) +NOW=$(date +%s -u) +readonly NOW while read -r name last_modified rest; do last_modified=$(date +%s -ud "$last_modified") if [ $(((NOW-last_modified)/86400)) -ge $DAYS_TO_RETAIN ]; then @@ -56,7 +58,7 @@ while read -r name last_modified rest; do done < <($WAL_E backup-list 2> /dev/null | sed '0,/^name\s*\(last_\)\?modified\s*/d') # we want keep at least N backups even if the number of days exceeded -if [ ! -z "$BEFORE" ] && [ $LEFT -ge $DAYS_TO_RETAIN ]; then +if [ -n "$BEFORE" ] && [ $LEFT -ge $DAYS_TO_RETAIN ]; then if [[ "$USE_WALG_BACKUP" == "true" ]]; then $WAL_E delete before FIND_FULL "$BEFORE" --confirm else diff --git a/postgres-appliance/scripts/restore_command.sh b/postgres-appliance/scripts/restore_command.sh index 80bcd3e7b..a4bb88939 100755 --- a/postgres-appliance/scripts/restore_command.sh +++ b/postgres-appliance/scripts/restore_command.sh @@ -25,8 +25,10 @@ readonly wal_destination=$2 [[ -z $wal_filename || -z $wal_destination ]] && exit 1 -readonly wal_dir=$(dirname "$wal_destination") -readonly wal_fast_source=$(dirname "$(dirname "$(realpath "$wal_dir")")")/wal_fast/$wal_filename +wal_dir=$(dirname "$wal_destination") +readonly wal_dir +wal_fast_source=$(dirname "$(dirname "$(realpath "$wal_dir")")")/wal_fast/$wal_filename +readonly wal_fast_source [[ -f $wal_fast_source ]] && exec mv "${wal_fast_source}" "${wal_destination}" diff --git a/postgres-appliance/scripts/wal-e-wal-fetch.sh b/postgres-appliance/scripts/wal-e-wal-fetch.sh index 82524f36c..c5302aae7 100755 --- a/postgres-appliance/scripts/wal-e-wal-fetch.sh +++ b/postgres-appliance/scripts/wal-e-wal-fetch.sh @@ -71,12 +71,12 @@ else exit 1 fi -if [[ ! -z $WALE_S3_ENDPOINT && $WALE_S3_ENDPOINT =~ ^([a-z\+]{2,10}://)?([^:\/?]+) ]]; then +if [[ -n $WALE_S3_ENDPOINT && $WALE_S3_ENDPOINT =~ ^([a-z\+]{2,10}://)?([^:\/?]+) ]]; then S3_HOST=${BASH_REMATCH[2]} fi if [[ -z $AWS_REGION ]]; then - if [[ ! -z $WALE_S3_ENDPOINT && $WALE_S3_ENDPOINT =~ ^([a-z\+]{2,10}://)?s3-([^\.]+) ]]; then + if [[ -n $WALE_S3_ENDPOINT && $WALE_S3_ENDPOINT =~ ^([a-z\+]{2,10}://)?s3-([^\.]+) ]]; then AWS_REGION=${BASH_REMATCH[2]} elif [[ "$AWS_INSTANCE_PROFILE" == "true" ]]; then load_region_from_aws_instance_profile @@ -95,7 +95,8 @@ fi readonly SERVICE=s3 readonly REQUEST=aws4_request readonly HOST=$BUCKET.$S3_HOST -readonly TIME=$(date +%Y%m%dT%H%M%SZ) +TIME=$(date +%Y%m%dT%H%M%SZ) +readonly TIME readonly DATE=${TIME%T*} readonly DRSR="$DATE/$AWS_REGION/$SERVICE/$REQUEST" readonly EMPTYHASH=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 @@ -105,10 +106,14 @@ function hmac_sha256() { } # Four-step signing key calculation -readonly DATE_KEY=$(hmac_sha256 key:"AWS4$AWS_SECRET_ACCESS_KEY" "$DATE") -readonly DATE_REGION_KEY=$(hmac_sha256 "hexkey:$DATE_KEY" "$AWS_REGION") -readonly DATE_REGION_SERVICE_KEY=$(hmac_sha256 "hexkey:$DATE_REGION_KEY" "$SERVICE") -readonly SIGNING_KEY=$(hmac_sha256 "hexkey:$DATE_REGION_SERVICE_KEY" "$REQUEST") +DATE_KEY=$(hmac_sha256 key:"AWS4$AWS_SECRET_ACCESS_KEY" "$DATE") +readonly DATE_KEY +DATE_REGION_KEY=$(hmac_sha256 "hexkey:$DATE_KEY" "$AWS_REGION") +readonly DATE_REGION_KEY +DATE_REGION_SERVICE_KEY=$(hmac_sha256 "hexkey:$DATE_REGION_KEY" "$SERVICE") +readonly DATE_REGION_SERVICE_KEY +SIGNING_KEY=$(hmac_sha256 "hexkey:$DATE_REGION_SERVICE_KEY" "$REQUEST") +readonly SIGNING_KEY if [[ -z $AWS_INSTANCE_PROFILE ]]; then readonly SIGNED_HEADERS="host;x-amz-content-sha256;x-amz-date" @@ -156,14 +161,14 @@ function generate_next_segments() { function clear_except() { set +e - for dir in $PREFETCHDIR/running/0*; do + for dir in "$PREFETCHDIR"/running/0*; do item=$(basename "$dir") if [[ $item =~ ^[0-9A-F]{24}$ ]]; then [[ " ${PREFETCHES[*]} " =~ \ $item\ ]] || rm -fr "$dir" fi done - for file in $PREFETCHDIR/0*; do + for file in "$PREFETCHDIR"/0*; do item=$(basename "$file") if [[ $item =~ ^[0-9A-F]{24}$ ]]; then [[ " ${PREFETCHES[*]} " =~ \ $item\ ]] || rm -f "$file" @@ -182,9 +187,11 @@ function try_to_promote_prefetched() { echo "$$ $SEGMENT" -readonly PREFETCHDIR=$(dirname "$DESTINATION")/.wal-e/prefetch +PREFETCHDIR=$(dirname "$DESTINATION")/.wal-e/prefetch +readonly PREFETCHDIR if [[ $prefetch -gt 0 && $SEGMENT =~ ^[0-9A-F]{24}$ ]]; then - readonly PREFETCHES=($(generate_next_segments "$prefetch")) + mapfile -t PREFETCHES < <(generate_next_segments "$prefetch") + readonly PREFETCHES for segment in "${PREFETCHES[@]}"; do running="$PREFETCHDIR/running/$segment" [[ -d $running || -f $PREFETCHDIR/$segment ]] && continue diff --git a/postgres-appliance/scripts/wale_restore.sh b/postgres-appliance/scripts/wale_restore.sh index 26a4e3bf2..d5c8e6448 100755 --- a/postgres-appliance/scripts/wale_restore.sh +++ b/postgres-appliance/scripts/wale_restore.sh @@ -43,9 +43,9 @@ while true; do [[ -z $wal_segment_backup_start ]] && wal_segment_backup_start=$($WAL_E backup-list 2> /dev/null \ | sed '0,/^name\s*\(last_\)\?modified\s*/d' | sort -bk2 | tail -n1 | awk '{print $3;}' | sed 's/_.*$//') - [[ ! -z "$CONNSTR" && $server_version == "-1" ]] && server_version=$(psql -d "$CONNSTR" -tAc 'show server_version_num' 2> /dev/null || echo "-1") + [[ -n "$CONNSTR" && $server_version == "-1" ]] && server_version=$(psql -d "$CONNSTR" -tAc 'show server_version_num' 2> /dev/null || echo "-1") - [[ ! -z $wal_segment_backup_start && ( -z "$CONNSTR" || $server_version != "-1") ]] && break + [[ -n $wal_segment_backup_start && ( -z "$CONNSTR" || $server_version != "-1") ]] && break [[ $((ATTEMPT++)) -ge $RETRIES ]] && break sleep 1 done @@ -69,7 +69,7 @@ if [[ $server_version != "-1" ]]; then while true; do [[ -z $diff_in_bytes ]] && diff_in_bytes=$(psql -d "$CONNSTR" -tAc "$query") [[ -z $cluster_size ]] && cluster_size=$(psql -d "$CONNSTR" -tAc "SELECT SUM(pg_catalog.pg_database_size(datname)) FROM pg_catalog.pg_database") - [[ ! -z $diff_in_bytes && ! -z $cluster_size ]] && break + [[ -n $diff_in_bytes && -n $cluster_size ]] && break [[ $((ATTEMPT++)) -ge $RETRIES ]] && break sleep 1 done