From 0b88c3436c4bacc170c61eb17d662d85f7f84e98 Mon Sep 17 00:00:00 2001 From: Christoffer Lindahl Date: Sat, 3 Sep 2022 22:17:36 +0200 Subject: [PATCH 1/5] Copied changes from https://github.com/arteria/automysqlbackup/commit/d94d1a26421335a0ef3d4cd9a73e308f6c72f340 * Removes support for dumping flush logs. * Changes default for "usessl" config from "yes" to "no" and adds support for SSL options in MySQL 5.6 and higher. * Added support for "login paths" https://dev.mysql.com/doc/refman/5.6/en/option-file-options.html#option_general_login-path instead of setting password in config file. * Fixes "du" and "sed" commands failing in MacOS. * Also dump events to backup. * Removes some debug print statements. --- automysqlbackup | 142 ++++++++++++++++++++++++++----------------- automysqlbackup.conf | 12 +++- 2 files changed, 98 insertions(+), 56 deletions(-) diff --git a/automysqlbackup b/automysqlbackup index 9d56991..8585c23 100755 --- a/automysqlbackup +++ b/automysqlbackup @@ -65,7 +65,7 @@ load_default_config() { CONFIG_rotation_weekly=35 CONFIG_rotation_monthly=150 CONFIG_mysql_dump_port=3306 - CONFIG_mysql_dump_usessl='yes' + CONFIG_mysql_dump_usessl='no' CONFIG_mysql_dump_username='root' CONFIG_mysql_dump_password='' CONFIG_mysql_dump_host='localhost' @@ -81,9 +81,11 @@ load_default_config() { CONFIG_mysql_dump_single_transaction='no' CONFIG_mysql_dump_master_data= CONFIG_mysql_dump_full_schema='yes' - CONFIG_mysql_dump_flush_logs='no' CONFIG_mysql_dump_dbstatus='yes' CONFIG_mysql_dump_differential='no' + CONFIG_mysql_dump_login_path='automysqldump' + CONFIG_mysql_dump_login_path_file='' + CONFIG_mysql_dump_encrypted_login='no' CONFIG_backup_local_files=() CONFIG_db_names=() CONFIG_db_month_names=() @@ -98,6 +100,32 @@ load_default_config() { CONFIG_encrypt_password='password0123' } +# OS dependent options (showstoppers on MacOS) + if [[ $OSTYPE == darwin* ]]; then + OS_sed_option='-E' + OS_du_extra_option='' + else + OS_sed_option='-r' + OS_du_extra_option='--si' + fi + +mysql_commands() { + VERSION=`mysql -V | grep -oE "[0-9]+\.[0-9]+\.[0-9]+"` + NODOT_VER=`echo $VERSION | sed $OS_sed_option 's/\.//g'` + if [ "${CONFIG_mysql_dump_encrypted_login}" = "yes" ]; then + export MYSQLDUMP="mysqldump --login-path=$CONFIG_mysql_dump_login_path" + export MYSQLSHOW="mysqlshow --login-path=$CONFIG_mysql_dump_login_path" + export MYSQL="mysql --login-path=$CONFIG_mysql_dump_login_path" + if [ -n "${CONFIG_mysql_dump_login_path_file}" ]; then + export MYSQL_TEST_LOGIN_FILE=$CONFIG_mysql_dump_login_path_file + fi + else + export MYSQLDUMP="mysqldump --user=${CONFIG_mysql_dump_username} --password=${CONFIG_mysql_dump_password} --host=${CONFIG_mysql_dump_host}"; + export MYSQLSHOW="mysqlshow --user=${CONFIG_mysql_dump_username} --password=${CONFIG_mysql_dump_password} --host=${CONFIG_mysql_dump_host}"; + export MYSQL="mysql --user=${CONFIG_mysql_dump_username} --password=${CONFIG_mysql_dump_password} --host=${CONFIG_mysql_dump_host}"; + fi +} + # @return: true, if variable is set; else false isSet() { if [[ ! ${!1} && ${!1-_} ]]; then return 1; else return 0; fi @@ -438,7 +466,7 @@ backup_local_files () { # @deps: load_default_config parse_configuration () { # OPT string for use with mysqldump ( see man mysqldump ) - opt=( '--quote-names' '--opt' ) + opt=( '--quote-names' '--opt' '--events' ) # OPT string for use with mysql (see man mysql ) mysql_opt=() @@ -449,21 +477,26 @@ parse_configuration () { # OPT string for use with mysqlstatus opt_dbstatus=( '--status' ) - [[ "${CONFIG_mysql_dump_usessl}" = "yes" ]] && { - opt=( "${opt[@]}" '--ssl' ) - mysql_opt=( "${mysql_opt[@]}" '--ssl' ) - opt_fullschema=( "${opt_fullschema[@]}" '--ssl' ) - opt_dbstatus=( "${opt_dbstatus[@]}" '--ssl' ) - } + if [ "$NODOT_VER" -ge 56 ]; then + [[ "${CONFIG_mysql_dump_usessl}" = "yes" ]] && { + opt=( "${opt[@]}" '--ssl-mode=REQUIRED' ) + mysql_opt=( "${mysql_opt[@]}" '--ssl-mode=REQUIRED' ) + opt_fullschema=( "${opt_fullschema[@]}" '--ssl-mode=REQUIRED' ) + opt_dbstatus=( "${opt_dbstatus[@]}" '--ssl-mode=REQUIRED' ) + } + else + [[ "${CONFIG_mysql_dump_usessl}" = "yes" ]] && { + opt=( "${opt[@]}" '--ssl' ) + mysql_opt=( "${mysql_opt[@]}" '--ssl' ) + opt_fullschema=( "${opt_fullschema[@]}" '--ssl' ) + opt_dbstatus=( "${opt_dbstatus[@]}" '--ssl' ) + } + fi [[ "${CONFIG_mysql_dump_master_data}" ]] && (( ${CONFIG_mysql_dump_master_data} == 1 || ${CONFIG_mysql_dump_master_data} == 2 )) && { opt=( "${opt[@]}" "--master-data=${CONFIG_mysql_dump_master_data}" );} [[ "${CONFIG_mysql_dump_single_transaction}" = "yes" ]] && { opt=( "${opt[@]}" '--single-transaction' ) opt_fullschema=( "${opt_fullschema[@]}" '--single-transaction' ) } - [[ "${CONFIG_mysql_dump_flush_logs}" = "yes" ]] && { - opt=( "${opt[@]}" '--flush-logs' ) - } - [[ "${CONFIG_mysql_dump_commcomp}" = "yes" ]] && { opt=( "${opt[@]}" '--compress' ) opt_fullschema=( "${opt_fullschema[@]}" '--compress' ) @@ -519,7 +552,7 @@ parse_configuration () { db=${i%.*} table=${i#"$db".} r='\*'; [[ "$i" =~ $r ]] || { tmp[z++]="$i"; continue; } - while read -r; do tmp[z++]="${db}.${REPLY}"; done < <(mysql --user="${CONFIG_mysql_dump_username}" --password="${CONFIG_mysql_dump_password}" --host="${CONFIG_mysql_dump_host}" "${mysql_opt[@]}" --batch --skip-column-names -e "select table_name from information_schema.tables where table_schema='${db}' and table_name like '${table//\*/%}';") + while read -r; do tmp[z++]="${db}.${REPLY}"; done < <($MYSQL "${mysql_opt[@]}" --batch --skip-column-names -e "select table_name from information_schema.tables where table_schema='${db}' and table_name like '${table//\*/%}';") done for l in "${tmp[@]}"; do echo "exclude $l";done CONFIG_table_exclude=("${tmp[@]}") @@ -540,28 +573,28 @@ dbstatus() { if (( $CONFIG_dryrun )); then case "${CONFIG_mysql_dump_compression}" in 'gzip') - echo "dry-running: mysqlshow --user=${CONFIG_mysql_dump_username} --password=${CONFIG_mysql_dump_password} --host=${CONFIG_mysql_dump_host} ${opt_dbstatus[@]} | gzip_compression > ${1}${suffix}"; + echo "dry-running: $MYSQLSHOW ${opt_dbstatus[@]} | gzip_compression > ${1}${suffix}"; ;; 'bzip2') - echo "dry-running: mysqlshow --user=${CONFIG_mysql_dump_username} --password=${CONFIG_mysql_dump_password} --host=${CONFIG_mysql_dump_host} ${opt_dbstatus[@]} | bzip2_compression > ${1}${suffix}"; + echo "dry-running: $MYSQLSHOW ${opt_dbstatus[@]} | bzip2_compression > ${1}${suffix}"; ;; *) - echo "dry-running: mysqlshow --user=${CONFIG_mysql_dump_username} --password=${CONFIG_mysql_dump_password} --host=${CONFIG_mysql_dump_host} ${opt_dbstatus[@]} > ${1}${suffix}"; + echo "dry-running: $MYSQLSHOW ${opt_dbstatus[@]} > ${1}${suffix}"; ;; esac return 0; else case "${CONFIG_mysql_dump_compression}" in 'gzip') - mysqlshow --user="${CONFIG_mysql_dump_username}" --password="${CONFIG_mysql_dump_password}" --host="${CONFIG_mysql_dump_host}" "${opt_dbstatus[@]}" | gzip_compression > "${1}${suffix}"; + $MYSQLSHOW "${opt_dbstatus[@]}" | gzip_compression > "${1}${suffix}"; return $? ;; 'bzip2') - mysqlshow --user="${CONFIG_mysql_dump_username}" --password="${CONFIG_mysql_dump_password}" --host="${CONFIG_mysql_dump_host}" "${opt_dbstatus[@]}" | bzip2_compression > "${1}${suffix}"; + $MYSQLSHOW "${opt_dbstatus[@]}" | bzip2_compression > "${1}${suffix}"; return $? ;; *) - mysqlshow --user="${CONFIG_mysql_dump_username}" --password="${CONFIG_mysql_dump_password}" --host="${CONFIG_mysql_dump_host}" "${opt_dbstatus[@]}" > "${1}${suffix}"; + $MYSQLSHOW "${opt_dbstatus[@]}" > "${1}${suffix}"; return $? ;; esac @@ -576,28 +609,28 @@ fullschema () { if (( $CONFIG_dryrun )); then case "${CONFIG_mysql_dump_compression}" in 'gzip') - echo "dry-running: mysqldump --user=${CONFIG_mysql_dump_username} --password=${CONFIG_mysql_dump_password} --host=${CONFIG_mysql_dump_host} ${opt_fullschema[@]} | gzip_compression > ${1}${suffix}"; + echo "dry-running: $MYSQLDUMP ${opt_fullschema[@]} | gzip_compression > ${1}${suffix}"; ;; 'bzip2') - echo "dry-running: mysqldump --user=${CONFIG_mysql_dump_username} --password=${CONFIG_mysql_dump_password} --host=${CONFIG_mysql_dump_host} ${opt_fullschema[@]} | bzip2_compression > ${1}${suffix}"; + echo "dry-running: $MYSQLDUMP ${opt_fullschema[@]} | bzip2_compression > ${1}${suffix}"; ;; *) - echo "dry-running: mysqldump --user=${CONFIG_mysql_dump_username} --password=${CONFIG_mysql_dump_password} --host=${CONFIG_mysql_dump_host} ${opt_fullschema[@]} > ${1}${suffix}"; + echo "dry-running: $MYSQLDUMP ${opt_fullschema[@]} > ${1}${suffix}"; ;; esac return 0; else case "${CONFIG_mysql_dump_compression}" in 'gzip') - mysqldump --user="${CONFIG_mysql_dump_username}" --password="${CONFIG_mysql_dump_password}" --host="${CONFIG_mysql_dump_host}" "${opt_fullschema[@]}" | gzip_compression > "${1}${suffix}"; + $MYSQLDUMP "${opt_fullschema[@]}" | gzip_compression > "${1}${suffix}"; return $? ;; 'bzip2') - mysqldump --user="${CONFIG_mysql_dump_username}" --password="${CONFIG_mysql_dump_password}" --host="${CONFIG_mysql_dump_host}" "${opt_fullschema[@]}" | bzip2_compression > "${1}${suffix}"; + $MYSQLDUMP "${opt_fullschema[@]}" | bzip2_compression > "${1}${suffix}"; return $? ;; *) - mysqldump --user="${CONFIG_mysql_dump_username}" --password="${CONFIG_mysql_dump_password}" --host="${CONFIG_mysql_dump_host}" "${opt_fullschema[@]}" > "${1}${suffix}"; + $MYSQLDUMP "${opt_fullschema[@]}" > "${1}${suffix}"; return $? ;; esac @@ -715,13 +748,13 @@ process_dbs() { uid="${uid:-8:8}" case "${CONFIG_mysql_dump_compression}" in 'gzip') - mysqldump --user="${CONFIG_mysql_dump_username}" --password="${CONFIG_mysql_dump_password}" --host="${CONFIG_mysql_dump_host}" "${opt[@]}" "$@" | gzip_compression > "$cfname"; + $MYSQLDUMP "${opt[@]}" "$@" | gzip_compression > "$cfname"; ;; 'bzip2') - mysqldump --user="${CONFIG_mysql_dump_username}" --password="${CONFIG_mysql_dump_password}" --host="${CONFIG_mysql_dump_host}" "${opt[@]}" "$@" | bzip2_compression > "$cfname"; + $MYSQLDUMP "${opt[@]}" "$@" | bzip2_compression > "$cfname"; ;; *) - mysqldump --user="${CONFIG_mysql_dump_username}" --password="${CONFIG_mysql_dump_password}" --host="${CONFIG_mysql_dump_host}" "${opt[@]}" "$@" > "$cfname"; + $MYSQLDUMP "${opt[@]}" "$@" > "$cfname"; ;; esac add_manifest_entry "$manifest_file" "$cfname" "$pid" "$db" && parse_manifest "$manifest_file" && cp -al "$cfname" "${CONFIG_backup_dir}"/latest/ && echo "Generated master backup $cfname" && return 0 || return 1 @@ -734,29 +767,29 @@ process_dbs() { case "${CONFIG_mysql_dump_compression}" in 'gzip') if (( $filename_flags & $filename_flag_gz )); then - diff <(gzip_compression -dc "${manifest_latest_master_entry[0]}") <(mysqldump --user="${CONFIG_mysql_dump_username}" --password="${CONFIG_mysql_dump_password}" --host="${CONFIG_mysql_dump_host}" "${opt[@]}" "$@") | gzip_compression > "$cfname"; + diff <(gzip_compression -dc "${manifest_latest_master_entry[0]}") <($MYSQLDUMP "${opt[@]}" "$@") | gzip_compression > "$cfname"; elif (( $filename_flags & $filename_flag_bz2 )); then - diff <(bzip2_compression -dc "${manifest_latest_master_entry[0]}") <(mysqldump --user="${CONFIG_mysql_dump_username}" --password="${CONFIG_mysql_dump_password}" --host="${CONFIG_mysql_dump_host}" "${opt[@]}" "$@") | gzip_compression > "$cfname"; + diff <(bzip2_compression -dc "${manifest_latest_master_entry[0]}") <($MYSQLDUMP "${opt[@]}" "$@") | gzip_compression > "$cfname"; else - diff "${manifest_latest_master_entry[0]}" <(mysqldump --user="${CONFIG_mysql_dump_username}" --password="${CONFIG_mysql_dump_password}" --host="${CONFIG_mysql_dump_host}" "${opt[@]}" "$@") | gzip_compression > "$cfname"; + diff "${manifest_latest_master_entry[0]}" <($MYSQLDUMP "${opt[@]}" "$@") | gzip_compression > "$cfname"; fi ;; 'bzip2') if (( $filename_flags & $filename_flag_gz )); then - diff <(gzip_compression -dc "${manifest_latest_master_entry[0]}") <(mysqldump --user="${CONFIG_mysql_dump_username}" --password="${CONFIG_mysql_dump_password}" --host="${CONFIG_mysql_dump_host}" "${opt[@]}" "$@") | bzip2_compression > "$cfname"; + diff <(gzip_compression -dc "${manifest_latest_master_entry[0]}") <($MYSQLDUMP "${opt[@]}" "$@") | bzip2_compression > "$cfname"; elif (( $filename_flags & $filename_flag_bz2 )); then - diff <(bzip2_compression -dc "${manifest_latest_master_entry[0]}") <(mysqldump --user="${CONFIG_mysql_dump_username}" --password="${CONFIG_mysql_dump_password}" --host="${CONFIG_mysql_dump_host}" "${opt[@]}" "$@") | bzip2_compression > "$cfname"; + diff <(bzip2_compression -dc "${manifest_latest_master_entry[0]}") <($MYSQLDUMP "${opt[@]}" "$@") | bzip2_compression > "$cfname"; else - diff "${manifest_latest_master_entry[0]}" <(mysqldump --user="${CONFIG_mysql_dump_username}" --password="${CONFIG_mysql_dump_password}" --host="${CONFIG_mysql_dump_host}" "${opt[@]}" "$@") | bzip2_compression > "$cfname"; + diff "${manifest_latest_master_entry[0]}" <($MYSQLDUMP "${opt[@]}" "$@") | bzip2_compression > "$cfname"; fi ;; *) if (( $filename_flags & $filename_flag_gz )); then - diff <(gzip_compression -dc "${manifest_latest_master_entry[0]}") <(mysqldump --user="${CONFIG_mysql_dump_username}" --password="${CONFIG_mysql_dump_password}" --host="${CONFIG_mysql_dump_host}" "${opt[@]}" "$@") > "$cfname"; + diff <(gzip_compression -dc "${manifest_latest_master_entry[0]}") <($MYSQLDUMP "${opt[@]}" "$@") > "$cfname"; elif (( $filename_flags & $filename_flag_bz2 )); then - diff <(bzip2_compression -dc "${manifest_latest_master_entry[0]}") <(mysqldump --user="${CONFIG_mysql_dump_username}" --password="${CONFIG_mysql_dump_password}" --host="${CONFIG_mysql_dump_host}" "${opt[@]}" "$@") > "$cfname"; + diff <(bzip2_compression -dc "${manifest_latest_master_entry[0]}") <($MYSQLDUMP "${opt[@]}" "$@") > "$cfname"; else - diff "${manifest_latest_master_entry[0]}" <(mysqldump --user="${CONFIG_mysql_dump_username}" --password="${CONFIG_mysql_dump_password}" --host="${CONFIG_mysql_dump_host}" "${opt[@]}" "$@") > "$cfname"; + diff "${manifest_latest_master_entry[0]}" <($MYSQLDUMP "${opt[@]}" "$@") > "$cfname"; fi ;; esac @@ -770,28 +803,28 @@ process_dbs() { if (( $CONFIG_dryrun )); then case "${CONFIG_mysql_dump_compression}" in 'gzip') - echo "dry-running: mysqldump --user=${CONFIG_mysql_dump_username} --password=${CONFIG_mysql_dump_password} --host=${CONFIG_mysql_dump_host} ${opt[@]} $@ | gzip_compression > ${cfname}" + echo "dry-running: $MYSQLDUMP ${opt[@]} $@ | gzip_compression > ${cfname}" ;; 'bzip2') - echo "dry-running: mysqldump --user=${CONFIG_mysql_dump_username} --password=${CONFIG_mysql_dump_password} --host=${CONFIG_mysql_dump_host} ${opt[@]} $@ | bzip2_compression > ${cfname}" + echo "dry-running: $MYSQLDUMP ${opt[@]} $@ | bzip2_compression > ${cfname}" ;; *) - echo "dry-running: mysqldump --user=${CONFIG_mysql_dump_username} --password=${CONFIG_mysql_dump_password} --host=${CONFIG_mysql_dump_host} ${opt[@]} $@ > ${cfname}" + echo "dry-running: $MYSQLDUMP ${opt[@]} $@ > ${cfname}" ;; esac return 0; else case "${CONFIG_mysql_dump_compression}" in 'gzip') - mysqldump --user="${CONFIG_mysql_dump_username}" --password="${CONFIG_mysql_dump_password}" --host="${CONFIG_mysql_dump_host}" "${opt[@]}" "$@" | gzip_compression > "${cfname}" + $MYSQLDUMP "${opt[@]}" "$@" | gzip_compression > "${cfname}" ret=$? ;; 'bzip2') - mysqldump --user="${CONFIG_mysql_dump_username}" --password="${CONFIG_mysql_dump_password}" --host="${CONFIG_mysql_dump_host}" "${opt[@]}" "$@" | bzip2_compression > "${cfname}" + $MYSQLDUMP "${opt[@]}" "$@" | bzip2_compression > "${cfname}" ret=$? ;; *) - mysqldump --user="${CONFIG_mysql_dump_username}" --password="${CONFIG_mysql_dump_password}" --host="${CONFIG_mysql_dump_host}" "${opt[@]}" "$@" > "${cfname}" + $MYSQLDUMP "${opt[@]}" "$@" > "${cfname}" ret=$? ;; esac @@ -855,7 +888,6 @@ directory_checks_enable_logging () { [[ "${CONFIG_mysql_dump_dbstatus}" = 'yes' ]] && { checkdirs=( "${checkdirs[@]}" "${CONFIG_backup_dir}/status" ); } tmp_permcheck=0 - printf '# Checking for permissions to write to folders:\n' # "dirname ${CONFIG_backup_dir}" exists? @@ -878,14 +910,9 @@ directory_checks_enable_logging () { # -> check base folder - printf 'base folder %s ... ' "$(dirname "${CONFIG_backup_dir}")" if [[ -d "$(dirname "${CONFIG_backup_dir}")" ]]; then - printf 'exists ... ok.\n' - printf 'backup folder %s ... ' "${CONFIG_backup_dir}" - if [[ -d "${CONFIG_backup_dir}" ]]; then - printf 'exists ... writable? ' if (( $CONFIG_dryrun )); then printf 'dry-running. Skipping. Logging to /tmp\n' log_file="/tmp/${CONFIG_mysql_dump_host}-`date +%N`.log" @@ -894,7 +921,6 @@ directory_checks_enable_logging () { tmp_permcheck=1 else if chk_folder_writable "${CONFIG_backup_dir}"; then - printf 'yes. Proceeding.\n' log_file="${CONFIG_backup_dir}/${CONFIG_mysql_dump_host}-`date +%N`.log" log_errfile="${CONFIG_backup_dir}/ERRORS_${CONFIG_mysql_dump_host}-`date +%N`.log" activateIO "$log_file" "$log_errfile" @@ -1068,7 +1094,7 @@ parse_databases() { printf "# Parsing databases ... " # bash 3.0 local i;i=0; - while read -r; do alldbnames[i++]="$REPLY"; done < <(mysql --user="${CONFIG_mysql_dump_username}" --password="${CONFIG_mysql_dump_password}" --host="${CONFIG_mysql_dump_host}" "${mysql_opt[@]}" --batch --skip-column-names -e "show databases") + while read -r; do alldbnames[i++]="$REPLY"; done < <($MYSQL "${mysql_opt[@]}" --batch --skip-column-names -e "show databases") unset i # mkfifo foo || exit; trap 'rm -f foo' EXIT @@ -1648,9 +1674,12 @@ method_backup () { load_default_config trap mail_cleanup EXIT SIGHUP SIGINT SIGQUIT SIGTERM - if [[ -r "${CONFIG_configfile}" ]]; then source "${CONFIG_configfile}"; echo "Parsed config file \"${CONFIG_configfile}\""; else let "N |= $N_config_file_missing"; fi; echo + if [[ -r "${CONFIG_configfile}" ]]; then source "${CONFIG_configfile}"; else let "N |= $N_config_file_missing"; fi if (( $opt_flag_config_file )); then if [[ -r "${opt_config_file}" ]]; then source "${opt_config_file}"; let "N |= $N_arg_conffile_parsed"; else let "N |= $N_arg_conffile_unreadable"; fi; else let "N |= $N_too_many_args"; fi + # load mysql commands + mysql_commands + (( $CONFIG_dryrun )) && { echo "NOTE: We are dry-running. That means, that the script just shows you what it would do, if it were operating normally." echo "THE PRINTED COMMANDS CAN'T BE COPIED AND EXECUTED IF THERE ARE SPECIAL CHARACTERS, SPACES, ETC. IN THERE THAT WOULD NEED TO BE PROPERLY QUOTED IN ORDER TO WORK. THESE WERE CORRECTLY QUOTED FOR THE OUTPUT COMMAND, BUT CAN'T BE SEEN NOW." @@ -1974,7 +2003,7 @@ method_backup () { # -> finished information echo "Total disk space used for backup storage..." echo "Size - Location" - echo `du -hsH "${CONFIG_backup_dir}"` + echo `du -hs $OS_du_extra_option "${CONFIG_backup_dir}"` echo echo "======================================================================" # <- finished information @@ -2018,6 +2047,9 @@ method_list_manifest_entries () { if [[ -r "${CONFIG_configfile}" ]]; then source "${CONFIG_configfile}"; echo "Parsed config file \"${CONFIG_configfile}\""; else let "N |= $N_config_file_missing"; fi; echo if (( $opt_flag_config_file )); then if [[ -r "${opt_config_file}" ]]; then source "${opt_config_file}"; let "N |= $N_arg_conffile_parsed"; else let "N |= $N_arg_conffile_unreadable"; fi; else let "N |= $N_too_many_args"; fi + # load mysql commands + mysql_commands + export LC_ALL=C PROGNAME=`basename $0` PATH=${PATH}:/usr/local/bin:/usr/bin:/bin:/usr/local/mysql/bin @@ -2214,7 +2246,7 @@ NO_ARGS=0 E_OPTERROR=85 if (( $# == $NO_ARGS )); then # Script invoked with no command-line args? - echo "Invoking backup method."; echo; method_backup + method_backup fi while getopts ":c:blh" Option diff --git a/automysqlbackup.conf b/automysqlbackup.conf index 33812ff..44c5e5e 100644 --- a/automysqlbackup.conf +++ b/automysqlbackup.conf @@ -20,6 +20,16 @@ # Password to access the MySQL server e.g. password #CONFIG_mysql_dump_password='' +# If a "login path" should be used instead of username and password above to access the MySQL server +# Read more at https://dev.mysql.com/doc/refman/5.6/en/option-file-options.html#option_general_login-path +#CONFIG_mysql_dump_encrypted_login='no' + +# The "login path" +#CONFIG_mysql_dump_login_path='automysqldump' + +# The path to file where "login path" is stored +#CONFIG_mysql_dump_login_path_file='' + # Host name (or IP address) of MySQL server e.g localhost #CONFIG_mysql_dump_host='localhost' @@ -104,7 +114,7 @@ #CONFIG_mysql_dump_commcomp='no' # Use ssl encryption with mysqldump? -#CONFIG_mysql_dump_usessl='yes' +#CONFIG_mysql_dump_usessl='no' # For connections to localhost. Sometimes the Unix socket file must be specified. #CONFIG_mysql_dump_socket='' From 31cad9eda3a9d617bc8edb199ddd4656ba8133ab Mon Sep 17 00:00:00 2001 From: Christoffer Lindahl Date: Sat, 3 Sep 2022 22:23:57 +0200 Subject: [PATCH 2/5] Compatibility with MariaDB which uses "extra files" instead of "login paths". https://mariadb.com/kb/en/mysql_config_editor-compatibility/ --- automysqlbackup | 19 +++++++++++++------ automysqlbackup.conf | 4 ++++ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/automysqlbackup b/automysqlbackup index 8585c23..38f6323 100755 --- a/automysqlbackup +++ b/automysqlbackup @@ -84,7 +84,8 @@ load_default_config() { CONFIG_mysql_dump_dbstatus='yes' CONFIG_mysql_dump_differential='no' CONFIG_mysql_dump_login_path='automysqldump' - CONFIG_mysql_dump_login_path_file='' + CONFIG_mysql_dump_login_path_file='automysqldump' + CONFIG_mysql_dump_config_extra_file='' CONFIG_mysql_dump_encrypted_login='no' CONFIG_backup_local_files=() CONFIG_db_names=() @@ -113,11 +114,17 @@ mysql_commands() { VERSION=`mysql -V | grep -oE "[0-9]+\.[0-9]+\.[0-9]+"` NODOT_VER=`echo $VERSION | sed $OS_sed_option 's/\.//g'` if [ "${CONFIG_mysql_dump_encrypted_login}" = "yes" ]; then - export MYSQLDUMP="mysqldump --login-path=$CONFIG_mysql_dump_login_path" - export MYSQLSHOW="mysqlshow --login-path=$CONFIG_mysql_dump_login_path" - export MYSQL="mysql --login-path=$CONFIG_mysql_dump_login_path" - if [ -n "${CONFIG_mysql_dump_login_path_file}" ]; then - export MYSQL_TEST_LOGIN_FILE=$CONFIG_mysql_dump_login_path_file + if [ "${CONFIG_mysql_dump_config_extra_file}" != "" ]; then + export MYSQLDUMP="mysqldump --defaults-extra-file=$CONFIG_mysql_dump_config_extra_file" + export MYSQLSHOW="mysqlshow --defaults-extra-file=$CONFIG_mysql_dump_config_extra_file" + export MYSQL="mysql --defaults-extra-file=$CONFIG_mysql_dump_config_extra_file" + else + export MYSQLDUMP="mysqldump --login-path=$CONFIG_mysql_dump_login_path" + export MYSQLSHOW="mysqlshow --login-path=$CONFIG_mysql_dump_login_path" + export MYSQL="mysql --login-path=$CONFIG_mysql_dump_login_path" + if [ -n "${CONFIG_mysql_dump_login_path_file}" ]; then + export MYSQL_TEST_LOGIN_FILE=$CONFIG_mysql_dump_login_path_file + fi fi else export MYSQLDUMP="mysqldump --user=${CONFIG_mysql_dump_username} --password=${CONFIG_mysql_dump_password} --host=${CONFIG_mysql_dump_host}"; diff --git a/automysqlbackup.conf b/automysqlbackup.conf index 44c5e5e..d4fb50e 100644 --- a/automysqlbackup.conf +++ b/automysqlbackup.conf @@ -30,6 +30,10 @@ # The path to file where "login path" is stored #CONFIG_mysql_dump_login_path_file='' +# If connecting to a MariaDB server, use the following "extra file" instead of "login path" +# Read more at https://mariadb.com/kb/en/mysql_config_editor-compatibility/ +#CONFIG_mysql_dump_config_extra_file='automysqldump' + # Host name (or IP address) of MySQL server e.g localhost #CONFIG_mysql_dump_host='localhost' From 99e9263f7a19297a7123b17e07004b6e96190019 Mon Sep 17 00:00:00 2001 From: Christoffer Lindahl Date: Mon, 12 Sep 2022 21:01:20 +0200 Subject: [PATCH 3/5] Cleaned up trailing whitespace. --- automysqlbackup | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/automysqlbackup b/automysqlbackup index 38f6323..8610810 100755 --- a/automysqlbackup +++ b/automysqlbackup @@ -9,7 +9,7 @@ # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA @@ -536,7 +536,7 @@ parse_configuration () { else opt=( "${opt[@]}" '--databases' ) fi - + # if differential backup is active and the specified rotation is smaller than 21 days, set it to 21 days to ensure, that # master backups aren't deleted. if [[ "x$CONFIG_mysql_dump_differential" = "xyes" ]] && (( ${CONFIG_rotation_daily} < 21 )); then @@ -684,7 +684,7 @@ process_dbs() { manifest_file="${CONFIG_backup_dir}/${subfolder}${subsubfolder}/Manifest" fname="${CONFIG_backup_dir}/${subfolder}${subsubfolder}/${prefix}${name}_${datetimestamp}${midfix}${extension}" - (( $CONFIG_debug )) && echo "DEBUG: process_dbs >> Setting manifest file to: ${manifest_file}" + (( $CONFIG_debug )) && echo "DEBUG: process_dbs >> Setting manifest file to: ${manifest_file}" if (( $multipledbs )); then # multiple databases @@ -708,7 +708,7 @@ process_dbs() { # -> generate diff file let "filename_flags=0x00" - + # ## -> get latest differential manifest entry for specified db # if get_latest_manifest_entry_for_db "$db" 1; then # pid="${manifest_entry[2]}" @@ -801,7 +801,7 @@ process_dbs() { ;; esac add_manifest_entry "$manifest_file" "$cfname" "$pid" "$db" && parse_manifest "$manifest_file" && cp -al "$cfname" "${manifest_latest_master_entry[0]}" "${CONFIG_backup_dir}"/latest/ && echo "generated $cfname" && return 0 || return 1 - + fi # <- generate diff filename @@ -838,7 +838,7 @@ process_dbs() { fi fi - + if (( $ret == 0 )); then echo "Rotating $(( ${rotation}/${rotation_divisor} )) ${rotation_string} backups for ${name}" @@ -1059,7 +1059,7 @@ check_dependencies () { else [[ "x$CONFIG_mysql_dump_compression" = 'xbzip2' ]] && dependencies=("${dependencies[@]}" 'bzip2' ) [[ "x$CONFIG_mysql_dump_compression" = 'xgzip' ]] && dependencies=("${dependencies[@]}" 'gzip' ) - fi + fi if [[ "x$CONFIG_mailcontent" = 'xlog' || "x$CONFIG_mailcontent" = 'xquiet' ]]; then dependencies=( "${dependencies[@]}" 'mail' ) @@ -1239,7 +1239,7 @@ parse_manifest() { (( $CONFIG_debug )) && echo "Found multiple entries with the same filename. Removing all but the first-found entry from manifest." # remove all entries with this filename and add a new one based on the values of the item already in the array rm_manifest_entry_by_filename "$1" "${manifest_array[i]}" 1 && add_manifest_entry "$1" "${manifest_array[i]}" "${manifest_array[i+3]}" - continue 2 # the original entry, to which we compared, is already in the manifest_array; no matter if this is resolved or not, we don't + continue 2 # the original entry, to which we compared, is already in the manifest_array; no matter if this is resolved or not, we don't # need to add this entry to the manifest_array elif [[ "x${manifest_array[i+1]}" = "x${line_arr[2]}" ]]; then # found entry with different filename but same md5sum - file copied and renamed?! if [[ ! -s "${line_arr[0]}" ]]; then # empty file - don't start to compare md5sums ... @@ -1248,7 +1248,7 @@ parse_manifest() { (( $CONFIG_debug )) && echo "Found multiple entries with the same md5sum but different filename." (( $CONFIG_debug )) && echo -e ">> fname_manifest:\t${manifest_array[i]}\t${manifest_array[i+1]}\n>> fname_line:\t\t${line_arr[0]}\t${line_arr[2]}" if [[ "x${line_arr[6]}" != "x0" ]]; then - if [[ "x${manifest_array[i+3]}" = "x${line_arr[6]}" ]]; then # parent id is the same; TODO inform user of this predicament and suggest solution + if [[ "x${manifest_array[i+3]}" = "x${line_arr[6]}" ]]; then # parent id is the same; TODO inform user of this predicament and suggest solution manifest_entries_user_action_required=( "${manifest_entries_user_action_required[@]}" "$1" "${manifest_array[i]}" "The file has an identical copy with the same parent id. If you don't know why it exists, it is safe to remove it." ) continue 2 else @@ -1265,7 +1265,7 @@ parse_manifest() { manifest_array[(n-1)*${fields}+i/2]="${line_arr[i]}" #echo "manifest array key $((($n-1)*4+$i/2)) with value ${line_arr[i]}" done - + ((n++)) done < <(sort -t $'\t' -k"1" "$1") (( $CONFIG_debug )) && echo "<<<<<<< # manifest entries: $((${#manifest_array[@]}/$fields))" @@ -1407,7 +1407,7 @@ get_latest_manifest_entry_for_db() { } # @params: manifest_file filename/md5sum/id/rel_id [1(=don't parse manifest after finished)] -# if second parameters +# if second parameters # # lock manifest -> use awk, print all lines that don't have second parameter at the appropriate field -> unlock manifest # param3=0 -> parse manifest @@ -1509,7 +1509,7 @@ extended_select() { echo -e "$i) [ ]\t${list[i]}" fi done - + echo -e "$i)\tDONE" done_id=$i @@ -1660,7 +1660,7 @@ extended_select() { for i in "${selection[@]}"; do [[ "x$i" != "x$done_id" ]] && { extended_select_return=("${extended_select_return[@]}" "${list[i]}"); extended_select_return_id=("${extended_select_return_id[@]}" "$i"); } done - + } # END _functions @@ -1695,7 +1695,7 @@ method_backup () { export LC_ALL=C PROGNAME=`basename $0` - PATH=${PATH}:/usr/local/bin:/usr/bin:/bin:/usr/local/mysql/bin + PATH=${PATH}:/usr/local/bin:/usr/bin:/bin:/usr/local/mysql/bin version=3.0 fields=5 # manifest fields @@ -1741,7 +1741,7 @@ method_backup () { echo "======================================================================" echo "AutoMySQLBackup version ${version}" echo "http://sourceforge.net/projects/automysqlbackup/" - echo + echo echo "Backup of Database Server - ${CONFIG_mysql_dump_host_friendly:-$CONFIG_mysql_dump_host}" ( IFS=,; echo "Databases - ${CONFIG_db_names[*]}" ) ( IFS=,; echo "Databases (monthly) - ${CONFIG_db_month_names[*]}" ) @@ -1766,7 +1766,7 @@ method_backup () { echo "Backup local files. Doing this weekly on CONFIG_do_weekly." echo backup_local_files "${CONFIG_backup_dir}/backup_local_files/bcf_weekly_${datetimestamp}_${date_weekno}.tar" - tmp_flags=$?; var=; + tmp_flags=$?; var=; if (( $? == 0 )); then echo "success!" backupfiles=( "${backupfiles[@]}" "${CONFIG_backup_dir}/backup_local_files/bcf_weekly_${datetimestamp}_${date_weekno}.tar" ) @@ -1840,7 +1840,7 @@ method_backup () { echo echo "======================================================================" echo - + fi # <- dump full schema @@ -1904,7 +1904,7 @@ method_backup () { echo echo "======================================================================" echo - + fi # <- dump status @@ -2059,7 +2059,7 @@ method_list_manifest_entries () { export LC_ALL=C PROGNAME=`basename $0` - PATH=${PATH}:/usr/local/bin:/usr/bin:/bin:/usr/local/mysql/bin + PATH=${PATH}:/usr/local/bin:/usr/bin:/bin:/usr/local/mysql/bin version=3.0 fields=5 # manifest fields From 5f84fa5882c5b6c5106a425061a3f406d0fe9fe8 Mon Sep 17 00:00:00 2001 From: Christoffer Lindahl Date: Mon, 12 Sep 2022 21:32:25 +0200 Subject: [PATCH 4/5] Reformatted the code with "shfmt". --- automysqlbackup | 3279 ++++++++++++++++++++++++----------------------- 1 file changed, 1690 insertions(+), 1589 deletions(-) diff --git a/automysqlbackup b/automysqlbackup index 8610810..39818c7 100755 --- a/automysqlbackup +++ b/automysqlbackup @@ -90,7 +90,7 @@ load_default_config() { CONFIG_backup_local_files=() CONFIG_db_names=() CONFIG_db_month_names=() - CONFIG_db_exclude=( 'information_schema' ) + CONFIG_db_exclude=('information_schema') CONFIG_table_exclude=() CONFIG_mailcontent='stdout' CONFIG_mail_maxattsize=4000 @@ -102,17 +102,17 @@ load_default_config() { } # OS dependent options (showstoppers on MacOS) - if [[ $OSTYPE == darwin* ]]; then - OS_sed_option='-E' - OS_du_extra_option='' - else - OS_sed_option='-r' - OS_du_extra_option='--si' - fi +if [[ $OSTYPE == darwin* ]]; then + OS_sed_option='-E' + OS_du_extra_option='' +else + OS_sed_option='-r' + OS_du_extra_option='--si' +fi mysql_commands() { - VERSION=`mysql -V | grep -oE "[0-9]+\.[0-9]+\.[0-9]+"` - NODOT_VER=`echo $VERSION | sed $OS_sed_option 's/\.//g'` + VERSION=$(mysql -V | grep -oE "[0-9]+\.[0-9]+\.[0-9]+") + NODOT_VER=$(echo $VERSION | sed $OS_sed_option 's/\.//g') if [ "${CONFIG_mysql_dump_encrypted_login}" = "yes" ]; then if [ "${CONFIG_mysql_dump_config_extra_file}" != "" ]; then export MYSQLDUMP="mysqldump --defaults-extra-file=$CONFIG_mysql_dump_config_extra_file" @@ -127,9 +127,9 @@ mysql_commands() { fi fi else - export MYSQLDUMP="mysqldump --user=${CONFIG_mysql_dump_username} --password=${CONFIG_mysql_dump_password} --host=${CONFIG_mysql_dump_host}"; - export MYSQLSHOW="mysqlshow --user=${CONFIG_mysql_dump_username} --password=${CONFIG_mysql_dump_password} --host=${CONFIG_mysql_dump_host}"; - export MYSQL="mysql --user=${CONFIG_mysql_dump_username} --password=${CONFIG_mysql_dump_password} --host=${CONFIG_mysql_dump_host}"; + export MYSQLDUMP="mysqldump --user=${CONFIG_mysql_dump_username} --password=${CONFIG_mysql_dump_password} --host=${CONFIG_mysql_dump_host}" + export MYSQLSHOW="mysqlshow --user=${CONFIG_mysql_dump_username} --password=${CONFIG_mysql_dump_password} --host=${CONFIG_mysql_dump_host}" + export MYSQL="mysql --user=${CONFIG_mysql_dump_username} --password=${CONFIG_mysql_dump_password} --host=${CONFIG_mysql_dump_host}" fi } @@ -147,78 +147,81 @@ isEmpty() { # It removes the IO redirection, mails any log file information and cleans up any temporary files. # @args: (none) # @return: (none) -mail_cleanup () { +mail_cleanup() { removeIO # if the variables $log_file and $log_errfile aren't set or are empty and both associated files don't exist, skip output methods. # this might happen if 'exit' occurs before they are set. - if [[ ! -e "$log_file" && ! -e "$log_errfile" ]];then + if [[ ! -e "$log_file" && ! -e "$log_errfile" ]]; then echo "Skipping normal output methods, since the program exited before any log files could be created." else case "${CONFIG_mailcontent}" in - 'files') - # Include error log if larger than zero. - if [[ -s "$log_errfile" ]]; then - backupfiles=( "${backupfiles[@]}" "$log_errfile" ) - errornote="WARNING: Error Reported - " - fi - temp="$(mktemp "$CONFIG_backup_dir"/tmp/mail_content.XXXXXX)" - # Get backup size - attsize=`du -c "${backupfiles[@]}" | awk 'END {print $1}'` - if (( ${CONFIG_mail_maxattsize} >= ${attsize} )); then - if [[ "x$CONFIG_mail_use_uuencoded_attachments" = "xyes" ]]; then - cat "$log_file" > "$temp" - for j in "${backupfiles[@]}"; do - uuencode "$j" "$j" >> "$temp" - done - mail -s "${errornote} MySQL Backup Log and SQL Files for ${CONFIG_mysql_dump_host_friendly:-$CONFIG_mysql_dump_host} - ${datetimestamp}" ${CONFIG_mail_address} < "$temp" - else - mutt -s "${errornote} MySQL Backup Log and SQL Files for ${CONFIG_mysql_dump_host_friendly:-$CONFIG_mysql_dump_host} - ${datetimestamp}" -a "${backupfiles[@]}" -- ${CONFIG_mail_address} < "$log_file" - fi - elif (( ${CONFIG_mail_maxattsize} <= ${attsize} )) && [[ "x$CONFIG_mail_splitandtar" = "xyes" ]]; then - if sPWD="$PWD"; cd "$CONFIG_backup_dir"/tmp && pax -wv "${backupfiles[@]}" | bzip2_compression | split -b $((CONFIG_mail_maxattsize*1000)) - mail_attachment_${datetimestamp}_ && cd "$sPWD"; then - files=("$CONFIG_backup_dir"/tmp/mail_attachment_${datetimestamp}_*) - echo -e "\n\nThe attachments have been split into multiple files.\nUse 'cat mail_attachment_2011-08-13_13h15m_* > mail_attachment_2011-08-13_13h15m.tar.bz2' to combine them and \ + 'files') + # Include error log if larger than zero. + if [[ -s "$log_errfile" ]]; then + backupfiles=("${backupfiles[@]}" "$log_errfile") + errornote="WARNING: Error Reported - " + fi + temp="$(mktemp "$CONFIG_backup_dir"/tmp/mail_content.XXXXXX)" + # Get backup size + attsize=$(du -c "${backupfiles[@]}" | awk 'END {print $1}') + if ((${CONFIG_mail_maxattsize} >= ${attsize})); then + if [[ "x$CONFIG_mail_use_uuencoded_attachments" = "xyes" ]]; then + cat "$log_file" >"$temp" + for j in "${backupfiles[@]}"; do + uuencode "$j" "$j" >>"$temp" + done + mail -s "${errornote} MySQL Backup Log and SQL Files for ${CONFIG_mysql_dump_host_friendly:-$CONFIG_mysql_dump_host} - ${datetimestamp}" ${CONFIG_mail_address} <"$temp" + else + mutt -s "${errornote} MySQL Backup Log and SQL Files for ${CONFIG_mysql_dump_host_friendly:-$CONFIG_mysql_dump_host} - ${datetimestamp}" -a "${backupfiles[@]}" -- ${CONFIG_mail_address} <"$log_file" + fi + elif ((${CONFIG_mail_maxattsize} <= ${attsize})) && [[ "x$CONFIG_mail_splitandtar" = "xyes" ]]; then + if + sPWD="$PWD" + cd "$CONFIG_backup_dir"/tmp && pax -wv "${backupfiles[@]}" | bzip2_compression | split -b $((CONFIG_mail_maxattsize * 1000)) - mail_attachment_${datetimestamp}_ && cd "$sPWD" + then + files=("$CONFIG_backup_dir"/tmp/mail_attachment_${datetimestamp}_*) + echo -e "\n\nThe attachments have been split into multiple files.\nUse 'cat mail_attachment_2011-08-13_13h15m_* > mail_attachment_2011-08-13_13h15m.tar.bz2' to combine them and \ 'bunzip2 "$temp" - uuencode "$j" "$j" >> "$temp" - else - uuencode "$j" "$j" > "$temp" - fi - mail -s "${errornote} MySQL Backup Log and SQL Files for ${CONFIG_mysql_dump_host_friendly:-$CONFIG_mysql_dump_host} - ${datetimestamp}" ${CONFIG_mail_address} < "$temp" - else - mutt -s "${errornote} MySQL Backup Log and SQL Files for ${CONFIG_mysql_dump_host_friendly:-$CONFIG_mysql_dump_host} - ${datetimestamp}; Part $((j+1))/${#files[@]}" -a "${files[j]}" -- ${CONFIG_mail_address} < "$log_file" - fi - done - else - cat "$log_file" | mail -s "WARNING! - MySQL Backup exceeds set maximum attachment size on ${CONFIG_mysql_dump_host_friendly:-$CONFIG_mysql_dump_host} - ${datetimestamp}" ${CONFIG_mail_address} - fi - else - cat "$log_file" | mail -s "WARNING! - MySQL Backup exceeds set maximum attachment size on ${CONFIG_mysql_dump_host_friendly:-$CONFIG_mysql_dump_host} - ${datetimestamp}" ${CONFIG_mail_address} - fi - rm "$temp" - ;; - 'log') - cat "$log_file" | mail -s "MySQL Backup Log for ${CONFIG_mysql_dump_host_friendly:-$CONFIG_mysql_dump_host} - ${datetimestamp}" ${CONFIG_mail_address} - [[ -s "$log_errfile" ]] && cat "$log_errfile" | mail -s "ERRORS REPORTED: MySQL Backup error Log for ${CONFIG_mysql_dump_host_friendly:-$CONFIG_mysql_dump_host} - ${datetimestamp}" ${CONFIG_mail_address} - ;; - 'quiet') - [[ -s "$log_errfile" ]] && cat "$log_errfile" | mail -s "ERRORS REPORTED: MySQL Backup error Log for ${CONFIG_mysql_dump_host_friendly:-$CONFIG_mysql_dump_host} - ${datetimestamp}" ${CONFIG_mail_address} - ;; - *) - if [[ -s "$log_errfile" ]]; then - cat "$log_file" - echo - echo "###### WARNING ######" - echo "Errors reported during AutoMySQLBackup execution.. Backup failed" - echo "Error log below.." - cat "$log_errfile" - else - cat "$log_file" - fi - ;; + for ((j = 0; j < "${#files[@]}"; j++)); do + if [[ "x$CONFIG_mail_use_uuencoded_attachments" = "xyes" ]]; then + if (( $j = 0 )); then + cat "$log_file" >"$temp" + uuencode "$j" "$j" >>"$temp" + else + uuencode "$j" "$j" >"$temp" + fi + mail -s "${errornote} MySQL Backup Log and SQL Files for ${CONFIG_mysql_dump_host_friendly:-$CONFIG_mysql_dump_host} - ${datetimestamp}" ${CONFIG_mail_address} <"$temp" + else + mutt -s "${errornote} MySQL Backup Log and SQL Files for ${CONFIG_mysql_dump_host_friendly:-$CONFIG_mysql_dump_host} - ${datetimestamp}; Part $((j + 1))/${#files[@]}" -a "${files[j]}" -- ${CONFIG_mail_address} <"$log_file" + fi + done + else + cat "$log_file" | mail -s "WARNING! - MySQL Backup exceeds set maximum attachment size on ${CONFIG_mysql_dump_host_friendly:-$CONFIG_mysql_dump_host} - ${datetimestamp}" ${CONFIG_mail_address} + fi + else + cat "$log_file" | mail -s "WARNING! - MySQL Backup exceeds set maximum attachment size on ${CONFIG_mysql_dump_host_friendly:-$CONFIG_mysql_dump_host} - ${datetimestamp}" ${CONFIG_mail_address} + fi + rm "$temp" + ;; + 'log') + cat "$log_file" | mail -s "MySQL Backup Log for ${CONFIG_mysql_dump_host_friendly:-$CONFIG_mysql_dump_host} - ${datetimestamp}" ${CONFIG_mail_address} + [[ -s "$log_errfile" ]] && cat "$log_errfile" | mail -s "ERRORS REPORTED: MySQL Backup error Log for ${CONFIG_mysql_dump_host_friendly:-$CONFIG_mysql_dump_host} - ${datetimestamp}" ${CONFIG_mail_address} + ;; + 'quiet') + [[ -s "$log_errfile" ]] && cat "$log_errfile" | mail -s "ERRORS REPORTED: MySQL Backup error Log for ${CONFIG_mysql_dump_host_friendly:-$CONFIG_mysql_dump_host} - ${datetimestamp}" ${CONFIG_mail_address} + ;; + *) + if [[ -s "$log_errfile" ]]; then + cat "$log_file" + echo + echo "###### WARNING ######" + echo "Errors reported during AutoMySQLBackup execution.. Backup failed" + echo "Error log below.." + cat "$log_errfile" + else + cat "$log_file" + fi + ;; esac ################################################################################### # Clean up and finish @@ -230,9 +233,11 @@ mail_cleanup () { # @params: #month #year # @deps: (none) days_of_month() { - m="$1"; y="$2"; a=$(( 30+(m+m/8)%2 )) - (( m==2 )) && a=$((a-2)) - (( m==2 && y%4==0 && ( y<100 || y%100>0 || y%400==0) )) && a=$((a+1)) + m="$1" + y="$2" + a=$((30 + (m + m / 8) % 2)) + ((m == 2)) && a=$((a - 2)) + ((m == 2 && y % 4 == 0 && (y < 100 || y % 100 > 0 || y % 400 == 0))) && a=$((a + 1)) printf '%d' $a } @@ -240,9 +245,10 @@ days_of_month() { # @args: folder to test # @return: returns false if creation of temporary file failed or it can't be removed afterwards; else true # @deps: (none) -chk_folder_writable () { - local temp; temp="$(mktemp "$1"/tmp.XXXXXX)" - if (( $? == 0 )); then +chk_folder_writable() { + local temp + temp="$(mktemp "$1"/tmp.XXXXXX)" + if (($? == 0)); then rm "${temp}" || return 1 return 0 else @@ -255,12 +261,12 @@ bzip2_compression() { var=("$@") re='^[0-9]*$' if [[ "x$CONFIG_multicore" = 'xyes' ]]; then - if [[ "x$CONFIG_multicore_threads" != 'xauto' ]] && [[ "x$CONFIG_multicore_threads" =~ $re ]]; then - var=( "-p${CONFIG_multicore_threads}" "${var[@]}" ) - fi - pbzip2 "${var[@]}" + if [[ "x$CONFIG_multicore_threads" != 'xauto' ]] && [[ "x$CONFIG_multicore_threads" =~ $re ]]; then + var=("-p${CONFIG_multicore_threads}" "${var[@]}") + fi + pbzip2 "${var[@]}" else - bzip2 "${var[@]}" + bzip2 "${var[@]}" fi } @@ -269,12 +275,12 @@ gzip_compression() { var=("$@") re='^[0-9]*$' if [[ "x$CONFIG_multicore" = 'xyes' ]]; then - if [[ "x$CONFIG_multicore_threads" != 'xauto' ]] && [[ "x$CONFIG_multicore_threads" =~ $re ]]; then - var=( "-p${CONFIG_multicore_threads}" "${var[@]}" ) - fi - pigz "${var[@]}" + if [[ "x$CONFIG_multicore_threads" != 'xauto' ]] && [[ "x$CONFIG_multicore_threads" =~ $re ]]; then + var=("-p${CONFIG_multicore_threads}" "${var[@]}") + fi + pigz "${var[@]}" else - gzip "${var[@]}" + gzip "${var[@]}" fi } @@ -282,7 +288,7 @@ gzip_compression() { # @args: filename # @return: (none) # @deps: (none) -remove_datetimeinfo () { +remove_datetimeinfo() { mv "$1" "$(echo "$1" | sed -re 's/_[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}h[0-9]{2}m_(Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday|January|February|March|April|May|June|July|August|September|October|November|December|[0-9]{1,2})//g')" } export -f remove_datetimeinfo @@ -291,16 +297,16 @@ export -f remove_datetimeinfo # @args: (none) # @deps: days_of_month set_datetime_vars() { - datetimestamp=`date +%Y-%m-%d_%Hh%Mm` # Datestamp e.g 2002-09-21_18h12m - date_stamp=`date +%Y-%m-%d` # Datestamp e.g 2002-09-21 - date_day_of_week=`date +%A` # Day of the week e.g. Monday - date_dayno_of_week=`date +%u` # Day number of the week 1 to 7 where 1 represents Monday - date_day_of_month=`date +%e | sed -e 's/^ //'` # Date of the Month e.g. 27 - date_month=`date +%B` # Month e.g January - date_weekno=`date +%V | sed -e 's/^0//'` # Week Number e.g 37 - year=`date +%Y` - month=`date +%m | sed -e 's/^0//'` - date_lastday_of_last_month=$(days_of_month $(( $month==1 ? 12 : $month-1 )) $(( $month==1 ? ($year-1):$year )) ) + datetimestamp=$(date +%Y-%m-%d_%Hh%Mm) # Datestamp e.g 2002-09-21_18h12m + date_stamp=$(date +%Y-%m-%d) # Datestamp e.g 2002-09-21 + date_day_of_week=$(date +%A) # Day of the week e.g. Monday + date_dayno_of_week=$(date +%u) # Day number of the week 1 to 7 where 1 represents Monday + date_day_of_month=$(date +%e | sed -e 's/^ //') # Date of the Month e.g. 27 + date_month=$(date +%B) # Month e.g January + date_weekno=$(date +%V | sed -e 's/^0//') # Week Number e.g 37 + year=$(date +%Y) + month=$(date +%m | sed -e 's/^0//') + date_lastday_of_last_month=$(days_of_month $(($month == 1 ? 12 : $month - 1)) $(($month == 1 ? ($year - 1) : $year))) date_lastday_of_this_month=$(days_of_month $month $year) } @@ -308,51 +314,51 @@ set_datetime_vars() { # hardlink-copying of files to a latest folder. # @return: flags # @deps: load_default_config -files_postprocessing () { - local flags - let "flags=0x00" - let "flags_files_postprocessing_success_encrypt=0x01" - - # -> CONFIG_encrypt - [[ "${CONFIG_encrypt}" = "yes" && "${CONFIG_encrypt_password}" ]] && { - if (( $CONFIG_dryrun )); then - printf 'dry-running: openssl enc -aes-256-cbc -e -in %s -out %s.enc -pass pass:%s\n' ${1} ${1} "${CONFIG_encrypt_password}" - else - openssl enc -aes-256-cbc -e -in ${1} -out ${1}.enc -pass pass:"${CONFIG_encrypt_password}" - if (( $? == 0 )); then - if rm ${1} 2>&1; then - echo "Successfully encrypted archive as ${1}.enc" - let "flags |= $flags_files_postprocessing_success_encrypt" - else - echo "Successfully encrypted archive as ${1}.enc, but could not remove cleartext file ${1}." - let "E |= $E_enc_cleartext_delfailed" - fi - else - let "E |= $E_enc_failed" - fi - fi - } - # <- CONFIG_encrypt - - # -> CONFIG_mysql_dump_latest - [[ "${CONFIG_mysql_dump_latest}" = "yes" ]] && { - if (( $flags & $flags_files_postprocessing_success_encrypt )); then - if (( $CONFIG_dryrun )); then - printf 'dry-running: cp -al %s.enc %s/latest/\n' "${1}" "${CONFIG_backup_dir}" - else - cp -al "${1}${suffix}.enc" "${CONFIG_backup_dir}"/latest/ - fi - else - if (( $CONFIG_dryrun )); then - printf 'dry-running: cp -al %s %s/latest/\n' "${1}" "${CONFIG_backup_dir}" - else - cp -al "${1}" "${CONFIG_backup_dir}"/latest/ - fi - fi - } - # <- CONFIG_mysql_dump_latest - - return $flags +files_postprocessing() { + local flags + let "flags=0x00" + let "flags_files_postprocessing_success_encrypt=0x01" + + # -> CONFIG_encrypt + [[ "${CONFIG_encrypt}" = "yes" && "${CONFIG_encrypt_password}" ]] && { + if (($CONFIG_dryrun)); then + printf 'dry-running: openssl enc -aes-256-cbc -e -in %s -out %s.enc -pass pass:%s\n' ${1} ${1} "${CONFIG_encrypt_password}" + else + openssl enc -aes-256-cbc -e -in ${1} -out ${1}.enc -pass pass:"${CONFIG_encrypt_password}" + if (($? == 0)); then + if rm ${1} 2>&1; then + echo "Successfully encrypted archive as ${1}.enc" + let "flags |= $flags_files_postprocessing_success_encrypt" + else + echo "Successfully encrypted archive as ${1}.enc, but could not remove cleartext file ${1}." + let "E |= $E_enc_cleartext_delfailed" + fi + else + let "E |= $E_enc_failed" + fi + fi + } + # <- CONFIG_encrypt + + # -> CONFIG_mysql_dump_latest + [[ "${CONFIG_mysql_dump_latest}" = "yes" ]] && { + if (($flags & $flags_files_postprocessing_success_encrypt)); then + if (($CONFIG_dryrun)); then + printf 'dry-running: cp -al %s.enc %s/latest/\n' "${1}" "${CONFIG_backup_dir}" + else + cp -al "${1}${suffix}.enc" "${CONFIG_backup_dir}"/latest/ + fi + else + if (($CONFIG_dryrun)); then + printf 'dry-running: cp -al %s %s/latest/\n' "${1}" "${CONFIG_backup_dir}" + else + cp -al "${1}" "${CONFIG_backup_dir}"/latest/ + fi + fi + } + # <- CONFIG_mysql_dump_latest + + return $flags } # @info: When called, sets error and notify strings matching their flags. It then goes through all @@ -360,7 +366,7 @@ files_postprocessing () { # @args: (none) # @return: true if no errors were set, otherwise false # @deps: log_base2, load_default_config -error_handler () { +error_handler() { errors=( [0x01]='dbdump() failed.' @@ -399,24 +405,24 @@ error_handler () { local n local e - n=$((${#notify[@]}-1)) - while (( N > 0 )); do - e=$((2**n)) - if (( N&e )); then + n=$((${#notify[@]} - 1)) + while ((N > 0)); do + e=$((2 ** n)) + if ((N & e)); then echo "Note:" ${notify[e]} let "N-=e" fi ((n--)) done - unset n; - - n=$((${#errors[@]}-1)) - if (( E > 0 )); then - while (( E > 0 )); do - e=$((2**n)) - if (( E&e )); then - echo "Error:" ${errors[e]} - let "E-=e" + unset n + + n=$((${#errors[@]} - 1)) + if ((E > 0)); then + while ((E > 0)); do + e=$((2 ** n)) + if ((E & e)); then + echo "Error:" ${errors[e]} + let "E-=e" fi ((n--)) done @@ -430,36 +436,36 @@ error_handler () { # @args: archive file without compression suffix, i.e. ending on .tar # @return: true in case of dry-run, otherwise the return value of tar -cvf # @deps: load_default_config -backup_local_files () { - if ((! ${#CONFIG_backup_local_files[@]})) ; then - if (( $CONFIG_dryrun )); then +backup_local_files() { + if ((!${#CONFIG_backup_local_files[@]})); then + if (($CONFIG_dryrun)); then case "${CONFIG_mysql_dump_compression}" in - 'gzip') - echo "tar -czvf ${1}${suffix} ${CONFIG_backup_local_files[@]}"; - ;; - 'bzip2') - echo "tar -cjvf ${1}${suffix} ${CONFIG_backup_local_files[@]}"; - ;; - *) - echo "tar -cvf ${1}${suffix} ${CONFIG_backup_local_files[@]}"; - ;; + 'gzip') + echo "tar -czvf ${1}${suffix} ${CONFIG_backup_local_files[@]}" + ;; + 'bzip2') + echo "tar -cjvf ${1}${suffix} ${CONFIG_backup_local_files[@]}" + ;; + *) + echo "tar -cvf ${1}${suffix} ${CONFIG_backup_local_files[@]}" + ;; esac echo "dry-running: tar -cv ${1} ${CONFIG_backup_local_files[@]}" - return 0; + return 0 else case "${CONFIG_mysql_dump_compression}" in - 'gzip') - tar -czvf "${1}${suffix}" "${CONFIG_backup_local_files[@]}"; - return $? - ;; - 'bzip2') - tar -cjvf "${1}${suffix}" "${CONFIG_backup_local_files[@]}"; - return $? - ;; - *) - tar -cvf "${1}${suffix}" "${CONFIG_backup_local_files[@]}"; - return $? - ;; + 'gzip') + tar -czvf "${1}${suffix}" "${CONFIG_backup_local_files[@]}" + return $? + ;; + 'bzip2') + tar -cjvf "${1}${suffix}" "${CONFIG_backup_local_files[@]}" + return $? + ;; + *) + tar -cvf "${1}${suffix}" "${CONFIG_backup_local_files[@]}" + return $? + ;; esac fi else @@ -471,105 +477,115 @@ backup_local_files () { # @info: Parses the configuration options and sets the variables appropriately. # @args: (none) # @deps: load_default_config -parse_configuration () { - # OPT string for use with mysqldump ( see man mysqldump ) - opt=( '--quote-names' '--opt' '--events' ) - - # OPT string for use with mysql (see man mysql ) - mysql_opt=() - - # OPT string for use with mysqldump fullschema - opt_fullschema=( '--all-databases' '--routines' '--no-data' ) - - # OPT string for use with mysqlstatus - opt_dbstatus=( '--status' ) - - if [ "$NODOT_VER" -ge 56 ]; then - [[ "${CONFIG_mysql_dump_usessl}" = "yes" ]] && { - opt=( "${opt[@]}" '--ssl-mode=REQUIRED' ) - mysql_opt=( "${mysql_opt[@]}" '--ssl-mode=REQUIRED' ) - opt_fullschema=( "${opt_fullschema[@]}" '--ssl-mode=REQUIRED' ) - opt_dbstatus=( "${opt_dbstatus[@]}" '--ssl-mode=REQUIRED' ) - } - else - [[ "${CONFIG_mysql_dump_usessl}" = "yes" ]] && { - opt=( "${opt[@]}" '--ssl' ) - mysql_opt=( "${mysql_opt[@]}" '--ssl' ) - opt_fullschema=( "${opt_fullschema[@]}" '--ssl' ) - opt_dbstatus=( "${opt_dbstatus[@]}" '--ssl' ) - } - fi - [[ "${CONFIG_mysql_dump_master_data}" ]] && (( ${CONFIG_mysql_dump_master_data} == 1 || ${CONFIG_mysql_dump_master_data} == 2 )) && { opt=( "${opt[@]}" "--master-data=${CONFIG_mysql_dump_master_data}" );} - [[ "${CONFIG_mysql_dump_single_transaction}" = "yes" ]] && { - opt=( "${opt[@]}" '--single-transaction' ) - opt_fullschema=( "${opt_fullschema[@]}" '--single-transaction' ) - } - [[ "${CONFIG_mysql_dump_commcomp}" = "yes" ]] && { - opt=( "${opt[@]}" '--compress' ) - opt_fullschema=( "${opt_fullschema[@]}" '--compress' ) - opt_dbstatus=( "${opt_dbstatus[@]}" '--compress' ) - } - [[ "${CONFIG_mysql_dump_max_allowed_packet}" ]] && { - opt=( "${opt[@]}" "--max_allowed_packet=${CONFIG_mysql_dump_max_allowed_packet}" ) - opt_fullschema=( "${opt_fullschema[@]}" "--max_allowed_packet=${CONFIG_mysql_dump_max_allowed_packet}" ) - } - [[ "${CONFIG_mysql_dump_socket}" ]] && { - opt=( "${opt[@]}" "--socket=${CONFIG_mysql_dump_socket}" ) - mysql_opt=( "${mysql_opt[@]}" "--socket=${CONFIG_mysql_dump_socket}" ) - opt_fullschema=( "${opt_fullschema[@]}" "--socket=${CONFIG_mysql_dump_socket}" ) - opt_dbstatus=( "${opt_dbstatus[@]}" "--socket=${CONFIG_mysql_dump_socket}" ) +parse_configuration() { + # OPT string for use with mysqldump ( see man mysqldump ) + opt=('--quote-names' '--opt' '--events') + + # OPT string for use with mysql (see man mysql ) + mysql_opt=() + + # OPT string for use with mysqldump fullschema + opt_fullschema=('--all-databases' '--routines' '--no-data') + + # OPT string for use with mysqlstatus + opt_dbstatus=('--status') + + if [ "$NODOT_VER" -ge 56 ]; then + [[ "${CONFIG_mysql_dump_usessl}" = "yes" ]] && { + opt=("${opt[@]}" '--ssl-mode=REQUIRED') + mysql_opt=("${mysql_opt[@]}" '--ssl-mode=REQUIRED') + opt_fullschema=("${opt_fullschema[@]}" '--ssl-mode=REQUIRED') + opt_dbstatus=("${opt_dbstatus[@]}" '--ssl-mode=REQUIRED') } - [[ "${CONFIG_mysql_dump_port}" ]] && { - opt=( "${opt[@]}" "--port=${CONFIG_mysql_dump_port}" ) - mysql_opt=( "${mysql_opt[@]}" "--port=${CONFIG_mysql_dump_port}" ) - opt_fullschema=( "${opt_fullschema[@]}" "--port=${CONFIG_mysql_dump_port}" ) - opt_dbstatus=( "${opt_dbstatus[@]}" "--port=${CONFIG_mysql_dump_port}" ) + else + [[ "${CONFIG_mysql_dump_usessl}" = "yes" ]] && { + opt=("${opt[@]}" '--ssl') + mysql_opt=("${mysql_opt[@]}" '--ssl') + opt_fullschema=("${opt_fullschema[@]}" '--ssl') + opt_dbstatus=("${opt_dbstatus[@]}" '--ssl') } + fi + [[ "${CONFIG_mysql_dump_master_data}" ]] && ((${CONFIG_mysql_dump_master_data} == 1 || ${CONFIG_mysql_dump_master_data} == 2)) && { opt=("${opt[@]}" "--master-data=${CONFIG_mysql_dump_master_data}"); } + [[ "${CONFIG_mysql_dump_single_transaction}" = "yes" ]] && { + opt=("${opt[@]}" '--single-transaction') + opt_fullschema=("${opt_fullschema[@]}" '--single-transaction') + } + [[ "${CONFIG_mysql_dump_commcomp}" = "yes" ]] && { + opt=("${opt[@]}" '--compress') + opt_fullschema=("${opt_fullschema[@]}" '--compress') + opt_dbstatus=("${opt_dbstatus[@]}" '--compress') + } + [[ "${CONFIG_mysql_dump_max_allowed_packet}" ]] && { + opt=("${opt[@]}" "--max_allowed_packet=${CONFIG_mysql_dump_max_allowed_packet}") + opt_fullschema=("${opt_fullschema[@]}" "--max_allowed_packet=${CONFIG_mysql_dump_max_allowed_packet}") + } + [[ "${CONFIG_mysql_dump_socket}" ]] && { + opt=("${opt[@]}" "--socket=${CONFIG_mysql_dump_socket}") + mysql_opt=("${mysql_opt[@]}" "--socket=${CONFIG_mysql_dump_socket}") + opt_fullschema=("${opt_fullschema[@]}" "--socket=${CONFIG_mysql_dump_socket}") + opt_dbstatus=("${opt_dbstatus[@]}" "--socket=${CONFIG_mysql_dump_socket}") + } + [[ "${CONFIG_mysql_dump_port}" ]] && { + opt=("${opt[@]}" "--port=${CONFIG_mysql_dump_port}") + mysql_opt=("${mysql_opt[@]}" "--port=${CONFIG_mysql_dump_port}") + opt_fullschema=("${opt_fullschema[@]}" "--port=${CONFIG_mysql_dump_port}") + opt_dbstatus=("${opt_dbstatus[@]}" "--port=${CONFIG_mysql_dump_port}") + } - # Check if CREATE DATABASE should be included in Dump - if [[ "${CONFIG_mysql_dump_use_separate_dirs}" = "yes" ]]; then - if [[ "${CONFIG_mysql_dump_create_database}" = "no" ]]; then - opt=( "${opt[@]}" '--no-create-db' ) - else - opt=( "${opt[@]}" '--databases' ) - fi + # Check if CREATE DATABASE should be included in Dump + if [[ "${CONFIG_mysql_dump_use_separate_dirs}" = "yes" ]]; then + if [[ "${CONFIG_mysql_dump_create_database}" = "no" ]]; then + opt=("${opt[@]}" '--no-create-db') else - opt=( "${opt[@]}" '--databases' ) + opt=("${opt[@]}" '--databases') fi + else + opt=("${opt[@]}" '--databases') + fi - # if differential backup is active and the specified rotation is smaller than 21 days, set it to 21 days to ensure, that - # master backups aren't deleted. - if [[ "x$CONFIG_mysql_dump_differential" = "xyes" ]] && (( ${CONFIG_rotation_daily} < 21 )); then - CONFIG_rotation_daily=21 - fi + # if differential backup is active and the specified rotation is smaller than 21 days, set it to 21 days to ensure, that + # master backups aren't deleted. + if [[ "x$CONFIG_mysql_dump_differential" = "xyes" ]] && ((${CONFIG_rotation_daily} < 21)); then + CONFIG_rotation_daily=21 + fi - # -> determine suffix - case "${CONFIG_mysql_dump_compression}" in - 'gzip') suffix='.gz';; - 'bzip2') suffix='.bz2';; - *) suffix='';; - esac - # <- determine suffix - - # -> check exclude tables for wildcards - local tmp;tmp=() - local z;z=0 - for i in "${CONFIG_table_exclude[@]}"; do - r='^[^*.]+\.[^.]+$'; [[ "$i" =~ $r ]] || { printf 'The entry %s in CONFIG_table_exclude has a wrong format. Ignoring the entry.' "$i"; continue; } - db=${i%.*} - table=${i#"$db".} - r='\*'; [[ "$i" =~ $r ]] || { tmp[z++]="$i"; continue; } - while read -r; do tmp[z++]="${db}.${REPLY}"; done < <($MYSQL "${mysql_opt[@]}" --batch --skip-column-names -e "select table_name from information_schema.tables where table_schema='${db}' and table_name like '${table//\*/%}';") - done - for l in "${tmp[@]}"; do echo "exclude $l";done - CONFIG_table_exclude=("${tmp[@]}") - # <- - - if ((${#CONFIG_table_exclude[@]})); then - for i in "${CONFIG_table_exclude[@]}"; do - opt=( "${opt[@]}" "--ignore-table=$i" ) - done - fi + # -> determine suffix + case "${CONFIG_mysql_dump_compression}" in + 'gzip') suffix='.gz' ;; + 'bzip2') suffix='.bz2' ;; + *) suffix='' ;; + esac + # <- determine suffix + + # -> check exclude tables for wildcards + local tmp + tmp=() + local z + z=0 + for i in "${CONFIG_table_exclude[@]}"; do + r='^[^*.]+\.[^.]+$' + [[ "$i" =~ $r ]] || { + printf 'The entry %s in CONFIG_table_exclude has a wrong format. Ignoring the entry.' "$i" + continue + } + db=${i%.*} + table=${i#"$db".} + r='\*' + [[ "$i" =~ $r ]] || { + tmp[z++]="$i" + continue + } + while read -r; do tmp[z++]="${db}.${REPLY}"; done < <($MYSQL "${mysql_opt[@]}" --batch --skip-column-names -e "select table_name from information_schema.tables where table_schema='${db}' and table_name like '${table//\*/%}';") + done + for l in "${tmp[@]}"; do echo "exclude $l"; done + CONFIG_table_exclude=("${tmp[@]}") + # <- + + if ((${#CONFIG_table_exclude[@]})); then + for i in "${CONFIG_table_exclude[@]}"; do + opt=("${opt[@]}" "--ignore-table=$i") + done + fi } # @info: Backup database status @@ -577,33 +593,33 @@ parse_configuration () { # @return: true in case of dry-run, otherwise the return value of mysqlshow # @deps: load_default_config, parse_configuration dbstatus() { - if (( $CONFIG_dryrun )); then + if (($CONFIG_dryrun)); then case "${CONFIG_mysql_dump_compression}" in - 'gzip') - echo "dry-running: $MYSQLSHOW ${opt_dbstatus[@]} | gzip_compression > ${1}${suffix}"; - ;; - 'bzip2') - echo "dry-running: $MYSQLSHOW ${opt_dbstatus[@]} | bzip2_compression > ${1}${suffix}"; - ;; - *) - echo "dry-running: $MYSQLSHOW ${opt_dbstatus[@]} > ${1}${suffix}"; - ;; + 'gzip') + echo "dry-running: $MYSQLSHOW ${opt_dbstatus[@]} | gzip_compression > ${1}${suffix}" + ;; + 'bzip2') + echo "dry-running: $MYSQLSHOW ${opt_dbstatus[@]} | bzip2_compression > ${1}${suffix}" + ;; + *) + echo "dry-running: $MYSQLSHOW ${opt_dbstatus[@]} > ${1}${suffix}" + ;; esac - return 0; + return 0 else case "${CONFIG_mysql_dump_compression}" in - 'gzip') - $MYSQLSHOW "${opt_dbstatus[@]}" | gzip_compression > "${1}${suffix}"; - return $? - ;; - 'bzip2') - $MYSQLSHOW "${opt_dbstatus[@]}" | bzip2_compression > "${1}${suffix}"; - return $? - ;; - *) - $MYSQLSHOW "${opt_dbstatus[@]}" > "${1}${suffix}"; - return $? - ;; + 'gzip') + $MYSQLSHOW "${opt_dbstatus[@]}" | gzip_compression >"${1}${suffix}" + return $? + ;; + 'bzip2') + $MYSQLSHOW "${opt_dbstatus[@]}" | bzip2_compression >"${1}${suffix}" + return $? + ;; + *) + $MYSQLSHOW "${opt_dbstatus[@]}" >"${1}${suffix}" + return $? + ;; esac fi } @@ -612,34 +628,34 @@ dbstatus() { # @args: filename to save data to # @return: true in case of dry-run, otherwise the return value of mysqldump # @deps: load_default_config, parse_configuration -fullschema () { - if (( $CONFIG_dryrun )); then +fullschema() { + if (($CONFIG_dryrun)); then case "${CONFIG_mysql_dump_compression}" in - 'gzip') - echo "dry-running: $MYSQLDUMP ${opt_fullschema[@]} | gzip_compression > ${1}${suffix}"; - ;; - 'bzip2') - echo "dry-running: $MYSQLDUMP ${opt_fullschema[@]} | bzip2_compression > ${1}${suffix}"; - ;; - *) - echo "dry-running: $MYSQLDUMP ${opt_fullschema[@]} > ${1}${suffix}"; - ;; + 'gzip') + echo "dry-running: $MYSQLDUMP ${opt_fullschema[@]} | gzip_compression > ${1}${suffix}" + ;; + 'bzip2') + echo "dry-running: $MYSQLDUMP ${opt_fullschema[@]} | bzip2_compression > ${1}${suffix}" + ;; + *) + echo "dry-running: $MYSQLDUMP ${opt_fullschema[@]} > ${1}${suffix}" + ;; esac - return 0; + return 0 else case "${CONFIG_mysql_dump_compression}" in - 'gzip') - $MYSQLDUMP "${opt_fullschema[@]}" | gzip_compression > "${1}${suffix}"; - return $? - ;; - 'bzip2') - $MYSQLDUMP "${opt_fullschema[@]}" | bzip2_compression > "${1}${suffix}"; - return $? - ;; - *) - $MYSQLDUMP "${opt_fullschema[@]}" > "${1}${suffix}"; - return $? - ;; + 'gzip') + $MYSQLDUMP "${opt_fullschema[@]}" | gzip_compression >"${1}${suffix}" + return $? + ;; + 'bzip2') + $MYSQLDUMP "${opt_fullschema[@]}" | bzip2_compression >"${1}${suffix}" + return $? + ;; + *) + $MYSQLDUMP "${opt_fullschema[@]}" >"${1}${suffix}" + return $? + ;; esac fi } @@ -663,18 +679,18 @@ process_dbs() { # only activate differential backup for daily backups [[ "x$subfolder" != "xdaily" ]] && activate_differential_backup=0 || activate_differential_backup=1 - if (( $multipledbs )); then - # multiple dbs - subsubfolder="" - name="all-databases" + if (($multipledbs)); then + # multiple dbs + subsubfolder="" + name="all-databases" else - # single db - subsubfolder="/$1" - name="$@" + # single db + subsubfolder="/$1" + name="$@" fi [[ -d "${CONFIG_backup_dir}/${subfolder}${subsubfolder}" ]] || { - if (( $CONFIG_dryrun )); then + if (($CONFIG_dryrun)); then printf 'dry-running: mkdir -p %s/${subfolder}%s\n' "${CONFIG_backup_dir}" "${subsubfolder}" else mkdir -p "${CONFIG_backup_dir}/${subfolder}${subsubfolder}" @@ -684,175 +700,178 @@ process_dbs() { manifest_file="${CONFIG_backup_dir}/${subfolder}${subsubfolder}/Manifest" fname="${CONFIG_backup_dir}/${subfolder}${subsubfolder}/${prefix}${name}_${datetimestamp}${midfix}${extension}" - (( $CONFIG_debug )) && echo "DEBUG: process_dbs >> Setting manifest file to: ${manifest_file}" + (($CONFIG_debug)) && echo "DEBUG: process_dbs >> Setting manifest file to: ${manifest_file}" - if (( $multipledbs )); then - # multiple databases - db="all-databases" + if (($multipledbs)); then + # multiple databases + db="all-databases" else - # single db - db="$1" + # single db + db="$1" fi - if [[ "x$CONFIG_mysql_dump_differential" = "xyes" ]] && [[ "x${CONFIG_encrypt}" != "xyes" ]] && (( $activate_differential_backup )); then - - - unset manifest_entry manifest_entry_to_check - - echo "## Reading in Manifest file" - parse_manifest "$manifest_file" - echo - echo "Number of manifest entries: $(num_manifest_entries)" - echo - - - # -> generate diff file - let "filename_flags=0x00" - -# ## -> get latest differential manifest entry for specified db -# if get_latest_manifest_entry_for_db "$db" 1; then -# pid="${manifest_entry[2]}" -# # filename format: prefix_db_YYYY-MM-DD_HHhMMm_[A-Za-z0-9]{8}(.sql|.diff)(.gz|.bz2)(.enc) -# FileStub=${manifest_entry[0]%.@(sql|diff)*} -# FileExt=${manifest_entry[0]#"$FileStub"} -# re=".*\.enc.*"; [[ "$FileExt" =~ $re ]] && let "filename_flags|=$filename_flag_encrypted" -# re=".*\.gz.*"; [[ "$FileExt" =~ $re ]] && let "filename_flags|=$filename_flag_gz" -# re=".*\.bz2.*"; [[ "$FileExt" =~ $re ]] && let "filename_flags|=$filename_flag_bz2" -# re=".*\.diff.*"; [[ "$FileExt" =~ $re ]] && let "filename_flags|=$filename_flag_diff" -# manifest_latest_diff_entry=("${manifest_entry[@]}") -# else # no entries in manifest -# pid=0 -# fi -# ## <- get latest differential manifest entry for specified db - - ## -> get latest master manifest entry for specified db - # Create a differential backup if a master entry in the manifest exists, it isn't the day we do weekly master backups or the master file we fetched is already from today. - if get_latest_manifest_entry_for_db "$db" 0 && ( (( ${date_dayno_of_week} != ${CONFIG_do_weekly} )) || [[ "${manifest_entry[0]}" = *_$(date +%Y-%m-%d)_* ]] ); then - pid="${manifest_entry[2]}" - # filename format: prefix_db_YYYY-MM-DD_HHhMMm_[A-Za-z0-9]{8}(.sql|.diff)(.gz|.bz2)(.enc) - FileStub="${manifest_entry[0]%.@(sql|diff)*}" - FileExt="${manifest_entry[0]#"$FileStub"}" - re=".*\.enc.*"; [[ "$FileExt" =~ $re ]] && let "filename_flags|=$filename_flag_encrypted" - re=".*\.gz.*"; [[ "$FileExt" =~ $re ]] && let "filename_flags|=$filename_flag_gz" - re=".*\.bz2.*"; [[ "$FileExt" =~ $re ]] && let "filename_flags|=$filename_flag_bz2" - re=".*\.diff.*"; [[ "$FileExt" =~ $re ]] && let "filename_flags|=$filename_flag_diff" - manifest_latest_master_entry=("${manifest_entry[@]}") - else # no entries in manifest - pid=0 - fi - ## <- get latest master manifest entry for specified db + if [[ "x$CONFIG_mysql_dump_differential" = "xyes" ]] && [[ "x${CONFIG_encrypt}" != "xyes" ]] && (($activate_differential_backup)); then + + unset manifest_entry manifest_entry_to_check + + echo "## Reading in Manifest file" + parse_manifest "$manifest_file" + echo + echo "Number of manifest entries: $(num_manifest_entries)" + echo + + # -> generate diff file + let "filename_flags=0x00" + + # ## -> get latest differential manifest entry for specified db + # if get_latest_manifest_entry_for_db "$db" 1; then + # pid="${manifest_entry[2]}" + # # filename format: prefix_db_YYYY-MM-DD_HHhMMm_[A-Za-z0-9]{8}(.sql|.diff)(.gz|.bz2)(.enc) + # FileStub=${manifest_entry[0]%.@(sql|diff)*} + # FileExt=${manifest_entry[0]#"$FileStub"} + # re=".*\.enc.*"; [[ "$FileExt" =~ $re ]] && let "filename_flags|=$filename_flag_encrypted" + # re=".*\.gz.*"; [[ "$FileExt" =~ $re ]] && let "filename_flags|=$filename_flag_gz" + # re=".*\.bz2.*"; [[ "$FileExt" =~ $re ]] && let "filename_flags|=$filename_flag_bz2" + # re=".*\.diff.*"; [[ "$FileExt" =~ $re ]] && let "filename_flags|=$filename_flag_diff" + # manifest_latest_diff_entry=("${manifest_entry[@]}") + # else # no entries in manifest + # pid=0 + # fi + # ## <- get latest differential manifest entry for specified db + + ## -> get latest master manifest entry for specified db + # Create a differential backup if a master entry in the manifest exists, it isn't the day we do weekly master backups or the master file we fetched is already from today. + if get_latest_manifest_entry_for_db "$db" 0 && ( ((${date_dayno_of_week} != ${CONFIG_do_weekly})) || [[ "${manifest_entry[0]}" = *_$(date +%Y-%m-%d)_* ]]); then + pid="${manifest_entry[2]}" + # filename format: prefix_db_YYYY-MM-DD_HHhMMm_[A-Za-z0-9]{8}(.sql|.diff)(.gz|.bz2)(.enc) + FileStub="${manifest_entry[0]%.@(sql|diff)*}" + FileExt="${manifest_entry[0]#"$FileStub"}" + re=".*\.enc.*" + [[ "$FileExt" =~ $re ]] && let "filename_flags|=$filename_flag_encrypted" + re=".*\.gz.*" + [[ "$FileExt" =~ $re ]] && let "filename_flags|=$filename_flag_gz" + re=".*\.bz2.*" + [[ "$FileExt" =~ $re ]] && let "filename_flags|=$filename_flag_bz2" + re=".*\.diff.*" + [[ "$FileExt" =~ $re ]] && let "filename_flags|=$filename_flag_diff" + manifest_latest_master_entry=("${manifest_entry[@]}") + else # no entries in manifest + pid=0 + fi + ## <- get latest master manifest entry for specified db fi - if [[ "x$CONFIG_mysql_dump_differential" = "xyes" ]] && [[ "x${CONFIG_encrypt}" != "xyes" ]] && (( $activate_differential_backup )) && ((! ($filename_flags & $filename_flag_encrypted) )); then - - # the master file is encrypted ... well this just shouldn't happen ^^ not going to decrypt or stuff like that ...at least not today :) - - if [[ "x$pid" = "x0" ]]; then - # -> create master backup - cfname="$(mktemp "${fname%.sql}_"XXXXXXXX".sql${suffix}")" - uid="${cfname%.@(diff|sql)*}" - uid="${uid:-8:8}" - case "${CONFIG_mysql_dump_compression}" in - 'gzip') - $MYSQLDUMP "${opt[@]}" "$@" | gzip_compression > "$cfname"; - ;; - 'bzip2') - $MYSQLDUMP "${opt[@]}" "$@" | bzip2_compression > "$cfname"; - ;; - *) - $MYSQLDUMP "${opt[@]}" "$@" > "$cfname"; - ;; - esac - add_manifest_entry "$manifest_file" "$cfname" "$pid" "$db" && parse_manifest "$manifest_file" && cp -al "$cfname" "${CONFIG_backup_dir}"/latest/ && echo "Generated master backup $cfname" && return 0 || return 1 - # <- create master backup - else - cfname="$(mktemp "${fname%.sql}_"XXXXXXXX".diff${suffix}")" - uid="${cfname%.@(diff|sql)*}" - uid=${uid:-8:8} - echo "Creating differential backup to ${manifest_entry[0]}:" - case "${CONFIG_mysql_dump_compression}" in - 'gzip') - if (( $filename_flags & $filename_flag_gz )); then - diff <(gzip_compression -dc "${manifest_latest_master_entry[0]}") <($MYSQLDUMP "${opt[@]}" "$@") | gzip_compression > "$cfname"; - elif (( $filename_flags & $filename_flag_bz2 )); then - diff <(bzip2_compression -dc "${manifest_latest_master_entry[0]}") <($MYSQLDUMP "${opt[@]}" "$@") | gzip_compression > "$cfname"; - else - diff "${manifest_latest_master_entry[0]}" <($MYSQLDUMP "${opt[@]}" "$@") | gzip_compression > "$cfname"; - fi - ;; - 'bzip2') - if (( $filename_flags & $filename_flag_gz )); then - diff <(gzip_compression -dc "${manifest_latest_master_entry[0]}") <($MYSQLDUMP "${opt[@]}" "$@") | bzip2_compression > "$cfname"; - elif (( $filename_flags & $filename_flag_bz2 )); then - diff <(bzip2_compression -dc "${manifest_latest_master_entry[0]}") <($MYSQLDUMP "${opt[@]}" "$@") | bzip2_compression > "$cfname"; - else - diff "${manifest_latest_master_entry[0]}" <($MYSQLDUMP "${opt[@]}" "$@") | bzip2_compression > "$cfname"; - fi - ;; - *) - if (( $filename_flags & $filename_flag_gz )); then - diff <(gzip_compression -dc "${manifest_latest_master_entry[0]}") <($MYSQLDUMP "${opt[@]}" "$@") > "$cfname"; - elif (( $filename_flags & $filename_flag_bz2 )); then - diff <(bzip2_compression -dc "${manifest_latest_master_entry[0]}") <($MYSQLDUMP "${opt[@]}" "$@") > "$cfname"; - else - diff "${manifest_latest_master_entry[0]}" <($MYSQLDUMP "${opt[@]}" "$@") > "$cfname"; - fi - ;; - esac - add_manifest_entry "$manifest_file" "$cfname" "$pid" "$db" && parse_manifest "$manifest_file" && cp -al "$cfname" "${manifest_latest_master_entry[0]}" "${CONFIG_backup_dir}"/latest/ && echo "generated $cfname" && return 0 || return 1 - - fi - # <- generate diff filename + if [[ "x$CONFIG_mysql_dump_differential" = "xyes" ]] && [[ "x${CONFIG_encrypt}" != "xyes" ]] && (($activate_differential_backup)) && ((!($filename_flags & $filename_flag_encrypted))); then + + # the master file is encrypted ... well this just shouldn't happen ^^ not going to decrypt or stuff like that ...at least not today :) + + if [[ "x$pid" = "x0" ]]; then + # -> create master backup + cfname="$(mktemp "${fname%.sql}_"XXXXXXXX".sql${suffix}")" + uid="${cfname%.@(diff|sql)*}" + uid="${uid:-8:8}" + case "${CONFIG_mysql_dump_compression}" in + 'gzip') + $MYSQLDUMP "${opt[@]}" "$@" | gzip_compression >"$cfname" + ;; + 'bzip2') + $MYSQLDUMP "${opt[@]}" "$@" | bzip2_compression >"$cfname" + ;; + *) + $MYSQLDUMP "${opt[@]}" "$@" >"$cfname" + ;; + esac + add_manifest_entry "$manifest_file" "$cfname" "$pid" "$db" && parse_manifest "$manifest_file" && cp -al "$cfname" "${CONFIG_backup_dir}"/latest/ && echo "Generated master backup $cfname" && return 0 || return 1 + # <- create master backup + else + cfname="$(mktemp "${fname%.sql}_"XXXXXXXX".diff${suffix}")" + uid="${cfname%.@(diff|sql)*}" + uid=${uid:-8:8} + echo "Creating differential backup to ${manifest_entry[0]}:" + case "${CONFIG_mysql_dump_compression}" in + 'gzip') + if (($filename_flags & $filename_flag_gz)); then + diff <(gzip_compression -dc "${manifest_latest_master_entry[0]}") <($MYSQLDUMP "${opt[@]}" "$@") | gzip_compression >"$cfname" + elif (($filename_flags & $filename_flag_bz2)); then + diff <(bzip2_compression -dc "${manifest_latest_master_entry[0]}") <($MYSQLDUMP "${opt[@]}" "$@") | gzip_compression >"$cfname" + else + diff "${manifest_latest_master_entry[0]}" <($MYSQLDUMP "${opt[@]}" "$@") | gzip_compression >"$cfname" + fi + ;; + 'bzip2') + if (($filename_flags & $filename_flag_gz)); then + diff <(gzip_compression -dc "${manifest_latest_master_entry[0]}") <($MYSQLDUMP "${opt[@]}" "$@") | bzip2_compression >"$cfname" + elif (($filename_flags & $filename_flag_bz2)); then + diff <(bzip2_compression -dc "${manifest_latest_master_entry[0]}") <($MYSQLDUMP "${opt[@]}" "$@") | bzip2_compression >"$cfname" + else + diff "${manifest_latest_master_entry[0]}" <($MYSQLDUMP "${opt[@]}" "$@") | bzip2_compression >"$cfname" + fi + ;; + *) + if (($filename_flags & $filename_flag_gz)); then + diff <(gzip_compression -dc "${manifest_latest_master_entry[0]}") <($MYSQLDUMP "${opt[@]}" "$@") >"$cfname" + elif (($filename_flags & $filename_flag_bz2)); then + diff <(bzip2_compression -dc "${manifest_latest_master_entry[0]}") <($MYSQLDUMP "${opt[@]}" "$@") >"$cfname" + else + diff "${manifest_latest_master_entry[0]}" <($MYSQLDUMP "${opt[@]}" "$@") >"$cfname" + fi + ;; + esac + add_manifest_entry "$manifest_file" "$cfname" "$pid" "$db" && parse_manifest "$manifest_file" && cp -al "$cfname" "${manifest_latest_master_entry[0]}" "${CONFIG_backup_dir}"/latest/ && echo "generated $cfname" && return 0 || return 1 + + fi + # <- generate diff filename else - cfname="${fname}${suffix}" - if (( $CONFIG_dryrun )); then - case "${CONFIG_mysql_dump_compression}" in - 'gzip') - echo "dry-running: $MYSQLDUMP ${opt[@]} $@ | gzip_compression > ${cfname}" - ;; - 'bzip2') - echo "dry-running: $MYSQLDUMP ${opt[@]} $@ | bzip2_compression > ${cfname}" - ;; - *) - echo "dry-running: $MYSQLDUMP ${opt[@]} $@ > ${cfname}" - ;; - esac - return 0; - else - case "${CONFIG_mysql_dump_compression}" in - 'gzip') - $MYSQLDUMP "${opt[@]}" "$@" | gzip_compression > "${cfname}" - ret=$? - ;; - 'bzip2') - $MYSQLDUMP "${opt[@]}" "$@" | bzip2_compression > "${cfname}" - ret=$? - ;; - *) - $MYSQLDUMP "${opt[@]}" "$@" > "${cfname}" - ret=$? - ;; - esac - fi + cfname="${fname}${suffix}" + if (($CONFIG_dryrun)); then + case "${CONFIG_mysql_dump_compression}" in + 'gzip') + echo "dry-running: $MYSQLDUMP ${opt[@]} $@ | gzip_compression > ${cfname}" + ;; + 'bzip2') + echo "dry-running: $MYSQLDUMP ${opt[@]} $@ | bzip2_compression > ${cfname}" + ;; + *) + echo "dry-running: $MYSQLDUMP ${opt[@]} $@ > ${cfname}" + ;; + esac + return 0 + else + case "${CONFIG_mysql_dump_compression}" in + 'gzip') + $MYSQLDUMP "${opt[@]}" "$@" | gzip_compression >"${cfname}" + ret=$? + ;; + 'bzip2') + $MYSQLDUMP "${opt[@]}" "$@" | bzip2_compression >"${cfname}" + ret=$? + ;; + *) + $MYSQLDUMP "${opt[@]}" "$@" >"${cfname}" + ret=$? + ;; + esac + fi fi - - if (( $ret == 0 )); then - echo "Rotating $(( ${rotation}/${rotation_divisor} )) ${rotation_string} backups for ${name}" - if (( $CONFIG_dryrun )); then - find "${CONFIG_backup_dir}/${subfolder}${subsubfolder}" -mtime +"${rotation}" -type f -exec echo "dry-running: rm" {} \; - else - find "${CONFIG_backup_dir}/${subfolder}${subsubfolder}" -mtime +"${rotation}" -type f -exec rm {} \; - fi - files_postprocessing "$cfname" - tmp_flags=$?; var=; (( $tmp_flags & $flags_files_postprocessing_success_encrypt )) && var=.enc - backupfiles=( "${backupfiles[@]}" "${cfname}${var}" ) + if (($ret == 0)); then + echo "Rotating $((${rotation} / ${rotation_divisor})) ${rotation_string} backups for ${name}" + if (($CONFIG_dryrun)); then + find "${CONFIG_backup_dir}/${subfolder}${subsubfolder}" -mtime +"${rotation}" -type f -exec echo "dry-running: rm" {} \; + else + find "${CONFIG_backup_dir}/${subfolder}${subsubfolder}" -mtime +"${rotation}" -type f -exec rm {} \; + fi + files_postprocessing "$cfname" + tmp_flags=$? + var= + (($tmp_flags & $flags_files_postprocessing_success_encrypt)) && var=.enc + backupfiles=("${backupfiles[@]}" "${cfname}${var}") else - let "E |= $E_dbdump_failed" - echo "dbdump with parameters \"${CONFIG_db_names[@]}\" \"${cfname}\" failed!" + let "E |= $E_dbdump_failed" + echo "dbdump with parameters \"${CONFIG_db_names[@]}\" \"${cfname}\" failed!" fi } @@ -864,146 +883,146 @@ activateIO() { # $1 = $log_file, $2 = $log_errfile #(( $CONFIG_debug )) || { - touch "$log_file" - exec 6>&1 # Link file descriptor #6 with stdout. Saves stdout. - exec > "$log_file" # stdout replaced with file $log_file. + touch "$log_file" + exec 6>&1 # Link file descriptor #6 with stdout. Saves stdout. + exec >"$log_file" # stdout replaced with file $log_file. - touch "$log_errfile" - exec 7>&2 # Link file descriptor #7 with stderr. Saves stderr. - exec 2> "$log_errfile" # stderr replaced with file $log_errfile. + touch "$log_errfile" + exec 7>&2 # Link file descriptor #7 with stderr. Saves stderr. + exec 2>"$log_errfile" # stderr replaced with file $log_errfile. #} } # @info: Restore stdout and stderr redirections. # @deps: (none) removeIO() { - exec 1>&6 6>&- # Restore stdout and close file descriptor #6. - exec 2>&7 7>&- # Restore stdout and close file descriptor #7. + exec 1>&6 6>&- # Restore stdout and close file descriptor #6. + exec 2>&7 7>&- # Restore stdout and close file descriptor #7. } # @info: Checks directories and subdirectories for existence and activates logging to either # $CONFIG_backup_dir or /tmp depending on what exists. # @args: (none) # @deps: load_default_config, activateIO, chk_folder_writable, error_handler -directory_checks_enable_logging () { - ################################################################################### - # Check directories and do cleanup work - - checkdirs=( "${CONFIG_backup_dir}"/{daily,weekly,monthly,latest,tmp} ) - [[ "${CONFIG_backup_local_files[@]}" ]] && { checkdirs=( "${checkdirs[@]}" "${CONFIG_backup_dir}/backup_local_files" ); } - [[ "${CONFIG_mysql_dump_full_schema}" = 'yes' ]] && { checkdirs=( "${checkdirs[@]}" "${CONFIG_backup_dir}/fullschema" ); } - [[ "${CONFIG_mysql_dump_dbstatus}" = 'yes' ]] && { checkdirs=( "${checkdirs[@]}" "${CONFIG_backup_dir}/status" ); } - - tmp_permcheck=0 - - - # "dirname ${CONFIG_backup_dir}" exists? - # Y -> ${CONFIG_backup_dir} exists? - # Y -> Dry-run? - # Y -> log to /tmp, proceed to test subdirs - # N -> check writable ${CONFIG_backup_dir}? - # Y -> proceed to test subdirs - # N -> error: can't write to ${CONFIG_backup_dir}. Exit. - # N -> Dry-run? - # N -> proceed without testing subdirs - # Y -> create directory ${CONFIG_backup_dir}? - # Y -> check writable ${CONFIG_backup_dir}? - # Y -> proceed to test subdirs - # N -> error: can't write to ${CONFIG_backup_dir}. Exit. - # N -> error: ${CONFIG_backup_dir} is not writable. Exit. - # N -> Dry-run? - # Y -> log to /tmp, proceed without testing subdirs - # N -> error: no basedir. Exit. - - - # -> check base folder - if [[ -d "$(dirname "${CONFIG_backup_dir}")" ]]; then - - if [[ -d "${CONFIG_backup_dir}" ]]; then - if (( $CONFIG_dryrun )); then - printf 'dry-running. Skipping. Logging to /tmp\n' - log_file="/tmp/${CONFIG_mysql_dump_host}-`date +%N`.log" - log_errfile="/tmp/ERRORS_${CONFIG_mysql_dump_host}-`date +%N`.log" - activateIO "$log_file" "$log_errfile" - tmp_permcheck=1 - else - if chk_folder_writable "${CONFIG_backup_dir}"; then - log_file="${CONFIG_backup_dir}/${CONFIG_mysql_dump_host}-`date +%N`.log" - log_errfile="${CONFIG_backup_dir}/ERRORS_${CONFIG_mysql_dump_host}-`date +%N`.log" - activateIO "$log_file" "$log_errfile" - tmp_permcheck=1 - else - printf 'no. Exiting.\n' - let "E |= $E_config_backupdir_not_writable" - error_handler - fi - fi - - else - - printf 'creating ... ' - if (( $CONFIG_dryrun )); then - printf 'dry-running. Skipping.\n' - else - if mkdir -p "${CONFIG_backup_dir}" >/dev/null 2>&1; then - printf 'success.\n' - log_file="${CONFIG_backup_dir}/${CONFIG_mysql_dump_host}-`date +%N`.log" - log_errfile="${CONFIG_backup_dir}/ERRORS_${CONFIG_mysql_dump_host}-`date +%N`.log" - activateIO "$log_file" "$log_errfile" - tmp_permcheck=1 - else - printf 'failed. Exiting.\n' - let "E |= $E_mkdir_basedir_failed" - error_handler - fi - fi - - fi +directory_checks_enable_logging() { + ################################################################################### + # Check directories and do cleanup work + + checkdirs=("${CONFIG_backup_dir}"/{daily,weekly,monthly,latest,tmp}) + [[ "${CONFIG_backup_local_files[@]}" ]] && { checkdirs=("${checkdirs[@]}" "${CONFIG_backup_dir}/backup_local_files"); } + [[ "${CONFIG_mysql_dump_full_schema}" = 'yes' ]] && { checkdirs=("${checkdirs[@]}" "${CONFIG_backup_dir}/fullschema"); } + [[ "${CONFIG_mysql_dump_dbstatus}" = 'yes' ]] && { checkdirs=("${checkdirs[@]}" "${CONFIG_backup_dir}/status"); } + + tmp_permcheck=0 + + # "dirname ${CONFIG_backup_dir}" exists? + # Y -> ${CONFIG_backup_dir} exists? + # Y -> Dry-run? + # Y -> log to /tmp, proceed to test subdirs + # N -> check writable ${CONFIG_backup_dir}? + # Y -> proceed to test subdirs + # N -> error: can't write to ${CONFIG_backup_dir}. Exit. + # N -> Dry-run? + # N -> proceed without testing subdirs + # Y -> create directory ${CONFIG_backup_dir}? + # Y -> check writable ${CONFIG_backup_dir}? + # Y -> proceed to test subdirs + # N -> error: can't write to ${CONFIG_backup_dir}. Exit. + # N -> error: ${CONFIG_backup_dir} is not writable. Exit. + # N -> Dry-run? + # Y -> log to /tmp, proceed without testing subdirs + # N -> error: no basedir. Exit. + + # -> check base folder + if [[ -d "$(dirname "${CONFIG_backup_dir}")" ]]; then + + if [[ -d "${CONFIG_backup_dir}" ]]; then + if (($CONFIG_dryrun)); then + printf 'dry-running. Skipping. Logging to /tmp\n' + log_file="/tmp/${CONFIG_mysql_dump_host}-$(date +%N).log" + log_errfile="/tmp/ERRORS_${CONFIG_mysql_dump_host}-$(date +%N).log" + activateIO "$log_file" "$log_errfile" + tmp_permcheck=1 + else + if chk_folder_writable "${CONFIG_backup_dir}"; then + log_file="${CONFIG_backup_dir}/${CONFIG_mysql_dump_host}-$(date +%N).log" + log_errfile="${CONFIG_backup_dir}/ERRORS_${CONFIG_mysql_dump_host}-$(date +%N).log" + activateIO "$log_file" "$log_errfile" + tmp_permcheck=1 + else + printf 'no. Exiting.\n' + let "E |= $E_config_backupdir_not_writable" + error_handler + fi + fi else - if (( $CONFIG_dryrun )); then - printf 'dry-running. Skipping. Logging to /tmp\n' - log_file="/tmp/${CONFIG_mysql_dump_host}-`date +%N`.log" - log_errfile="/tmp/ERRORS_${CONFIG_mysql_dump_host}-`date +%N`.log" - activateIO "$log_file" "$log_errfile" - else - printf 'does not exist. Exiting.\n' - let "E |= $E_no_basedir" - error_handler - fi + printf 'creating ... ' + if (($CONFIG_dryrun)); then + printf 'dry-running. Skipping.\n' + else + if mkdir -p "${CONFIG_backup_dir}" >/dev/null 2>&1; then + printf 'success.\n' + log_file="${CONFIG_backup_dir}/${CONFIG_mysql_dump_host}-$(date +%N).log" + log_errfile="${CONFIG_backup_dir}/ERRORS_${CONFIG_mysql_dump_host}-$(date +%N).log" + activateIO "$log_file" "$log_errfile" + tmp_permcheck=1 + else + printf 'failed. Exiting.\n' + let "E |= $E_mkdir_basedir_failed" + error_handler + fi + fi fi - # <- check base folder - - - # -> check subdirs - if (( $tmp_permcheck == 1 )); then - - (( $CONFIG_dryrun )) || [[ -r "${CONFIG_backup_dir}" && -x "${CONFIG_backup_dir}" ]] || { let "E |= $E_perm_basedir"; error_handler; } - - for i in "${checkdirs[@]}"; do - printf 'checking directory "%s" ... ' "$i" - if [[ -d "$i" ]]; then - printf 'exists.\n' - else - printf 'creating ... ' - if (( $CONFIG_dryrun )); then - printf 'dry-running. Skipping.\n' - else - if mkdir -p "$i" >/dev/null 2>&1; then - printf 'success.\n' - else - printf 'failed. Exiting.\n' - let "E |= $E_mkdir_subdirs_failed" - error_handler - fi - fi - fi - done + else + + if (($CONFIG_dryrun)); then + printf 'dry-running. Skipping. Logging to /tmp\n' + log_file="/tmp/${CONFIG_mysql_dump_host}-$(date +%N).log" + log_errfile="/tmp/ERRORS_${CONFIG_mysql_dump_host}-$(date +%N).log" + activateIO "$log_file" "$log_errfile" + else + printf 'does not exist. Exiting.\n' + let "E |= $E_no_basedir" + error_handler fi - # <- check subdirs + + fi + # <- check base folder + + # -> check subdirs + if (($tmp_permcheck == 1)); then + + (($CONFIG_dryrun)) || [[ -r "${CONFIG_backup_dir}" && -x "${CONFIG_backup_dir}" ]] || { + let "E |= $E_perm_basedir" + error_handler + } + + for i in "${checkdirs[@]}"; do + printf 'checking directory "%s" ... ' "$i" + if [[ -d "$i" ]]; then + printf 'exists.\n' + else + printf 'creating ... ' + if (($CONFIG_dryrun)); then + printf 'dry-running. Skipping.\n' + else + if mkdir -p "$i" >/dev/null 2>&1; then + printf 'success.\n' + else + printf 'failed. Exiting.\n' + let "E |= $E_mkdir_subdirs_failed" + error_handler + fi + fi + fi + done + + fi + # <- check subdirs } @@ -1011,76 +1030,76 @@ directory_checks_enable_logging () { # be cleaned. # @args: (none) # @deps: load_default_config -cleanup_latest () { - # -> latest cleanup - if [[ "${CONFIG_mysql_dump_latest}" = "yes" ]]; then - printf 'Cleaning up latest directory ... ' - if (( $CONFIG_dryrun )); then - printf 'dry-running. Skipping.\n' +cleanup_latest() { + # -> latest cleanup + if [[ "${CONFIG_mysql_dump_latest}" = "yes" ]]; then + printf 'Cleaning up latest directory ... ' + if (($CONFIG_dryrun)); then + printf 'dry-running. Skipping.\n' + else + if rm -f "${CONFIG_backup_dir}"/latest/* >/dev/null 2>&1; then + printf 'success.\n' else - if rm -f "${CONFIG_backup_dir}"/latest/* >/dev/null 2>&1; then - printf 'success.\n' - else - printf 'failed. Continuing anyway, activating Note-Flag.\n' - let "N |= $N_latest_cleanup_failed" - fi + printf 'failed. Continuing anyway, activating Note-Flag.\n' + let "N |= $N_latest_cleanup_failed" fi fi - # <- latest cleanup + fi + # <- latest cleanup } # @info: Checks for dependencies in form of external programs, that need to be available when running # this program. # @args: (none) # @deps: load_default_config -check_dependencies () { - echo - echo "# Testing for installed programs" - dependencies=( 'mysql' 'mysqldump' ) - - if [[ "x$CONFIG_multicore" = 'xyes' ]]; then - - if [[ "x$CONFIG_mysql_dump_compression" = 'xbzip2' ]]; then - if type pbzip2 &>/dev/null; then - echo "pbzip2 ... found." - else - CONFIG_multicore='no' # turn off multicore support, since the program isn't there - echo "WARNING: Turning off multicore support, since pbzip2 isn't there." - fi - elif [[ "x$CONFIG_mysql_dump_compression" = 'xgzip' ]]; then - if type pigz &>/dev/null; then - echo "pigz ... found." - else - CONFIG_multicore='no' # turn off multicore support, since the program isn't there - echo "WARNING: Turning off multicore support, since pigz isn't there." - fi - fi - - else - [[ "x$CONFIG_mysql_dump_compression" = 'xbzip2' ]] && dependencies=("${dependencies[@]}" 'bzip2' ) - [[ "x$CONFIG_mysql_dump_compression" = 'xgzip' ]] && dependencies=("${dependencies[@]}" 'gzip' ) - fi - - if [[ "x$CONFIG_mailcontent" = 'xlog' || "x$CONFIG_mailcontent" = 'xquiet' ]]; then - dependencies=( "${dependencies[@]}" 'mail' ) - elif [[ "x$CONFIG_mailcontent" = 'xfiles' ]]; then - dependencies=( "${dependencies[@]}" 'mail' ) - if [[ "x$CONFIG_mail_use_uuencoded_attachments" != 'xyes' ]]; then - dependencies=( "${dependencies[@]}" 'mutt' ) - fi - fi +check_dependencies() { + echo + echo "# Testing for installed programs" + dependencies=('mysql' 'mysqldump') - for i in "${dependencies[@]}"; do - printf '%s ... ' "$i" - if type "$i" &>/dev/null; then - printf 'found.\n' + if [[ "x$CONFIG_multicore" = 'xyes' ]]; then + + if [[ "x$CONFIG_mysql_dump_compression" = 'xbzip2' ]]; then + if type pbzip2 &>/dev/null; then + echo "pbzip2 ... found." else - printf 'not found. Aborting.\n'; - let "E |= $E_missing_deps" - error_handler + CONFIG_multicore='no' # turn off multicore support, since the program isn't there + echo "WARNING: Turning off multicore support, since pbzip2 isn't there." fi - done - echo + elif [[ "x$CONFIG_mysql_dump_compression" = 'xgzip' ]]; then + if type pigz &>/dev/null; then + echo "pigz ... found." + else + CONFIG_multicore='no' # turn off multicore support, since the program isn't there + echo "WARNING: Turning off multicore support, since pigz isn't there." + fi + fi + + else + [[ "x$CONFIG_mysql_dump_compression" = 'xbzip2' ]] && dependencies=("${dependencies[@]}" 'bzip2') + [[ "x$CONFIG_mysql_dump_compression" = 'xgzip' ]] && dependencies=("${dependencies[@]}" 'gzip') + fi + + if [[ "x$CONFIG_mailcontent" = 'xlog' || "x$CONFIG_mailcontent" = 'xquiet' ]]; then + dependencies=("${dependencies[@]}" 'mail') + elif [[ "x$CONFIG_mailcontent" = 'xfiles' ]]; then + dependencies=("${dependencies[@]}" 'mail') + if [[ "x$CONFIG_mail_use_uuencoded_attachments" != 'xyes' ]]; then + dependencies=("${dependencies[@]}" 'mutt') + fi + fi + + for i in "${dependencies[@]}"; do + printf '%s ... ' "$i" + if type "$i" &>/dev/null; then + printf 'found.\n' + else + printf 'not found. Aborting.\n' + let "E |= $E_missing_deps" + error_handler + fi + done + echo } # @info: Get database list and remove excluded ones. @@ -1100,23 +1119,27 @@ parse_databases() { printf "# Parsing databases ... " # bash 3.0 - local i;i=0; + local i + i=0 while read -r; do alldbnames[i++]="$REPLY"; done < <($MYSQL "${mysql_opt[@]}" --batch --skip-column-names -e "show databases") unset i # mkfifo foo || exit; trap 'rm -f foo' EXIT - ((! "${#alldbnames[@]}" )) && { let "E |= $E_db_empty"; error_handler; } + ((!"${#alldbnames[@]}")) && { + let "E |= $E_db_empty" + error_handler + } # -> remove excluded dbs from list for exclude in "${CONFIG_db_exclude[@]}"; do - for i in "${!alldbnames[@]}"; do if [[ "x${alldbnames[$i]}" = "x${exclude}" ]]; then unset 'alldbnames[i]'; fi; done + for i in "${!alldbnames[@]}"; do if [[ "x${alldbnames[$i]}" = "x${exclude}" ]]; then unset 'alldbnames[i]'; fi; done done # <- remove excluded dbs from list # check for empty array lists and copy all dbs - ((! ${#CONFIG_db_names[@]})) && CONFIG_db_names=( "${alldbnames[@]}" ) - ((! ${#CONFIG_db_month_names[@]})) && CONFIG_db_month_names=( "${alldbnames[@]}" ) + ((!${#CONFIG_db_names[@]})) && CONFIG_db_names=("${alldbnames[@]}") + ((!${#CONFIG_db_month_names[@]})) && CONFIG_db_month_names=("${alldbnames[@]}") printf "done.\n" } @@ -1175,9 +1198,9 @@ check_manifest_entry() { [[ ! -e "${manifest_entry_to_check[0]}" ]] && { rm_manifest_entry_by_filename "${manifest_entry_to_check[0]}" 1 && return 1 || return 2; } entry_md5sum="$(md5sum "${manifest_entry_to_check[0]}" | awk '{print $1}')" if [[ "${entry_md5sum}" != "${manifest_entry_to_check[1]}" ]]; then - printf 'g/%s/s//%s/g\nw\nq' "${manifest_entry_to_check[1]}" "${entry_md5sum}" | ed -s "$1" + printf 'g/%s/s//%s/g\nw\nq' "${manifest_entry_to_check[1]}" "${entry_md5sum}" | ed -s "$1" else - return 0 + return 0 fi } @@ -1201,75 +1224,83 @@ check_manifest_entry() { # parse_manifest() { local i n re line line_arr check - unset manifest_array; manifest_array=() + unset manifest_array + manifest_array=() local tmp_md5sum # array ( filename_1, md5sum_1, id_1[, rel_id_1] ), ... ) # reserving 4 members for each entry, thus each filename entry in the array has array key 4(n-1)+1 - (( $CONFIG_debug )) && echo ">>>>>>> Parsing manifest file: $1" + (($CONFIG_debug)) && echo ">>>>>>> Parsing manifest file: $1" n=1 [[ -s "$1" ]] && - while read line - do - # ANY CHANGES INSIDE HERE ON THE MANIFEST_FILE HAVE NO IMPACT ON THE LINES WE LOOP OVER; THE sort COMMAND READS THE FILE ENTIRELY AT THE BEGINNING AND PASSES THE OUTPUT TO THE LOOP - # check if line has expected format, i.e. check against regular expression - re=$'^[^\t]*\tmd5sum\t[^\t]*\tdiff_id\t[A-Za-z0-9]{8}\trel_id\t(0|[A-Za-z0-9]{8})\tdb\t[^\t]*$' - [[ $line =~ $re ]] || { echo "Corrupted line: $line"; manifest_entries_corrupted=( "${manifest_entries_corrupted[@]}" "$1" "$line" ); continue; } - IFS=$'\t' read -ra line_arr <<< "$line" - - # prepare array of the current line - manifest_entry_to_check=() - for ((i=0;i<${#line_arr[@]};i=$i+2)); do - manifest_entry_to_check[i/2]="${line_arr[i]}" - done - # check manifest entry, which uses the array manifest_entry_to_check - check_manifest_entry "$1" - check=$? - case $check in - 1) (( $CONFIG_debug )) && echo "File for manifest entry $line does not exist. Entry removed." - continue # file doesn't exist - removed entry from manifest - break;; - 2) (( $CONFIG_debug )) && echo "File for manifest entry $line does not exist. Failed to remove the entry." - continue # file doesn't exist - tried to remove entry from manifest, but failed - break;; - esac - - # loop through the manifest_array, as it has been filled by now and check if an entry already exists with the same values - for ((i=0;$i<"${#manifest_array[@]}";i=$i+$fields)); do - if [[ "x${manifest_array[i]}" = "x${line_arr[0]}" ]]; then # found entry with the same filename - (( $CONFIG_debug )) && echo "Found multiple entries with the same filename. Removing all but the first-found entry from manifest." - # remove all entries with this filename and add a new one based on the values of the item already in the array - rm_manifest_entry_by_filename "$1" "${manifest_array[i]}" 1 && add_manifest_entry "$1" "${manifest_array[i]}" "${manifest_array[i+3]}" - continue 2 # the original entry, to which we compared, is already in the manifest_array; no matter if this is resolved or not, we don't - # need to add this entry to the manifest_array - elif [[ "x${manifest_array[i+1]}" = "x${line_arr[2]}" ]]; then # found entry with different filename but same md5sum - file copied and renamed?! - if [[ ! -s "${line_arr[0]}" ]]; then # empty file - don't start to compare md5sums ... - (( $CONFIG_debug )) && echo "Found empty file ${line_arr[0]}." - else - (( $CONFIG_debug )) && echo "Found multiple entries with the same md5sum but different filename." - (( $CONFIG_debug )) && echo -e ">> fname_manifest:\t${manifest_array[i]}\t${manifest_array[i+1]}\n>> fname_line:\t\t${line_arr[0]}\t${line_arr[2]}" - if [[ "x${line_arr[6]}" != "x0" ]]; then - if [[ "x${manifest_array[i+3]}" = "x${line_arr[6]}" ]]; then # parent id is the same; TODO inform user of this predicament and suggest solution - manifest_entries_user_action_required=( "${manifest_entries_user_action_required[@]}" "$1" "${manifest_array[i]}" "The file has an identical copy with the same parent id. If you don't know why it exists, it is safe to remove it." ) - continue 2 - else - manifest_entries_user_action_required=( "${manifest_entries_user_action_required[@]}" "$1" "${manifest_array[i]}" "The file has an identical copy with different parent id. This should not happen. Remove the file, which is not the correct follow-up to the previous differential or master backup." ) - continue 2 - fi - fi - fi - fi - done + while read line; do + # ANY CHANGES INSIDE HERE ON THE MANIFEST_FILE HAVE NO IMPACT ON THE LINES WE LOOP OVER; THE sort COMMAND READS THE FILE ENTIRELY AT THE BEGINNING AND PASSES THE OUTPUT TO THE LOOP + # check if line has expected format, i.e. check against regular expression + re=$'^[^\t]*\tmd5sum\t[^\t]*\tdiff_id\t[A-Za-z0-9]{8}\trel_id\t(0|[A-Za-z0-9]{8})\tdb\t[^\t]*$' + [[ $line =~ $re ]] || { + echo "Corrupted line: $line" + manifest_entries_corrupted=("${manifest_entries_corrupted[@]}" "$1" "$line") + continue + } + IFS=$'\t' read -ra line_arr <<<"$line" + + # prepare array of the current line + manifest_entry_to_check=() + for ((i = 0; i < ${#line_arr[@]}; i = $i + 2)); do + manifest_entry_to_check[i / 2]="${line_arr[i]}" + done + # check manifest entry, which uses the array manifest_entry_to_check + check_manifest_entry "$1" + check=$? + case $check in + 1) + (($CONFIG_debug)) && echo "File for manifest entry $line does not exist. Entry removed." + continue # file doesn't exist - removed entry from manifest + break + ;; + 2) + (($CONFIG_debug)) && echo "File for manifest entry $line does not exist. Failed to remove the entry." + continue # file doesn't exist - tried to remove entry from manifest, but failed + break + ;; + esac - # add entry to manifest array - for ((i=0;i<${#line_arr[@]};i=$i+2)); do - manifest_array[(n-1)*${fields}+i/2]="${line_arr[i]}" - #echo "manifest array key $((($n-1)*4+$i/2)) with value ${line_arr[i]}" - done + # loop through the manifest_array, as it has been filled by now and check if an entry already exists with the same values + for ((i = 0; $i < "${#manifest_array[@]}"; i = $i + $fields)); do + if [[ "x${manifest_array[i]}" = "x${line_arr[0]}" ]]; then # found entry with the same filename + (($CONFIG_debug)) && echo "Found multiple entries with the same filename. Removing all but the first-found entry from manifest." + # remove all entries with this filename and add a new one based on the values of the item already in the array + rm_manifest_entry_by_filename "$1" "${manifest_array[i]}" 1 && add_manifest_entry "$1" "${manifest_array[i]}" "${manifest_array[i + 3]}" + continue 2 # the original entry, to which we compared, is already in the manifest_array; no matter if this is resolved or not, we don't + # need to add this entry to the manifest_array + elif [[ "x${manifest_array[i + 1]}" = "x${line_arr[2]}" ]]; then # found entry with different filename but same md5sum - file copied and renamed?! + if [[ ! -s "${line_arr[0]}" ]]; then # empty file - don't start to compare md5sums ... + (($CONFIG_debug)) && echo "Found empty file ${line_arr[0]}." + else + (($CONFIG_debug)) && echo "Found multiple entries with the same md5sum but different filename." + (($CONFIG_debug)) && echo -e ">> fname_manifest:\t${manifest_array[i]}\t${manifest_array[i + 1]}\n>> fname_line:\t\t${line_arr[0]}\t${line_arr[2]}" + if [[ "x${line_arr[6]}" != "x0" ]]; then + if [[ "x${manifest_array[i + 3]}" = "x${line_arr[6]}" ]]; then # parent id is the same; TODO inform user of this predicament and suggest solution + manifest_entries_user_action_required=("${manifest_entries_user_action_required[@]}" "$1" "${manifest_array[i]}" "The file has an identical copy with the same parent id. If you don't know why it exists, it is safe to remove it.") + continue 2 + else + manifest_entries_user_action_required=("${manifest_entries_user_action_required[@]}" "$1" "${manifest_array[i]}" "The file has an identical copy with different parent id. This should not happen. Remove the file, which is not the correct follow-up to the previous differential or master backup.") + continue 2 + fi + fi + fi + fi + done - ((n++)) - done < <(sort -t $'\t' -k"1" "$1") - (( $CONFIG_debug )) && echo "<<<<<<< # manifest entries: $((${#manifest_array[@]}/$fields))" - (( $CONFIG_debug )) && echo "<<<<<<< FINISHED" + # add entry to manifest array + for ((i = 0; i < ${#line_arr[@]}; i = $i + 2)); do + manifest_array[(n - 1) * ${fields} + i / 2]="${line_arr[i]}" + #echo "manifest array key $((($n-1)*4+$i/2)) with value ${line_arr[i]}" + done + + ((n++)) + done < <(sort -t $'\t' -k"1" "$1") + (($CONFIG_debug)) && echo "<<<<<<< # manifest entries: $((${#manifest_array[@]} / $fields))" + (($CONFIG_debug)) && echo "<<<<<<< FINISHED" return 0 } @@ -1278,19 +1309,19 @@ parse_manifest() { get_manifest_entry_by_filename() { local i if [[ "x$2" = "xregexp" ]]; then - for ((i=0;$i<"${#manifest_array[@]}";i=$i+$fields)); do + for ((i = 0; $i < "${#manifest_array[@]}"; i = $i + $fields)); do if [[ "${manifest_array[i]}" =~ $1 ]]; then - manifest_entry=( "${manifest_array[i]}" "${manifest_array[i+1]}" "${manifest_array[i+2]}" "${manifest_array[i+3]}" "${manifest_array[i+4]}" ) - return 0 - break; + manifest_entry=("${manifest_array[i]}" "${manifest_array[i + 1]}" "${manifest_array[i + 2]}" "${manifest_array[i + 3]}" "${manifest_array[i + 4]}") + return 0 + break fi done else - for ((i=0;$i<"${#manifest_array[@]}";i=$i+$fields)); do + for ((i = 0; $i < "${#manifest_array[@]}"; i = $i + $fields)); do if [[ "x${manifest_array[i]}" = "x$1" ]]; then - manifest_entry=( "${manifest_array[i]}" "${manifest_array[i+1]}" "${manifest_array[i+2]}" "${manifest_array[i+3]}" "${manifest_array[i+4]}" ) - return 0 - break; + manifest_entry=("${manifest_array[i]}" "${manifest_array[i + 1]}" "${manifest_array[i + 2]}" "${manifest_array[i + 3]}" "${manifest_array[i + 4]}") + return 0 + break fi done fi @@ -1299,20 +1330,20 @@ get_manifest_entry_by_filename() { get_manifest_entry_by_md5sum() { local i if [[ "x$2" = "xregexp" ]]; then - for ((i=0;$i<"${#manifest_array[@]}";i=$i+$fields)); do - if [[ "${manifest_array[i+1]}" =~ $1 ]]; then - manifest_entry=( "${manifest_array[i]}" "${manifest_array[i+1]}" "${manifest_array[i+2]}" "${manifest_array[i+3]}" "${manifest_array[i+4]}" ) - return 0 - break; + for ((i = 0; $i < "${#manifest_array[@]}"; i = $i + $fields)); do + if [[ "${manifest_array[i + 1]}" =~ $1 ]]; then + manifest_entry=("${manifest_array[i]}" "${manifest_array[i + 1]}" "${manifest_array[i + 2]}" "${manifest_array[i + 3]}" "${manifest_array[i + 4]}") + return 0 + break fi done else - for ((i=0;$i<"${#manifest_array[@]}";i=$i+$fields)); do - if [[ "x${manifest_array[i+1]}" = "x$1" ]]; then - manifest_entry=( "${manifest_array[i]}" "${manifest_array[i+1]}" "${manifest_array[i+2]}" "${manifest_array[i+3]}" "${manifest_array[i+4]}" ) - return 0 - break; - fi + for ((i = 0; $i < "${#manifest_array[@]}"; i = $i + $fields)); do + if [[ "x${manifest_array[i + 1]}" = "x$1" ]]; then + manifest_entry=("${manifest_array[i]}" "${manifest_array[i + 1]}" "${manifest_array[i + 2]}" "${manifest_array[i + 3]}" "${manifest_array[i + 4]}") + return 0 + break + fi done fi return 1 @@ -1320,19 +1351,19 @@ get_manifest_entry_by_md5sum() { get_manifest_entry_by_id() { local i if [[ "x$2" = "xregexp" ]]; then - for ((i=0;$i<"${#manifest_array[@]}";i=$i+$fields)); do - if [[ "${manifest_array[i+2]}" =~ $1 ]]; then - manifest_entry=( "${manifest_array[i]}" "${manifest_array[i+1]}" "${manifest_array[i+2]}" "${manifest_array[i+3]}" "${manifest_array[i+4]}" ) - return 0 - break; + for ((i = 0; $i < "${#manifest_array[@]}"; i = $i + $fields)); do + if [[ "${manifest_array[i + 2]}" =~ $1 ]]; then + manifest_entry=("${manifest_array[i]}" "${manifest_array[i + 1]}" "${manifest_array[i + 2]}" "${manifest_array[i + 3]}" "${manifest_array[i + 4]}") + return 0 + break fi done else - for ((i=0;$i<"${#manifest_array[@]}";i=$i+$fields)); do - if [[ "x${manifest_array[i+2]}" = "x$1" ]]; then - manifest_entry=( "${manifest_array[i]}" "${manifest_array[i+1]}" "${manifest_array[i+2]}" "${manifest_array[i+3]}" "${manifest_array[i+4]}" ) - return 0 - break; + for ((i = 0; $i < "${#manifest_array[@]}"; i = $i + $fields)); do + if [[ "x${manifest_array[i + 2]}" = "x$1" ]]; then + manifest_entry=("${manifest_array[i]}" "${manifest_array[i + 1]}" "${manifest_array[i + 2]}" "${manifest_array[i + 3]}" "${manifest_array[i + 4]}") + return 0 + break fi done fi @@ -1341,19 +1372,19 @@ get_manifest_entry_by_id() { get_manifest_entry_by_rel_id() { local i if [[ "x$2" = "xregexp" ]]; then - for ((i=0;$i<"${#manifest_array[@]}";i=$i+$fields)); do - if [[ "${manifest_array[i+3]}" =~ $1 ]]; then - manifest_entry=( "${manifest_array[i]}" "${manifest_array[i+1]}" "${manifest_array[i+2]}" "${manifest_array[i+3]}" "${manifest_array[i+4]}" ) - return 0 - break; + for ((i = 0; $i < "${#manifest_array[@]}"; i = $i + $fields)); do + if [[ "${manifest_array[i + 3]}" =~ $1 ]]; then + manifest_entry=("${manifest_array[i]}" "${manifest_array[i + 1]}" "${manifest_array[i + 2]}" "${manifest_array[i + 3]}" "${manifest_array[i + 4]}") + return 0 + break fi done else - for ((i=0;$i<"${#manifest_array[@]}";i=$i+$fields)); do - if [[ "x${manifest_array[i+3]}" = "x$1" ]]; then - manifest_entry=( "${manifest_array[i]}" "${manifest_array[i+1]}" "${manifest_array[i+2]}" "${manifest_array[i+3]}" "${manifest_array[i+4]}" ) - return 0 - break; + for ((i = 0; $i < "${#manifest_array[@]}"; i = $i + $fields)); do + if [[ "x${manifest_array[i + 3]}" = "x$1" ]]; then + manifest_entry=("${manifest_array[i]}" "${manifest_array[i + 1]}" "${manifest_array[i + 2]}" "${manifest_array[i + 3]}" "${manifest_array[i + 4]}") + return 0 + break fi done fi @@ -1362,19 +1393,19 @@ get_manifest_entry_by_rel_id() { get_manifest_entry_by_db() { local i if [[ "x$2" = "xregexp" ]]; then - for ((i=0;$i<"${#manifest_array[@]}";i=$i+$fields)); do - if [[ "${manifest_array[i+4]}" =~ $1 ]]; then - manifest_entry=( "${manifest_array[i]}" "${manifest_array[i+1]}" "${manifest_array[i+2]}" "${manifest_array[i+3]}" "${manifest_array[i+4]}" ) - return 0 - break; + for ((i = 0; $i < "${#manifest_array[@]}"; i = $i + $fields)); do + if [[ "${manifest_array[i + 4]}" =~ $1 ]]; then + manifest_entry=("${manifest_array[i]}" "${manifest_array[i + 1]}" "${manifest_array[i + 2]}" "${manifest_array[i + 3]}" "${manifest_array[i + 4]}") + return 0 + break fi done else - for ((i=0;$i<"${#manifest_array[@]}";i=$i+$fields)); do - if [[ "x${manifest_array[i+4]}" = "x$1" ]]; then - manifest_entry=( "${manifest_array[i]}" "${manifest_array[i+1]}" "${manifest_array[i+2]}" "${manifest_array[i+3]}" "${manifest_array[i+4]}" ) - return 0 - break; + for ((i = 0; $i < "${#manifest_array[@]}"; i = $i + $fields)); do + if [[ "x${manifest_array[i + 4]}" = "x$1" ]]; then + manifest_entry=("${manifest_array[i]}" "${manifest_array[i + 1]}" "${manifest_array[i + 2]}" "${manifest_array[i + 3]}" "${manifest_array[i + 4]}") + return 0 + break fi done fi @@ -1387,22 +1418,23 @@ get_manifest_entry_by_db() { # 0: all fine, match is in array 'manifest_entry' get_latest_manifest_entry_for_db() { local db_array newarray i - for ((i=0;$i<"${#manifest_array[@]}";i=$i+$fields)); do - if (( $2 )); then # latest differential or master backup, i.e. just take the latest one! - if [[ "x${manifest_array[i+4]}" = "x$1" ]]; then - db_array=( "${db_array[@]}" "${manifest_array[i]}" ) - fi - else # latest master backup, pid=0 - if [[ "x${manifest_array[i+4]}" = "x$1" && "x${manifest_array[i+3]}" = "x0" ]]; then - db_array=( "${db_array[@]}" "${manifest_array[i]}") - fi - fi + for ((i = 0; $i < "${#manifest_array[@]}"; i = $i + $fields)); do + if (($2)); then # latest differential or master backup, i.e. just take the latest one! + if [[ "x${manifest_array[i + 4]}" = "x$1" ]]; then + db_array=("${db_array[@]}" "${manifest_array[i]}") + fi + else # latest master backup, pid=0 + if [[ "x${manifest_array[i + 4]}" = "x$1" && "x${manifest_array[i + 3]}" = "x0" ]]; then + db_array=("${db_array[@]}" "${manifest_array[i]}") + fi + fi done - if (( "${#db_array[@]}" == 0 )); then return 2; + if (("${#db_array[@]}" == 0)); then + return 2 else - #newarray=(); while IFS= read -r -d '' line; do newarray+=("$line"); done < <(printf '%s\0' "${db_array[@]}" | sort -z) - get_manifest_entry_by_filename "${db_array[@]:(-1)}" # last entry of db_array, has, due to the way sort works, to be the latest one - return $? + #newarray=(); while IFS= read -r -d '' line; do newarray+=("$line"); done < <(printf '%s\0' "${db_array[@]}" | sort -z) + get_manifest_entry_by_filename "${db_array[@]:(-1)}" # last entry of db_array, has, due to the way sort works, to be the latest one + return $? fi } @@ -1413,28 +1445,28 @@ get_latest_manifest_entry_for_db() { # param3=0 -> parse manifest # rm_manifest_entry_by_filename() { - lock_manifest "$1" && awk -F"\t" -v v="$2" '$1 != v' "$1" > "$1".tmp && mv "$1".tmp "$1" && unlock_manifest "$1" || return 1 - (( "$3" )) || parse_manifest "$1" + lock_manifest "$1" && awk -F"\t" -v v="$2" '$1 != v' "$1" >"$1".tmp && mv "$1".tmp "$1" && unlock_manifest "$1" || return 1 + (("$3")) || parse_manifest "$1" return 0 } rm_manifest_entry_by_md5sum() { - lock_manifest "$1" && awk -F"\t" -v v="$2" '$3 != v' "$1" > "$1".tmp && mv "$1".tmp "$1" && unlock_manifest "$1" || return 1 - (( "$3" )) || parse_manifest "$1" + lock_manifest "$1" && awk -F"\t" -v v="$2" '$3 != v' "$1" >"$1".tmp && mv "$1".tmp "$1" && unlock_manifest "$1" || return 1 + (("$3")) || parse_manifest "$1" return 0 } rm_manifest_entry_by_id() { - lock_manifest "$1" && awk -F"\t" -v v="$2" '$5 != v' "$1" > "$1".tmp && mv "$1".tmp "$1" && unlock_manifest "$1" || return 1 - (( "$3" )) || parse_manifest "$1" + lock_manifest "$1" && awk -F"\t" -v v="$2" '$5 != v' "$1" >"$1".tmp && mv "$1".tmp "$1" && unlock_manifest "$1" || return 1 + (("$3")) || parse_manifest "$1" return 0 } rm_manifest_entry_by_rel_id() { - lock_manifest "$1" && awk -F"\t" -v v="$2" '$7 != v' "$1" > "$1".tmp && mv "$1".tmp "$1" && unlock_manifest "$1" || return 1 - (( "$3" )) || parse_manifest "$1" + lock_manifest "$1" && awk -F"\t" -v v="$2" '$7 != v' "$1" >"$1".tmp && mv "$1".tmp "$1" && unlock_manifest "$1" || return 1 + (("$3")) || parse_manifest "$1" return 0 } rm_manifest_entry_by_db() { - lock_manifest "$1" && awk -F"\t" -v v="$2" '$9 != v' "$1" > "$1".tmp && mv "$1".tmp "$1" && unlock_manifest "$1" || return 1 - (( "$3" )) || parse_manifest "$1" + lock_manifest "$1" && awk -F"\t" -v v="$2" '$9 != v' "$1" >"$1".tmp && mv "$1".tmp "$1" && unlock_manifest "$1" || return 1 + (("$3")) || parse_manifest "$1" return 0 } @@ -1454,16 +1486,16 @@ add_manifest_entry() { #id="$(echo $filename | sed -re 's/.*_[0-9]{2}h[0-9]{2}m_([^\.]*)\..*/\1/')" md5sum="$(md5sum "$filename" | awk '{print $1}')" if [[ "x$parent_id" = 'x' ]]; then - echo -e "${filename}\tmd5sum\t${md5sum}\tdiff_id\t${id}\trel_id\t0\tdb\t${db}" >> "$1" + echo -e "${filename}\tmd5sum\t${md5sum}\tdiff_id\t${id}\trel_id\t0\tdb\t${db}" >>"$1" else - echo -e "${filename}\tmd5sum\t${md5sum}\tdiff_id\t${id}\trel_id\t${parent_id}\tdb\t${db}" >> "$1" + echo -e "${filename}\tmd5sum\t${md5sum}\tdiff_id\t${id}\trel_id\t${parent_id}\tdb\t${db}" >>"$1" fi unlock_manifest "$1" || return 1 } # @info: Echos number of manifest entries. num_manifest_entries() { - echo "$((${#manifest_array[@]}/$fields))" + echo "$((${#manifest_array[@]} / $fields))" } # @info: Test if a value is in the array testarray @@ -1472,815 +1504,882 @@ num_manifest_entries() { # @return 0 if a match was found, otherwise 1 in_array() { local j - for ((j=0;j<"${#testarray[@]}";j++)); do - if [[ "x${testarray[j]}" = "x$1" ]]; then - in_array_index=$j - return 0 - fi + for ((j = 0; j < "${#testarray[@]}"; j++)); do + if [[ "x${testarray[j]}" = "x$1" ]]; then + in_array_index=$j + return 0 + fi done return 1 } # @param: clear(0/1), meta_information, list_value1, list_value2, ... extended_select() { - local a c k m i r r_number meta_information choice selection do_clear - meta_information="$2" - do_clear="$1" - shift 2 - declare -a list=("$@") - selection=() - # BEGIN _select_filenames - #tput sc - while true; do - if (( $do_clear )); then - clear - else - : #tput rc - fi - declare -a testarray=("${selection[@]}") - echo "Selection for <$meta_information>" - echo "Notation: 1,2-4,-5,-6-9 or * or -* ('-' will remove selections)." - # print options - - for ((i=0;i<"${#list[@]}";i++)); do - if in_array $i; then - echo -e "$i) [+]\t${list[i]}" - else - echo -e "$i) [ ]\t${list[i]}" - fi - done - - echo -e "$i)\tDONE" - done_id=$i - - min=0 - max=${#list[@]} # we have to account for the last possible number of DONE - # evaluate response - while true; do - printf '#? ' - read choice - r='^((-?[0-9]+(-[0-9]+)?,)*-?[0-9]+(-[0-9]+)?|-?\*)$' - [[ $choice =~ $r ]] || continue - if [[ "x$choice" = 'x*' ]]; then - unset m - for ((m=0;m<"${#list[@]}";m++)); do - selection=("${selection[@]}" "$m") - done - continue 2 - elif [[ "x$choice" = 'x-*' ]]; then - selection=() - continue 2 - else - unset string num1 num2 op op_rm - r_number='^[0-9]$' - - # BEGIN process_choice - for ((a=0;a<${#choice};a++)) - do - c="${choice:a:1}" - declare -a testarray=("${selection[@]}") - - if (( ${#string} == 0 )) && [[ "x$c" = "x-" ]] && ! (($op)); then - op_rm=1 - continue - elif [[ $c =~ $r ]]; then - string=${string}"$c" - if (( $a == (${#choice}-1) )); then # last character - # we have a A-B case - if (($op)); then - num2="$string" - unset k - for ((k=$num1;k<=$num2;k++)); do - (( $k >= $min )) && (( $k <= $max )) || continue - if ! in_array $k; then - selection=("${selection[@]}" $k) - else - if (( $op_rm )); then - new_array=() - for ((m="$((${#selection[@]}-1))";m>=0;m--)); do - if [[ "x${selection[m]}" != "x$k" ]]; then - new_array=("${new_array[@]}" "${selection[m]}") - fi - done - declare -a selection=("${new_array[@]}") - fi - fi - done - unset op op_rm num1 num2 string - continue - else - (( $string >= $min )) && (( $string <= $max )) || continue - if ! in_array "$string"; then - selection=("${selection[@]}" "$string") - else - if (($op_rm)); then - unset m - new_array=() - for ((m=0;m<"${#selection[@]}";m++)); do - if [[ "x${selection[m]}" != "x$string" ]]; then - new_array=("${new_array[@]}" "${selection[m]}") - fi - done - declare -a selection=("${new_array[@]}") - fi - fi - fi - else - continue - fi - elif [[ "x$c" = "x-" ]]; then - num1="$string" - unset string - op=1 - if (( $a == (${#choice}-1) )); then - break - else - continue - fi - elif [[ "x$c" = "x," ]]; then - # we have a A-B case - if (($op)); then - num2="$string" - unset k - for ((k=$num1;k<=$num2;k++)); do - (( $k >= $min )) && (( $k <= $max )) || continue - if ! in_array $k; then - selection=("${selection[@]}" $k) - else - if (( $op_rm )); then - unset m - new_array=() - for ((m=0;m<"${#selection[@]}";m++)); do - if [[ "x${selection[m]}" != "x$k" ]]; then - new_array=("${new_array[@]}" "${selection[m]}") - fi - done - declare -a selection=("${new_array[@]}") - fi - fi - done - unset op op_rm num1 num2 string - continue - else # it's just a single number - (( $string >= $min )) && (( $string <= $max )) || { unset op op_rm num1 num2 string; continue; } - if ! in_array "$string"; then - selection=("${selection[@]}" "$string") - else - if (($op_rm)); then - unset m - new_array=() - for ((m=0;m<"${#selection[@]}";m++)); do - if [[ "x${selection[m]}" != "x$string" ]]; then - new_array=("${new_array[@]}" "${selection[m]}") - fi - done - declare -a selection=("${new_array[@]}") - fi - fi - unset op op_rm num1 num2 string - continue - fi - else - continue 2; # this should not happen - fi - done - # END process_choice - - declare -a testarray=("${selection[@]}") - if in_array "$done_id"; then - break 2 - else - continue 2 - fi - fi - done - done - extended_select_return=() - extended_select_return_id=() - for i in "${selection[@]}"; do - [[ "x$i" != "x$done_id" ]] && { extended_select_return=("${extended_select_return[@]}" "${list[i]}"); extended_select_return_id=("${extended_select_return_id[@]}" "$i"); } - done + local a c k m i r r_number meta_information choice selection do_clear + meta_information="$2" + do_clear="$1" + shift 2 + declare -a list=("$@") + selection=() + # BEGIN _select_filenames + #tput sc + while true; do + if (($do_clear)); then + clear + else + : #tput rc + fi + declare -a testarray=("${selection[@]}") + echo "Selection for <$meta_information>" + echo "Notation: 1,2-4,-5,-6-9 or * or -* ('-' will remove selections)." + # print options + + for ((i = 0; i < "${#list[@]}"; i++)); do + if in_array $i; then + echo -e "$i) [+]\t${list[i]}" + else + echo -e "$i) [ ]\t${list[i]}" + fi + done + + echo -e "$i)\tDONE" + done_id=$i + + min=0 + max=${#list[@]} # we have to account for the last possible number of DONE + # evaluate response + while true; do + printf '#? ' + read choice + r='^((-?[0-9]+(-[0-9]+)?,)*-?[0-9]+(-[0-9]+)?|-?\*)$' + [[ $choice =~ $r ]] || continue + if [[ "x$choice" = 'x*' ]]; then + unset m + for ((m = 0; m < "${#list[@]}"; m++)); do + selection=("${selection[@]}" "$m") + done + continue 2 + elif [[ "x$choice" = 'x-*' ]]; then + selection=() + continue 2 + else + unset string num1 num2 op op_rm + r_number='^[0-9]$' + + # BEGIN process_choice + for ((a = 0; a < ${#choice}; a++)); do + c="${choice:a:1}" + declare -a testarray=("${selection[@]}") + + if ((${#string} == 0)) && [[ "x$c" = "x-" ]] && ! (($op)); then + op_rm=1 + continue + elif [[ $c =~ $r ]]; then + string=${string}"$c" + if (($a == (${#choice} - 1))); then # last character + # we have a A-B case + if (($op)); then + num2="$string" + unset k + for ((k = $num1; k <= $num2; k++)); do + (($k >= $min)) && (($k <= $max)) || continue + if ! in_array $k; then + selection=("${selection[@]}" $k) + else + if (($op_rm)); then + new_array=() + for ((m = "$((${#selection[@]} - 1))"; m >= 0; m--)); do + if [[ "x${selection[m]}" != "x$k" ]]; then + new_array=("${new_array[@]}" "${selection[m]}") + fi + done + declare -a selection=("${new_array[@]}") + fi + fi + done + unset op op_rm num1 num2 string + continue + else + (($string >= $min)) && (($string <= $max)) || continue + if ! in_array "$string"; then + selection=("${selection[@]}" "$string") + else + if (($op_rm)); then + unset m + new_array=() + for ((m = 0; m < "${#selection[@]}"; m++)); do + if [[ "x${selection[m]}" != "x$string" ]]; then + new_array=("${new_array[@]}" "${selection[m]}") + fi + done + declare -a selection=("${new_array[@]}") + fi + fi + fi + else + continue + fi + elif [[ "x$c" = "x-" ]]; then + num1="$string" + unset string + op=1 + if (($a == (${#choice} - 1))); then + break + else + continue + fi + elif [[ "x$c" = "x," ]]; then + # we have a A-B case + if (($op)); then + num2="$string" + unset k + for ((k = $num1; k <= $num2; k++)); do + (($k >= $min)) && (($k <= $max)) || continue + if ! in_array $k; then + selection=("${selection[@]}" $k) + else + if (($op_rm)); then + unset m + new_array=() + for ((m = 0; m < "${#selection[@]}"; m++)); do + if [[ "x${selection[m]}" != "x$k" ]]; then + new_array=("${new_array[@]}" "${selection[m]}") + fi + done + declare -a selection=("${new_array[@]}") + fi + fi + done + unset op op_rm num1 num2 string + continue + else # it's just a single number + (($string >= $min)) && (($string <= $max)) || { + unset op op_rm num1 num2 string + continue + } + if ! in_array "$string"; then + selection=("${selection[@]}" "$string") + else + if (($op_rm)); then + unset m + new_array=() + for ((m = 0; m < "${#selection[@]}"; m++)); do + if [[ "x${selection[m]}" != "x$string" ]]; then + new_array=("${new_array[@]}" "${selection[m]}") + fi + done + declare -a selection=("${new_array[@]}") + fi + fi + unset op op_rm num1 num2 string + continue + fi + else + continue 2 # this should not happen + fi + done + # END process_choice + + declare -a testarray=("${selection[@]}") + if in_array "$done_id"; then + break 2 + else + continue 2 + fi + fi + done + done + extended_select_return=() + extended_select_return_id=() + for i in "${selection[@]}"; do + [[ "x$i" != "x$done_id" ]] && { + extended_select_return=("${extended_select_return[@]}" "${list[i]}") + extended_select_return_id=("${extended_select_return_id[@]}" "$i") + } + done } # END _functions - # BEGIN _methods # @info: Backup method -method_backup () { - manifest_entries_corrupted=() - manifest_entries_user_action_required=() - - - # END __FUNCTIONS - ############################################################################################################## - # BEGIN __STARTUP - - load_default_config - - trap mail_cleanup EXIT SIGHUP SIGINT SIGQUIT SIGTERM - if [[ -r "${CONFIG_configfile}" ]]; then source "${CONFIG_configfile}"; else let "N |= $N_config_file_missing"; fi - if (( $opt_flag_config_file )); then if [[ -r "${opt_config_file}" ]]; then source "${opt_config_file}"; let "N |= $N_arg_conffile_parsed"; else let "N |= $N_arg_conffile_unreadable"; fi; else let "N |= $N_too_many_args"; fi - - # load mysql commands - mysql_commands - - (( $CONFIG_dryrun )) && { - echo "NOTE: We are dry-running. That means, that the script just shows you what it would do, if it were operating normally." - echo "THE PRINTED COMMANDS CAN'T BE COPIED AND EXECUTED IF THERE ARE SPECIAL CHARACTERS, SPACES, ETC. IN THERE THAT WOULD NEED TO BE PROPERLY QUOTED IN ORDER TO WORK. THESE WERE CORRECTLY QUOTED FOR THE OUTPUT COMMAND, BUT CAN'T BE SEEN NOW." - echo - } - - export LC_ALL=C - PROGNAME=`basename $0` - PATH=${PATH}:/usr/local/bin:/usr/bin:/bin:/usr/local/mysql/bin - version=3.0 - fields=5 # manifest fields - - directory_checks_enable_logging - cleanup_latest - set_datetime_vars - check_dependencies # check for required programs - parse_configuration # parse configuration and set variables appropriately - - - # END __STARTUP - #-------------------------------------------------------------------------------------------------------------------------------------- - # BEGIN __PREPARE - - backupfiles=() - parse_databases - - # debug output of variables - (( $CONFIG_debug )) && { echo; echo "# DEBUG: printing all current variables"; declare -p | egrep -o '.* (CONFIG_[a-z_]*|opt|mysql_opt|opt_dbstatus|opt_fullschema)=.*'; echo; } - (( $CONFIG_debug )) && { echo "DEBUG: before pre-backup"; ( IFS=,; echo "DEBUG: CONFIG_db_names '${CONFIG_db_names[*]}'" ); ( IFS=,; echo "DEBUG: CONFIG_db_month_names '${CONFIG_db_month_names[*]}'" );} - - - # END __PREPARE - #-------------------------------------------------------------------------------------------------------------------------------------- - # BEGIN __MAIN - - ### filename formats - ## - ## example date values: - # 14'th of August (08) 2011 - # week number: 32 - # Sunday (date_dayno_of_week: 7) - ## - ## separate db's: - # monthly_DBNAME_2011-08-14_18h12m_August.sql(.enc).{gz,bzip2} - # weekly_DBNAME_2011-08-14_18h12m_32.sql(.enc).{gz,bzip2} - # daily_DBNAME_2011-08-14_18h12m_7.sql(.enc).{gz,bzip2} - ## all-databases: - # monthly_all-databases_DBNAME_2011-08-14_18h12m_August.sql(.enc).{gz,bzip2} - # weekly_all-databases_DBNAME_2011-08-14_18h12m_32.sql(.enc).{gz,bzip2} - # daily_all-databases_DBNAME_2011-08-14_18h12m_7.sql(.enc).{gz,bzip2} - - echo "======================================================================" - echo "AutoMySQLBackup version ${version}" - echo "http://sourceforge.net/projects/automysqlbackup/" - echo - echo "Backup of Database Server - ${CONFIG_mysql_dump_host_friendly:-$CONFIG_mysql_dump_host}" - ( IFS=,; echo "Databases - ${CONFIG_db_names[*]}" ) - ( IFS=,; echo "Databases (monthly) - ${CONFIG_db_month_names[*]}" ) - echo "======================================================================" - - - # -> preback commands - if [[ "${CONFIG_prebackup}" ]]; then - echo "======================================================================" - echo "Prebackup command output." - echo - source ${CONFIG_prebackup} - echo - echo "======================================================================" - echo - fi - # <- preback commands - - # -> backup local files - if [[ "${CONFIG_backup_local_files[@]}" ]] && [[ ${CONFIG_do_weekly} != 0 && ${date_dayno_of_week} = ${CONFIG_do_weekly} ]] && (shopt -s nullglob dotglob; f=("${CONFIG_backup_dir}/backup_local_files/bcf_weekly_${date_stamp}_"[0-9][0-9]"h"[0-9][0-9]"m_${date_weekno}.tar${suffix}"); ((! ${#f[@]}))); then - echo "======================================================================" - echo "Backup local files. Doing this weekly on CONFIG_do_weekly." - echo - backup_local_files "${CONFIG_backup_dir}/backup_local_files/bcf_weekly_${datetimestamp}_${date_weekno}.tar" - tmp_flags=$?; var=; - if (( $? == 0 )); then - echo "success!" - backupfiles=( "${backupfiles[@]}" "${CONFIG_backup_dir}/backup_local_files/bcf_weekly_${datetimestamp}_${date_weekno}.tar" ) - else - let "E |= $E_backup_local_failed" - echo "failed!" - fi - echo - echo "======================================================================" - echo - fi - # <- backup local files - - # -> dump full schema - if [[ "${CONFIG_mysql_dump_full_schema}" = 'yes' ]]; then - echo "======================================================================" - echo "Dump full schema." - echo - - # monthly - if (( ${CONFIG_do_monthly} != 0 && (${date_day_of_month} == ${CONFIG_do_monthly} || $date_day_of_month == $date_lastday_of_this_month && $date_lastday_of_this_month < ${CONFIG_do_monthly}) )) && (shopt -s nullglob dotglob; f=("${CONFIG_backup_dir}/fullschema/fullschema_monthly_${date_stamp}_"[0-9][0-9]"h"[0-9][0-9]"m_${date_month}.sql${suffix}"); ((! ${#f[@]}))); then - fullschema "${CONFIG_backup_dir}/fullschema/fullschema_monthly_${datetimestamp}_${date_month}.sql" - if (( $? == 0 )); then - echo "Rotating $(( ${CONFIG_rotation_monthly}/31 )) month backups for ${mdb}" - if (( $CONFIG_dryrun )); then - find "${CONFIG_backup_dir}/fullschema" -mtime +"${CONFIG_rotation_monthly}" -type f -name 'fullschema_monthly*' -exec echo "dry-running: rm" {} \; - else - find "${CONFIG_backup_dir}/fullschema" -mtime +"${CONFIG_rotation_monthly}" -type f -name 'fullschema_monthly*' -exec rm {} \; - fi - files_postprocessing "${CONFIG_backup_dir}/fullschema/fullschema_monthly_${datetimestamp}_${date_month}.sql${suffix}" - tmp_flags=$?; var=; (( $tmp_flags & $flags_files_postprocessing_success_encrypt )) && var=.enc - backupfiles=( "${backupfiles[@]}" "${CONFIG_backup_dir}/fullschema/fullschema_monthly_${datetimestamp}_${date_month}.sql${suffix}${var}" ) - else - let "E |= $E_dump_fullschema_failed" - fi - fi - - # weekly - if [[ ${CONFIG_do_weekly} != 0 && ${date_dayno_of_week} = ${CONFIG_do_weekly} ]] && (shopt -s nullglob dotglob; f=("${CONFIG_backup_dir}/fullschema/fullschema_weekly_${date_stamp}_"[0-9][0-9]"h"[0-9][0-9]"m_${date_weekno}.sql${suffix}"); ((! ${#f[@]}))); then - fullschema "${CONFIG_backup_dir}/fullschema/fullschema_weekly_${datetimestamp}_${date_weekno}.sql" - if (( $? == 0 )); then - echo "Rotating $(( ${CONFIG_rotation_monthly}/31 )) month backups for ${mdb}" - if (( $CONFIG_dryrun )); then - find "${CONFIG_backup_dir}/fullschema" -mtime +"${CONFIG_rotation_weekly}" -type f -name 'fullschema_weekly*' -exec echo "dry-running: rm" {} \; - else - find "${CONFIG_backup_dir}/fullschema" -mtime +"${CONFIG_rotation_weekly}" -type f -name 'fullschema_weekly*' -exec rm {} \; - fi - files_postprocessing "${CONFIG_backup_dir}/fullschema/fullschema_weekly_${datetimestamp}_${date_weekno}.sql${suffix}" - tmp_flags=$?; var=; (( $tmp_flags & $flags_files_postprocessing_success_encrypt )) && var=.enc - backupfiles=( "${backupfiles[@]}" "${CONFIG_backup_dir}/fullschema/fullschema_weekly_${datetimestamp}_${date_weekno}.sql${suffix}${var}" ) - else - let "E |= $E_dump_fullschema_failed" - fi - fi - - # daily - fullschema "${CONFIG_backup_dir}/fullschema/fullschema_daily_${datetimestamp}_${date_day_of_week}.sql" - if (( $? == 0 )); then - echo "Rotating $(( ${CONFIG_rotation_monthly}/31 )) month backups for ${mdb}" - if (( $CONFIG_dryrun )); then - find "${CONFIG_backup_dir}/fullschema" -mtime +"${CONFIG_rotation_daily}" -type f -name 'fullschema_daily*' -exec echo "dry-running: rm" {} \; - else - find "${CONFIG_backup_dir}/fullschema" -mtime +"${CONFIG_rotation_daily}" -type f -name 'fullschema_daily*' -exec rm {} \; - fi - files_postprocessing "${CONFIG_backup_dir}/fullschema/fullschema_daily_${datetimestamp}_${date_day_of_week}.sql${suffix}" - tmp_flags=$?; var=; (( $tmp_flags & $flags_files_postprocessing_success_encrypt )) && var=.enc - backupfiles=( "${backupfiles[@]}" "${CONFIG_backup_dir}/fullschema/fullschema_daily_${datetimestamp}_${date_day_of_week}.sql${suffix}${var}" ) - else - let "E |= $E_dump_fullschema_failed" - fi - echo - echo "======================================================================" - echo - - fi - # <- dump full schema - - # -> dump status - if [[ "${CONFIG_mysql_dump_dbstatus}" = 'yes' ]]; then - echo "======================================================================" - echo "Dump status." - echo - - # monthly - if (( ${CONFIG_do_monthly} != 0 && (${date_day_of_month} == ${CONFIG_do_monthly} || $date_day_of_month == $date_lastday_of_this_month && $date_lastday_of_this_month < ${CONFIG_do_monthly}) )) && (shopt -s nullglob dotglob; f=("${CONFIG_backup_dir}/status/status_monthly_${date_stamp}_"[0-9][0-9]"h"[0-9][0-9]"m_${date_month}.txt${suffix}"); ((! ${#f[@]}))); then - dbstatus "${CONFIG_backup_dir}/status/status_monthly_${datetimestamp}_${date_month}.txt" - if (( $? == 0 )); then - echo "Rotating $(( ${CONFIG_rotation_monthly}/31 )) month backups for ${mdb}" - if (( $CONFIG_dryrun )); then - find "${CONFIG_backup_dir}/status" -mtime +"${CONFIG_rotation_monthly}" -type f -name 'status_monthly*' -exec echo "dry-running: rm" {} \; - else - find "${CONFIG_backup_dir}/status" -mtime +"${CONFIG_rotation_monthly}" -type f -name 'status_monthly*' -exec rm {} \; - fi - files_postprocessing "${CONFIG_backup_dir}/status/status_monthly_${datetimestamp}_${date_month}.txt${suffix}" - tmp_flags=$?; var=; (( $tmp_flags & $flags_files_postprocessing_success_encrypt )) && var=.enc - backupfiles=( "${backupfiles[@]}" "${CONFIG_backup_dir}/status/status_monthly_${datetimestamp}_${date_month}.txt${suffix}${var}" ) - else - let "E |= $E_dump_status_failed" - fi - fi - - # weekly - if [[ ${CONFIG_do_weekly} != 0 && ${date_dayno_of_week} = ${CONFIG_do_weekly} ]] && (shopt -s nullglob dotglob; f=("${CONFIG_backup_dir}/status/status_weekly_${date_stamp}_"[0-9][0-9]"h"[0-9][0-9]"m_${date_weekno}.txt${suffix}"); ((! ${#f[@]}))); then - dbstatus "${CONFIG_backup_dir}/status/status_weekly_${datetimestamp}_${date_weekno}.txt" - if (( $? == 0 )); then - echo "Rotating $(( ${CONFIG_rotation_monthly}/31 )) month backups for ${mdb}" - if (( $CONFIG_dryrun )); then - find "${CONFIG_backup_dir}/status" -mtime +"${CONFIG_rotation_weekly}" -type f -name 'status_weekly*' -exec echo "dry-running: rm" {} \; - else - find "${CONFIG_backup_dir}/status" -mtime +"${CONFIG_rotation_weekly}" -type f -name 'status_weekly*' -exec rm {} \; - fi - files_postprocessing "${CONFIG_backup_dir}/status/status_weekly_${datetimestamp}_${date_weekno}.txt${suffix}" - tmp_flags=$?; var=; (( $tmp_flags & $flags_files_postprocessing_success_encrypt )) && var=.enc - backupfiles=( "${backupfiles[@]}" "${CONFIG_backup_dir}/status/status_weekly_${datetimestamp}_${date_weekno}.txt${suffix}${var}" ) - else - let "E |= $E_dump_status_failed" - fi - fi - - # daily - dbstatus "${CONFIG_backup_dir}/status/status_daily_${datetimestamp}_${date_day_of_week}.txt" - if (( $? == 0 )); then - echo "Rotating $(( ${CONFIG_rotation_monthly}/31 )) month backups for ${mdb}" - if (( $CONFIG_dryrun )); then - find "${CONFIG_backup_dir}/status" -mtime +"${CONFIG_rotation_daily}" -type f -name 'status_daily*' -exec echo "dry-running: rm" {} \; - else - find "${CONFIG_backup_dir}/status" -mtime +"${CONFIG_rotation_daily}" -type f -name 'status_daily*' -exec rm {} \; - fi - files_postprocessing "${CONFIG_backup_dir}/status/status_daily_${datetimestamp}_${date_day_of_week}.txt${suffix}" - tmp_flags=$?; var=; (( $tmp_flags & $flags_files_postprocessing_success_encrypt )) && var=.enc - backupfiles=( "${backupfiles[@]}" "${CONFIG_backup_dir}/status/status_daily_${datetimestamp}_${date_day_of_week}.txt${suffix}${var}" ) - else - let "E |= $E_dump_status_failed" - fi - echo - echo "======================================================================" - echo - - fi - # <- dump status - - - # -> BACKUP DATABASES - echo "Backup Start Time `date`" - echo "======================================================================" - - ## <- monthly backup, unique per month - if (( ${CONFIG_do_monthly} != 0 && (${date_day_of_month} == ${CONFIG_do_monthly} || $date_day_of_month == $date_lastday_of_this_month && $date_lastday_of_this_month < ${CONFIG_do_monthly}) )); then - echo "Monthly Backup ..." - echo - - subfolder="monthly" - prefix="monthly_" - midfix="_${date_month}" - extension=".sql" - rotation="${CONFIG_rotation_monthly}" - rotation_divisor="31" - rotation_string="month" - - if [[ "${CONFIG_mysql_dump_use_separate_dirs}" = "yes" ]]; then - for db in "${CONFIG_db_month_names[@]}"; do - echo "Monthly Backup of Database ( ${db} )" - (shopt -s nullglob dotglob; f=("${CONFIG_backup_dir}/${subfolder}/${db}/${prefix}${db}_${date_stamp}_"[0-9][0-9]"h"[0-9][0-9]"m${midfix}${extension}${suffix}"); ((${#f[@]}))) && continue - process_dbs "$subfolder" "$prefix" "$midfix" "$extension" "$rotation" "$rotation_divisor" "$rotation_string" 0 "$db" - echo ---------------------------------------------------------------------- - done - else - echo "Monthly backup of databases ( ${CONFIG_db_month_names[@]} )." - (shopt -s nullglob dotglob; f=("${CONFIG_backup_dir}/${subfolder}/${prefix}all-databases_${date_stamp}_"[0-9][0-9]"h"[0-9][0-9]"m${midfix}${extension}${suffix}"); ((${#f[@]}))) && - process_dbs "$subfolder" "$prefix" "$midfix" "$extension" "$rotation" "$rotation_divisor" "$rotation_string" 1 "${CONFIG_db_month_names[@]}" - echo "----------------------------------------------------------------------" - fi - fi - ## <- monthly backup - - ## <- weekly backup, unique per week - if (( ${CONFIG_do_weekly} != 0 && ${date_dayno_of_week} == ${CONFIG_do_weekly} )); then - echo "Weekly Backup ..." - echo - - subfolder="weekly" - prefix="weekly_" - midfix="_${date_weekno}" - extension=".sql" - rotation="${CONFIG_rotation_weekly}" - rotation_divisor="7" - rotation_string="week" - if [[ "${CONFIG_mysql_dump_use_separate_dirs}" = "yes" ]]; then - for db in "${CONFIG_db_names[@]}"; do - echo "Weekly Backup of Database ( ${db} )" - (shopt -s nullglob dotglob; f=("${CONFIG_backup_dir}/${subfolder}/${db}/${prefix}${db}_${date_stamp}_"[0-9][0-9]"h"[0-9][0-9]"m${midfix}${extension}${suffix}"); ((${#f[@]}))) && continue - process_dbs "$subfolder" "$prefix" "$midfix" "$extension" "$rotation" "$rotation_divisor" "$rotation_string" 0 "$db" - echo "----------------------------------------------------------------------" - done - else - echo "Weekly backup of databases ( ${CONFIG_db_names[@]} )." - (shopt -s nullglob dotglob; f=("${CONFIG_backup_dir}/${subfolder}/${prefix}all-databases_${date_stamp}_"[0-9][0-9]"h"[0-9][0-9]"m${midfix}${extension}${suffix}"); ((${#f[@]}))) && - process_dbs "$subfolder" "$prefix" "$midfix" "$extension" "$rotation" "$rotation_divisor" "$rotation_string" 1 "${CONFIG_db_names[@]}" - echo "----------------------------------------------------------------------" - fi - fi - ## <- weekly backup - - ## -> daily backup, test (( 1 )) is always true, just creates a grouping for Kate, which can be closed ^^ - if (( 1 )); then - echo "Daily Backup ..." - echo - - subfolder="daily" - prefix="daily_" - midfix="_${date_day_of_week}" - extension=".sql" - rotation="${CONFIG_rotation_daily}" - rotation_divisor="1" - rotation_string="day" - - if [[ "${CONFIG_mysql_dump_use_separate_dirs}" = "yes" ]]; then - for db in "${CONFIG_db_names[@]}"; do - echo "Daily Backup of Database ( ${db} )" - process_dbs "$subfolder" "$prefix" "$midfix" "$extension" "$rotation" "$rotation_divisor" "$rotation_string" 0 "$db" - echo "----------------------------------------------------------------------" - done - else - echo "Daily backup of databases ( ${CONFIG_db_names[@]} )." - process_dbs "$subfolder" "$prefix" "$midfix" "$extension" "$rotation" "$rotation_divisor" "$rotation_string" 1 "${CONFIG_db_names[@]}" - echo "----------------------------------------------------------------------" - fi - fi - ## <- daily backup - - echo - echo "Backup End Time `date`" - echo "======================================================================" - # <- BACKUP DATABASES - - - # -> clean latest filenames - [[ "${CONFIG_mysql_dump_latest_clean_filenames}" = 'yes' ]] && find "${CONFIG_backup_dir}"/latest/ -type f -exec bash -c 'remove_datetimeinfo "$@"' -- {} \; - # <- clean latest filenames - - # -> finished information - echo "Total disk space used for backup storage..." - echo "Size - Location" - echo `du -hs $OS_du_extra_option "${CONFIG_backup_dir}"` - echo - echo "======================================================================" - # <- finished information - - # -> postbackup commands - if [[ "${CONFIG_postbackup}" ]];then - echo "======================================================================" - echo "Postbackup command output." - echo - source ${CONFIG_postbackup} - echo - echo "======================================================================" - fi - # <- postbackup commands - - if [[ -s "$log_errfile" ]];then status=1; else status=0; fi - - exit ${status} +method_backup() { + manifest_entries_corrupted=() + manifest_entries_user_action_required=() + + # END __FUNCTIONS + ############################################################################################################## + # BEGIN __STARTUP + + load_default_config + + trap mail_cleanup EXIT SIGHUP SIGINT SIGQUIT SIGTERM + if [[ -r "${CONFIG_configfile}" ]]; then source "${CONFIG_configfile}"; else let "N |= $N_config_file_missing"; fi + if (($opt_flag_config_file)); then if [[ -r "${opt_config_file}" ]]; then + source "${opt_config_file}" + let "N |= $N_arg_conffile_parsed" + else let "N |= $N_arg_conffile_unreadable"; fi; else let "N |= $N_too_many_args"; fi + + # load mysql commands + mysql_commands + + (($CONFIG_dryrun)) && { + echo "NOTE: We are dry-running. That means, that the script just shows you what it would do, if it were operating normally." + echo "THE PRINTED COMMANDS CAN'T BE COPIED AND EXECUTED IF THERE ARE SPECIAL CHARACTERS, SPACES, ETC. IN THERE THAT WOULD NEED TO BE PROPERLY QUOTED IN ORDER TO WORK. THESE WERE CORRECTLY QUOTED FOR THE OUTPUT COMMAND, BUT CAN'T BE SEEN NOW." + echo + } + + export LC_ALL=C + PROGNAME=$(basename $0) + PATH=${PATH}:/usr/local/bin:/usr/bin:/bin:/usr/local/mysql/bin + version=3.0 + fields=5 # manifest fields + + directory_checks_enable_logging + cleanup_latest + set_datetime_vars + check_dependencies # check for required programs + parse_configuration # parse configuration and set variables appropriately + + # END __STARTUP + #-------------------------------------------------------------------------------------------------------------------------------------- + # BEGIN __PREPARE + + backupfiles=() + parse_databases + + # debug output of variables + (($CONFIG_debug)) && { + echo + echo "# DEBUG: printing all current variables" + declare -p | egrep -o '.* (CONFIG_[a-z_]*|opt|mysql_opt|opt_dbstatus|opt_fullschema)=.*' + echo + } + (($CONFIG_debug)) && { + echo "DEBUG: before pre-backup" + ( + IFS=, + echo "DEBUG: CONFIG_db_names '${CONFIG_db_names[*]}'" + ) + ( + IFS=, + echo "DEBUG: CONFIG_db_month_names '${CONFIG_db_month_names[*]}'" + ) + } + + # END __PREPARE + #-------------------------------------------------------------------------------------------------------------------------------------- + # BEGIN __MAIN + + ### filename formats + ## + ## example date values: + # 14'th of August (08) 2011 + # week number: 32 + # Sunday (date_dayno_of_week: 7) + ## + ## separate db's: + # monthly_DBNAME_2011-08-14_18h12m_August.sql(.enc).{gz,bzip2} + # weekly_DBNAME_2011-08-14_18h12m_32.sql(.enc).{gz,bzip2} + # daily_DBNAME_2011-08-14_18h12m_7.sql(.enc).{gz,bzip2} + ## all-databases: + # monthly_all-databases_DBNAME_2011-08-14_18h12m_August.sql(.enc).{gz,bzip2} + # weekly_all-databases_DBNAME_2011-08-14_18h12m_32.sql(.enc).{gz,bzip2} + # daily_all-databases_DBNAME_2011-08-14_18h12m_7.sql(.enc).{gz,bzip2} + + echo "======================================================================" + echo "AutoMySQLBackup version ${version}" + echo "http://sourceforge.net/projects/automysqlbackup/" + echo + echo "Backup of Database Server - ${CONFIG_mysql_dump_host_friendly:-$CONFIG_mysql_dump_host}" + ( + IFS=, + echo "Databases - ${CONFIG_db_names[*]}" + ) + ( + IFS=, + echo "Databases (monthly) - ${CONFIG_db_month_names[*]}" + ) + echo "======================================================================" + + # -> preback commands + if [[ "${CONFIG_prebackup}" ]]; then + echo "======================================================================" + echo "Prebackup command output." + echo + source ${CONFIG_prebackup} + echo + echo "======================================================================" + echo + fi + # <- preback commands + + # -> backup local files + if [[ "${CONFIG_backup_local_files[@]}" ]] && [[ ${CONFIG_do_weekly} != 0 && ${date_dayno_of_week} = ${CONFIG_do_weekly} ]] && (shopt -s nullglob dotglob; f=("${CONFIG_backup_dir}/backup_local_files/bcf_weekly_${date_stamp}_"[0-9][0-9]"h"[0-9][0-9]"m_${date_weekno}.tar${suffix}"); ((! ${#f[@]}))); then + echo "======================================================================" + echo "Backup local files. Doing this weekly on CONFIG_do_weekly." + echo + backup_local_files "${CONFIG_backup_dir}/backup_local_files/bcf_weekly_${datetimestamp}_${date_weekno}.tar" + tmp_flags=$? + var= + if (($? == 0)); then + echo "success!" + backupfiles=("${backupfiles[@]}" "${CONFIG_backup_dir}/backup_local_files/bcf_weekly_${datetimestamp}_${date_weekno}.tar") + else + let "E |= $E_backup_local_failed" + echo "failed!" + fi + echo + echo "======================================================================" + echo + fi + # <- backup local files + + # -> dump full schema + if [[ "${CONFIG_mysql_dump_full_schema}" = 'yes' ]]; then + echo "======================================================================" + echo "Dump full schema." + echo + + # monthly + if (( ${CONFIG_do_monthly} != 0 && (${date_day_of_month} == ${CONFIG_do_monthly} || $date_day_of_month == $date_lastday_of_this_month && $date_lastday_of_this_month < ${CONFIG_do_monthly}) )) && (shopt -s nullglob dotglob; f=("${CONFIG_backup_dir}/fullschema/fullschema_monthly_${date_stamp}_"[0-9][0-9]"h"[0-9][0-9]"m_${date_month}.sql${suffix}"); ((! ${#f[@]}))); then + fullschema "${CONFIG_backup_dir}/fullschema/fullschema_monthly_${datetimestamp}_${date_month}.sql" + if (($? == 0)); then + echo "Rotating $((${CONFIG_rotation_monthly} / 31)) month backups for ${mdb}" + if (($CONFIG_dryrun)); then + find "${CONFIG_backup_dir}/fullschema" -mtime +"${CONFIG_rotation_monthly}" -type f -name 'fullschema_monthly*' -exec echo "dry-running: rm" {} \; + else + find "${CONFIG_backup_dir}/fullschema" -mtime +"${CONFIG_rotation_monthly}" -type f -name 'fullschema_monthly*' -exec rm {} \; + fi + files_postprocessing "${CONFIG_backup_dir}/fullschema/fullschema_monthly_${datetimestamp}_${date_month}.sql${suffix}" + tmp_flags=$? + var= + (($tmp_flags & $flags_files_postprocessing_success_encrypt)) && var=.enc + backupfiles=("${backupfiles[@]}" "${CONFIG_backup_dir}/fullschema/fullschema_monthly_${datetimestamp}_${date_month}.sql${suffix}${var}") + else + let "E |= $E_dump_fullschema_failed" + fi + fi + + # weekly + if [[ ${CONFIG_do_weekly} != 0 && ${date_dayno_of_week} = ${CONFIG_do_weekly} ]] && (shopt -s nullglob dotglob; f=("${CONFIG_backup_dir}/fullschema/fullschema_weekly_${date_stamp}_"[0-9][0-9]"h"[0-9][0-9]"m_${date_weekno}.sql${suffix}"); ((! ${#f[@]}))); then + fullschema "${CONFIG_backup_dir}/fullschema/fullschema_weekly_${datetimestamp}_${date_weekno}.sql" + if (($? == 0)); then + echo "Rotating $((${CONFIG_rotation_monthly} / 31)) month backups for ${mdb}" + if (($CONFIG_dryrun)); then + find "${CONFIG_backup_dir}/fullschema" -mtime +"${CONFIG_rotation_weekly}" -type f -name 'fullschema_weekly*' -exec echo "dry-running: rm" {} \; + else + find "${CONFIG_backup_dir}/fullschema" -mtime +"${CONFIG_rotation_weekly}" -type f -name 'fullschema_weekly*' -exec rm {} \; + fi + files_postprocessing "${CONFIG_backup_dir}/fullschema/fullschema_weekly_${datetimestamp}_${date_weekno}.sql${suffix}" + tmp_flags=$? + var= + (($tmp_flags & $flags_files_postprocessing_success_encrypt)) && var=.enc + backupfiles=("${backupfiles[@]}" "${CONFIG_backup_dir}/fullschema/fullschema_weekly_${datetimestamp}_${date_weekno}.sql${suffix}${var}") + else + let "E |= $E_dump_fullschema_failed" + fi + fi + + # daily + fullschema "${CONFIG_backup_dir}/fullschema/fullschema_daily_${datetimestamp}_${date_day_of_week}.sql" + if (($? == 0)); then + echo "Rotating $((${CONFIG_rotation_monthly} / 31)) month backups for ${mdb}" + if (($CONFIG_dryrun)); then + find "${CONFIG_backup_dir}/fullschema" -mtime +"${CONFIG_rotation_daily}" -type f -name 'fullschema_daily*' -exec echo "dry-running: rm" {} \; + else + find "${CONFIG_backup_dir}/fullschema" -mtime +"${CONFIG_rotation_daily}" -type f -name 'fullschema_daily*' -exec rm {} \; + fi + files_postprocessing "${CONFIG_backup_dir}/fullschema/fullschema_daily_${datetimestamp}_${date_day_of_week}.sql${suffix}" + tmp_flags=$? + var= + (($tmp_flags & $flags_files_postprocessing_success_encrypt)) && var=.enc + backupfiles=("${backupfiles[@]}" "${CONFIG_backup_dir}/fullschema/fullschema_daily_${datetimestamp}_${date_day_of_week}.sql${suffix}${var}") + else + let "E |= $E_dump_fullschema_failed" + fi + echo + echo "======================================================================" + echo + + fi + # <- dump full schema + + # -> dump status + if [[ "${CONFIG_mysql_dump_dbstatus}" = 'yes' ]]; then + echo "======================================================================" + echo "Dump status." + echo + + # monthly + if (( ${CONFIG_do_monthly} != 0 && (${date_day_of_month} == ${CONFIG_do_monthly} || $date_day_of_month == $date_lastday_of_this_month && $date_lastday_of_this_month < ${CONFIG_do_monthly}) )) && (shopt -s nullglob dotglob; f=("${CONFIG_backup_dir}/status/status_monthly_${date_stamp}_"[0-9][0-9]"h"[0-9][0-9]"m_${date_month}.txt${suffix}"); ((! ${#f[@]}))); then + dbstatus "${CONFIG_backup_dir}/status/status_monthly_${datetimestamp}_${date_month}.txt" + if (($? == 0)); then + echo "Rotating $((${CONFIG_rotation_monthly} / 31)) month backups for ${mdb}" + if (($CONFIG_dryrun)); then + find "${CONFIG_backup_dir}/status" -mtime +"${CONFIG_rotation_monthly}" -type f -name 'status_monthly*' -exec echo "dry-running: rm" {} \; + else + find "${CONFIG_backup_dir}/status" -mtime +"${CONFIG_rotation_monthly}" -type f -name 'status_monthly*' -exec rm {} \; + fi + files_postprocessing "${CONFIG_backup_dir}/status/status_monthly_${datetimestamp}_${date_month}.txt${suffix}" + tmp_flags=$? + var= + (($tmp_flags & $flags_files_postprocessing_success_encrypt)) && var=.enc + backupfiles=("${backupfiles[@]}" "${CONFIG_backup_dir}/status/status_monthly_${datetimestamp}_${date_month}.txt${suffix}${var}") + else + let "E |= $E_dump_status_failed" + fi + fi + + # weekly + if [[ ${CONFIG_do_weekly} != 0 && ${date_dayno_of_week} = ${CONFIG_do_weekly} ]] && (shopt -s nullglob dotglob; f=("${CONFIG_backup_dir}/status/status_weekly_${date_stamp}_"[0-9][0-9]"h"[0-9][0-9]"m_${date_weekno}.txt${suffix}"); ((! ${#f[@]}))); then + dbstatus "${CONFIG_backup_dir}/status/status_weekly_${datetimestamp}_${date_weekno}.txt" + if (($? == 0)); then + echo "Rotating $((${CONFIG_rotation_monthly} / 31)) month backups for ${mdb}" + if (($CONFIG_dryrun)); then + find "${CONFIG_backup_dir}/status" -mtime +"${CONFIG_rotation_weekly}" -type f -name 'status_weekly*' -exec echo "dry-running: rm" {} \; + else + find "${CONFIG_backup_dir}/status" -mtime +"${CONFIG_rotation_weekly}" -type f -name 'status_weekly*' -exec rm {} \; + fi + files_postprocessing "${CONFIG_backup_dir}/status/status_weekly_${datetimestamp}_${date_weekno}.txt${suffix}" + tmp_flags=$? + var= + (($tmp_flags & $flags_files_postprocessing_success_encrypt)) && var=.enc + backupfiles=("${backupfiles[@]}" "${CONFIG_backup_dir}/status/status_weekly_${datetimestamp}_${date_weekno}.txt${suffix}${var}") + else + let "E |= $E_dump_status_failed" + fi + fi + + # daily + dbstatus "${CONFIG_backup_dir}/status/status_daily_${datetimestamp}_${date_day_of_week}.txt" + if (($? == 0)); then + echo "Rotating $((${CONFIG_rotation_monthly} / 31)) month backups for ${mdb}" + if (($CONFIG_dryrun)); then + find "${CONFIG_backup_dir}/status" -mtime +"${CONFIG_rotation_daily}" -type f -name 'status_daily*' -exec echo "dry-running: rm" {} \; + else + find "${CONFIG_backup_dir}/status" -mtime +"${CONFIG_rotation_daily}" -type f -name 'status_daily*' -exec rm {} \; + fi + files_postprocessing "${CONFIG_backup_dir}/status/status_daily_${datetimestamp}_${date_day_of_week}.txt${suffix}" + tmp_flags=$? + var= + (($tmp_flags & $flags_files_postprocessing_success_encrypt)) && var=.enc + backupfiles=("${backupfiles[@]}" "${CONFIG_backup_dir}/status/status_daily_${datetimestamp}_${date_day_of_week}.txt${suffix}${var}") + else + let "E |= $E_dump_status_failed" + fi + echo + echo "======================================================================" + echo + + fi + # <- dump status + + # -> BACKUP DATABASES + echo "Backup Start Time $(date)" + echo "======================================================================" + + ## <- monthly backup, unique per month + if ((${CONFIG_do_monthly} != 0 && (${date_day_of_month} == ${CONFIG_do_monthly} || $date_day_of_month == $date_lastday_of_this_month && $date_lastday_of_this_month < ${CONFIG_do_monthly}))); then + echo "Monthly Backup ..." + echo + + subfolder="monthly" + prefix="monthly_" + midfix="_${date_month}" + extension=".sql" + rotation="${CONFIG_rotation_monthly}" + rotation_divisor="31" + rotation_string="month" + + if [[ "${CONFIG_mysql_dump_use_separate_dirs}" = "yes" ]]; then + for db in "${CONFIG_db_month_names[@]}"; do + echo "Monthly Backup of Database ( ${db} )" + (shopt -s nullglob dotglob; f=("${CONFIG_backup_dir}/${subfolder}/${db}/${prefix}${db}_${date_stamp}_"[0-9][0-9]"h"[0-9][0-9]"m${midfix}${extension}${suffix}"); ((${#f[@]}))) && continue + process_dbs "$subfolder" "$prefix" "$midfix" "$extension" "$rotation" "$rotation_divisor" "$rotation_string" 0 "$db" + echo ---------------------------------------------------------------------- + done + else + echo "Monthly backup of databases ( ${CONFIG_db_month_names[@]} )." + (shopt -s nullglob dotglob; f=("${CONFIG_backup_dir}/${subfolder}/${prefix}all-databases_${date_stamp}_"[0-9][0-9]"h"[0-9][0-9]"m${midfix}${extension}${suffix}"); ((${#f[@]}))) && + process_dbs "$subfolder" "$prefix" "$midfix" "$extension" "$rotation" "$rotation_divisor" "$rotation_string" 1 "${CONFIG_db_month_names[@]}" + echo "----------------------------------------------------------------------" + fi + fi + ## <- monthly backup + + ## <- weekly backup, unique per week + if ((${CONFIG_do_weekly} != 0 && ${date_dayno_of_week} == ${CONFIG_do_weekly})); then + echo "Weekly Backup ..." + echo + + subfolder="weekly" + prefix="weekly_" + midfix="_${date_weekno}" + extension=".sql" + rotation="${CONFIG_rotation_weekly}" + rotation_divisor="7" + rotation_string="week" + if [[ "${CONFIG_mysql_dump_use_separate_dirs}" = "yes" ]]; then + for db in "${CONFIG_db_names[@]}"; do + echo "Weekly Backup of Database ( ${db} )" + (shopt -s nullglob dotglob; f=("${CONFIG_backup_dir}/${subfolder}/${db}/${prefix}${db}_${date_stamp}_"[0-9][0-9]"h"[0-9][0-9]"m${midfix}${extension}${suffix}"); ((${#f[@]}))) && continue + process_dbs "$subfolder" "$prefix" "$midfix" "$extension" "$rotation" "$rotation_divisor" "$rotation_string" 0 "$db" + echo "----------------------------------------------------------------------" + done + else + echo "Weekly backup of databases ( ${CONFIG_db_names[@]} )." + (shopt -s nullglob dotglob; f=("${CONFIG_backup_dir}/${subfolder}/${prefix}all-databases_${date_stamp}_"[0-9][0-9]"h"[0-9][0-9]"m${midfix}${extension}${suffix}"); ((${#f[@]}))) && + process_dbs "$subfolder" "$prefix" "$midfix" "$extension" "$rotation" "$rotation_divisor" "$rotation_string" 1 "${CONFIG_db_names[@]}" + echo "----------------------------------------------------------------------" + fi + fi + ## <- weekly backup + + ## -> daily backup, test (( 1 )) is always true, just creates a grouping for Kate, which can be closed ^^ + if ((1)); then + echo "Daily Backup ..." + echo + + subfolder="daily" + prefix="daily_" + midfix="_${date_day_of_week}" + extension=".sql" + rotation="${CONFIG_rotation_daily}" + rotation_divisor="1" + rotation_string="day" + + if [[ "${CONFIG_mysql_dump_use_separate_dirs}" = "yes" ]]; then + for db in "${CONFIG_db_names[@]}"; do + echo "Daily Backup of Database ( ${db} )" + process_dbs "$subfolder" "$prefix" "$midfix" "$extension" "$rotation" "$rotation_divisor" "$rotation_string" 0 "$db" + echo "----------------------------------------------------------------------" + done + else + echo "Daily backup of databases ( ${CONFIG_db_names[@]} )." + process_dbs "$subfolder" "$prefix" "$midfix" "$extension" "$rotation" "$rotation_divisor" "$rotation_string" 1 "${CONFIG_db_names[@]}" + echo "----------------------------------------------------------------------" + fi + fi + ## <- daily backup + + echo + echo "Backup End Time $(date)" + echo "======================================================================" + # <- BACKUP DATABASES + + # -> clean latest filenames + [[ "${CONFIG_mysql_dump_latest_clean_filenames}" = 'yes' ]] && find "${CONFIG_backup_dir}"/latest/ -type f -exec bash -c 'remove_datetimeinfo "$@"' -- {} \; + # <- clean latest filenames + + # -> finished information + echo "Total disk space used for backup storage..." + echo "Size - Location" + echo $(du -hs $OS_du_extra_option "${CONFIG_backup_dir}") + echo + echo "======================================================================" + # <- finished information + + # -> postbackup commands + if [[ "${CONFIG_postbackup}" ]]; then + echo "======================================================================" + echo "Postbackup command output." + echo + source ${CONFIG_postbackup} + echo + echo "======================================================================" + fi + # <- postbackup commands + + if [[ -s "$log_errfile" ]]; then status=1; else status=0; fi + + exit ${status} } # @return variable method_list_manifest_entries_array -method_list_manifest_entries () { - local files files_master files_manifest file db manifest_files manifest_files_db selected_dbs i z l master_flags master to_rm actions - manifest_entries_corrupted=() - manifest_entries_user_action_required=() - files=() - files_master=() - files_manifest=() - master_flags=0 - let "filename_flag_encrypted=0x01" - let "filename_flag_gz=0x02" - let "filename_flag_bz2=0x04" - let "filename_flag_diff=0x08" - - ############################################################################################################## - # BEGIN __STARTUP - - load_default_config - - if [[ -r "${CONFIG_configfile}" ]]; then source "${CONFIG_configfile}"; echo "Parsed config file \"${CONFIG_configfile}\""; else let "N |= $N_config_file_missing"; fi; echo - if (( $opt_flag_config_file )); then if [[ -r "${opt_config_file}" ]]; then source "${opt_config_file}"; let "N |= $N_arg_conffile_parsed"; else let "N |= $N_arg_conffile_unreadable"; fi; else let "N |= $N_too_many_args"; fi - - # load mysql commands - mysql_commands - - export LC_ALL=C - PROGNAME=`basename $0` - PATH=${PATH}:/usr/local/bin:/usr/bin:/bin:/usr/local/mysql/bin - version=3.0 - fields=5 # manifest fields - - set_datetime_vars - check_dependencies # check for required programs - parse_configuration # parse configuration and set variables appropriately - - # BEGIN __MAIN - unset manifest_files manifest_files_db i db - while IFS= read -r -d '' file; do - db="${file#/var/backup/db/@(daily|monthly|weekly|latest)/}"; - db="${db%/Manifest}"; - manifest_files_db[i]="$db" - manifest_files[i++]="$file" - done < <(find "${CONFIG_backup_dir}"/ -type f -name 'Manifest' -print0) - - extended_select 0 "Databases" "${manifest_files_db[@]}" - declare -a selected_dbs=("${extended_select_return[@]}") - - for db in "${selected_dbs[@]}"; do - selected_available_files=() - for ((i=0;i<"${#manifest_files_db[@]}";i++)); do - if [[ "x${manifest_files_db[i]}" = "x$db" ]]; then - selected_available_files[j++]="${manifest_files[i]}" - fi - done - if (( "${#selected_available_files[@]}" > 0 )); then - extended_select 1 "$db" "${selected_available_files[@]}" - declare -a selected_entries=("${extended_select_return[@]}") - if (( "${#selected_entries[@]}" > 0 )); then - for z in "${selected_entries[@]}"; do - parse_manifest "$z" - list=() - list_id=() - for ((i=0;$i<"${#manifest_array[@]}";i=$i+$fields)); do - if [[ "${manifest_array[i+3]}" != 0 ]]; then # only add differential backups - list=("${list[@]}" "${manifest_array[i]}") - list_id=("${list_id[@]}" "${manifest_array[i+3]}") # save rel_id, so we can retrieve the master backup file - fi - done - if (( "${#list[@]}" > 0 )); then - extended_select 1 "$z" "${list[@]}" - if (( "${#extended_select_return[@]}" > 0 )); then - for ((i=0;$i<"${#extended_select_return[@]}";i++)); do - if get_manifest_entry_by_id "${list_id[${extended_select_return_id[i]}]}"; then - files=("${files[@]}" "${extended_select_return[i]}") - files_master=("${files_master[@]}" "${manifest_entry[0]}") - files_manifest=("${files_manifest[@]}" "$z") - else - echo "no found master for id ${list_id[${extended_select_return_id[i]}]}" - fi - done - fi - fi - done - fi - fi - done - # END _select_filenames - declare -a method_list_manifest_entries_array=("${files[@]}") - declare -a method_list_manifest_entries_array_master=("${files_master[@]}") - declare -a method_list_manifest_entries_array_manifest=("${files_manifest[@]}") - - clear - echo "You have selected the following files:" - for i in "${files[@]}"; do printf '>>> %s\n' "$i"; done - echo - actions=('diff to full' 'remove files (also from Manifest)') - extended_select 0 "Actions" "${actions[@]}" - for action in "${extended_select_return[@]}"; do - case "$action" in - 'diff to full') - for ((l=0;$l<"${#files[@]}";l++)); do - # put the unpacking of the master file in here, so that in the case of multiple diffs with the same - # master file don't cause the script to unpack the same master file multiple times - master="${files_master[l]}" - diff="${files[l]}" - FileStub="${master%.@(sql|master)*}" - FileExt="${master#"$FileStub"}" - re=".*\.enc.*"; [[ "$FileExt" =~ $re ]] && let "master_flags|=$filename_flag_encrypted" - re=".*\.gz.*"; [[ "$FileExt" =~ $re ]] && let "master_flags|=$filename_flag_gz" - re=".*\.bz2.*"; [[ "$FileExt" =~ $re ]] && let "master_flags|=$filename_flag_bz2" - re=".*\.diff.*"; [[ "$FileExt" =~ $re ]] && let "master_flags|=$filename_flag_diff" - if (( $master_flags & $filename_flag_gz )); then - declare -a testarray=("${to_rm[@]}") - if ! in_array "${master%.gz}"; then - gzip_compression -dc "$master" > "${master%.gz}" - to_rm=("${to_rm[@]}" "${master%.gz}") - fi - master="${master%.gz}" - elif (( $master_flags & $filename_flag_bz2 )); then - declare -a testarray=("${to_rm[@]}") - if ! in_array "${master%.bz2}"; then - bzip2_compression -dc "$master" > "${master%.bz2}" - to_rm=("${to_rm[@]}" "${master%.bz2}") - fi - master="${master%.bz2}" - else - : - fi - method_diff_to_full "$master" "$diff" - #printf '%s\n>>> master: %s\n>>> manifest: %s\n' "${files[l]}" "${files_master[l]}" "${files_manifest[l]}" - done - # cleanup all unpacked master files ... the unpacked diff files are cleaned up by method_diff_to_full - for i in "${to_rm[@]}"; do rm "$i"; done - ;; - 'remove files (also from Manifest)') - for ((l=0;$l<"${#files[@]}";l++)); do - if rm_manifest_entry_by_filename "${files_manifest[l]}" "${files[l]}" 1; then - rm "${files[l]}" - fi - done - ;; - *) - echo "Unrecognized option. This Should not happen! Error!" - ;; - esac - done - - # END __MAIN +method_list_manifest_entries() { + local files files_master files_manifest file db manifest_files manifest_files_db selected_dbs i z l master_flags master to_rm actions + manifest_entries_corrupted=() + manifest_entries_user_action_required=() + files=() + files_master=() + files_manifest=() + master_flags=0 + let "filename_flag_encrypted=0x01" + let "filename_flag_gz=0x02" + let "filename_flag_bz2=0x04" + let "filename_flag_diff=0x08" + + ############################################################################################################## + # BEGIN __STARTUP + + load_default_config + + if [[ -r "${CONFIG_configfile}" ]]; then + source "${CONFIG_configfile}" + echo "Parsed config file \"${CONFIG_configfile}\"" + else let "N |= $N_config_file_missing"; fi + echo + if (($opt_flag_config_file)); then if [[ -r "${opt_config_file}" ]]; then + source "${opt_config_file}" + let "N |= $N_arg_conffile_parsed" + else let "N |= $N_arg_conffile_unreadable"; fi; else let "N |= $N_too_many_args"; fi + + # load mysql commands + mysql_commands + + export LC_ALL=C + PROGNAME=$(basename $0) + PATH=${PATH}:/usr/local/bin:/usr/bin:/bin:/usr/local/mysql/bin + version=3.0 + fields=5 # manifest fields + + set_datetime_vars + check_dependencies # check for required programs + parse_configuration # parse configuration and set variables appropriately + + # BEGIN __MAIN + unset manifest_files manifest_files_db i db + while IFS= read -r -d '' file; do + db="${file#/var/backup/db/@(daily|monthly|weekly|latest)/}" + db="${db%/Manifest}" + manifest_files_db[i]="$db" + manifest_files[i++]="$file" + done < <(find "${CONFIG_backup_dir}"/ -type f -name 'Manifest' -print0) + + extended_select 0 "Databases" "${manifest_files_db[@]}" + declare -a selected_dbs=("${extended_select_return[@]}") + + for db in "${selected_dbs[@]}"; do + selected_available_files=() + for ((i = 0; i < "${#manifest_files_db[@]}"; i++)); do + if [[ "x${manifest_files_db[i]}" = "x$db" ]]; then + selected_available_files[j++]="${manifest_files[i]}" + fi + done + if (("${#selected_available_files[@]}" > 0)); then + extended_select 1 "$db" "${selected_available_files[@]}" + declare -a selected_entries=("${extended_select_return[@]}") + if (("${#selected_entries[@]}" > 0)); then + for z in "${selected_entries[@]}"; do + parse_manifest "$z" + list=() + list_id=() + for ((i = 0; $i < "${#manifest_array[@]}"; i = $i + $fields)); do + if [[ "${manifest_array[i + 3]}" != 0 ]]; then # only add differential backups + list=("${list[@]}" "${manifest_array[i]}") + list_id=("${list_id[@]}" "${manifest_array[i + 3]}") # save rel_id, so we can retrieve the master backup file + fi + done + if (("${#list[@]}" > 0)); then + extended_select 1 "$z" "${list[@]}" + if (("${#extended_select_return[@]}" > 0)); then + for ((i = 0; $i < "${#extended_select_return[@]}"; i++)); do + if get_manifest_entry_by_id "${list_id[${extended_select_return_id[i]}]}"; then + files=("${files[@]}" "${extended_select_return[i]}") + files_master=("${files_master[@]}" "${manifest_entry[0]}") + files_manifest=("${files_manifest[@]}" "$z") + else + echo "no found master for id ${list_id[${extended_select_return_id[i]}]}" + fi + done + fi + fi + done + fi + fi + done + # END _select_filenames + declare -a method_list_manifest_entries_array=("${files[@]}") + declare -a method_list_manifest_entries_array_master=("${files_master[@]}") + declare -a method_list_manifest_entries_array_manifest=("${files_manifest[@]}") + + clear + echo "You have selected the following files:" + for i in "${files[@]}"; do printf '>>> %s\n' "$i"; done + echo + actions=('diff to full' 'remove files (also from Manifest)') + extended_select 0 "Actions" "${actions[@]}" + for action in "${extended_select_return[@]}"; do + case "$action" in + 'diff to full') + for ((l = 0; $l < "${#files[@]}"; l++)); do + # put the unpacking of the master file in here, so that in the case of multiple diffs with the same + # master file don't cause the script to unpack the same master file multiple times + master="${files_master[l]}" + diff="${files[l]}" + FileStub="${master%.@(sql|master)*}" + FileExt="${master#"$FileStub"}" + re=".*\.enc.*" + [[ "$FileExt" =~ $re ]] && let "master_flags|=$filename_flag_encrypted" + re=".*\.gz.*" + [[ "$FileExt" =~ $re ]] && let "master_flags|=$filename_flag_gz" + re=".*\.bz2.*" + [[ "$FileExt" =~ $re ]] && let "master_flags|=$filename_flag_bz2" + re=".*\.diff.*" + [[ "$FileExt" =~ $re ]] && let "master_flags|=$filename_flag_diff" + if (($master_flags & $filename_flag_gz)); then + declare -a testarray=("${to_rm[@]}") + if ! in_array "${master%.gz}"; then + gzip_compression -dc "$master" >"${master%.gz}" + to_rm=("${to_rm[@]}" "${master%.gz}") + fi + master="${master%.gz}" + elif (($master_flags & $filename_flag_bz2)); then + declare -a testarray=("${to_rm[@]}") + if ! in_array "${master%.bz2}"; then + bzip2_compression -dc "$master" >"${master%.bz2}" + to_rm=("${to_rm[@]}" "${master%.bz2}") + fi + master="${master%.bz2}" + else + : + fi + method_diff_to_full "$master" "$diff" + #printf '%s\n>>> master: %s\n>>> manifest: %s\n' "${files[l]}" "${files_master[l]}" "${files_manifest[l]}" + done + # cleanup all unpacked master files ... the unpacked diff files are cleaned up by method_diff_to_full + for i in "${to_rm[@]}"; do rm "$i"; done + ;; + 'remove files (also from Manifest)') + for ((l = 0; $l < "${#files[@]}"; l++)); do + if rm_manifest_entry_by_filename "${files_manifest[l]}" "${files[l]}" 1; then + rm "${files[l]}" + fi + done + ;; + *) + echo "Unrecognized option. This Should not happen! Error!" + ;; + esac + done + + # END __MAIN } # @info: Convert a differential backup file to a full one. # @param: master_backup_file diff_backup_file method_diff_to_full() { - local diff full diff_flags master_flags to_rm - master="$1" - diff="$2" - diff_flags=0 - master_flags=0 - to_rm=() - - FileStub="${diff%.@(sql|diff)*}" - FileExt="${diff#"$FileStub"}" - re=".*\.enc.*"; [[ "$FileExt" =~ $re ]] && let "diff_flags|=$filename_flag_encrypted" - re=".*\.gz.*"; [[ "$FileExt" =~ $re ]] && let "diff_flags|=$filename_flag_gz" - re=".*\.bz2.*"; [[ "$FileExt" =~ $re ]] && let "diff_flags|=$filename_flag_bz2" - re=".*\.diff.*"; [[ "$FileExt" =~ $re ]] && let "diff_flags|=$filename_flag_diff" - FileStub="${master%.@(sql|master)*}" - FileExt="${master#"$FileStub"}" - re=".*\.enc.*"; [[ "$FileExt" =~ $re ]] && let "master_flags|=$filename_flag_encrypted" - re=".*\.gz.*"; [[ "$FileExt" =~ $re ]] && let "master_flags|=$filename_flag_gz" - re=".*\.bz2.*"; [[ "$FileExt" =~ $re ]] && let "master_flags|=$filename_flag_bz2" - re=".*\.diff.*"; [[ "$FileExt" =~ $re ]] && let "master_flags|=$filename_flag_diff" - - # TODO: Differential backup with encryption is not yet implemented! - if (( $diff_flags & $filename_flag_encrypted )); then - : #decrypt it - fi - - if (( $master_flags & $filename_flag_encrypted )); then - : #decrypt it - fi - - if (( $diff_flags & $filename_flag_gz )); then - gzip_compression -dc "$diff" > "${diff%.gz}" - to_rm=("${to_rm[@]}" "${diff%.gz}") - diff="${diff%.gz}" - elif (( $diff_flags & $filename_flag_bz2 )); then - bzip2_compression -dc "$diff" > "${diff%.bz2}" - to_rm=("${to_rm[@]}" "${diff%.bz2}") - diff="${diff%.bz2}" - else - : - fi - - if (( $master_flags & $filename_flag_gz )); then - gzip_compression -dc "$master" > "${master%.gz}" - to_rm=("${to_rm[@]}" "${master%.gz}") - master="${master%.gz}" - elif (( $master_flags & $filename_flag_bz2 )); then - bzip2_compression -dc "$master" > "${master%.bz2}" - to_rm=("${to_rm[@]}" "${master%.bz2}") - master="${master%.bz2}" - else - : - fi - - patch "$master" "$diff" -o "${diff/diff/sql}" - - # cleanup - for i in "${to_rm[@]}"; do rm "$i"; done + local diff full diff_flags master_flags to_rm + master="$1" + diff="$2" + diff_flags=0 + master_flags=0 + to_rm=() + + FileStub="${diff%.@(sql|diff)*}" + FileExt="${diff#"$FileStub"}" + re=".*\.enc.*" + [[ "$FileExt" =~ $re ]] && let "diff_flags|=$filename_flag_encrypted" + re=".*\.gz.*" + [[ "$FileExt" =~ $re ]] && let "diff_flags|=$filename_flag_gz" + re=".*\.bz2.*" + [[ "$FileExt" =~ $re ]] && let "diff_flags|=$filename_flag_bz2" + re=".*\.diff.*" + [[ "$FileExt" =~ $re ]] && let "diff_flags|=$filename_flag_diff" + FileStub="${master%.@(sql|master)*}" + FileExt="${master#"$FileStub"}" + re=".*\.enc.*" + [[ "$FileExt" =~ $re ]] && let "master_flags|=$filename_flag_encrypted" + re=".*\.gz.*" + [[ "$FileExt" =~ $re ]] && let "master_flags|=$filename_flag_gz" + re=".*\.bz2.*" + [[ "$FileExt" =~ $re ]] && let "master_flags|=$filename_flag_bz2" + re=".*\.diff.*" + [[ "$FileExt" =~ $re ]] && let "master_flags|=$filename_flag_diff" + + # TODO: Differential backup with encryption is not yet implemented! + if (($diff_flags & $filename_flag_encrypted)); then + : #decrypt it + fi + + if (($master_flags & $filename_flag_encrypted)); then + : #decrypt it + fi + + if (($diff_flags & $filename_flag_gz)); then + gzip_compression -dc "$diff" >"${diff%.gz}" + to_rm=("${to_rm[@]}" "${diff%.gz}") + diff="${diff%.gz}" + elif (($diff_flags & $filename_flag_bz2)); then + bzip2_compression -dc "$diff" >"${diff%.bz2}" + to_rm=("${to_rm[@]}" "${diff%.bz2}") + diff="${diff%.bz2}" + else + : + fi + + if (($master_flags & $filename_flag_gz)); then + gzip_compression -dc "$master" >"${master%.gz}" + to_rm=("${to_rm[@]}" "${master%.gz}") + master="${master%.gz}" + elif (($master_flags & $filename_flag_bz2)); then + bzip2_compression -dc "$master" >"${master%.bz2}" + to_rm=("${to_rm[@]}" "${master%.bz2}") + master="${master%.bz2}" + else + : + fi + + patch "$master" "$diff" -o "${diff/diff/sql}" + + # cleanup + for i in "${to_rm[@]}"; do rm "$i"; done } # END _methods - # BEGIN __main NO_ARGS=0 E_OPTERROR=85 -if (( $# == $NO_ARGS )); then # Script invoked with no command-line args? +if (($# == $NO_ARGS)); then # Script invoked with no command-line args? method_backup fi -while getopts ":c:blh" Option -do +while getopts ":c:blh" Option; do case $Option in - c ) echo "Using \"$OPTARG\" as optional config file."; echo; opt_config_file="$OPTARG"; opt_flag_config_file=1;; - b ) echo "MySQL backup method invoked."; echo; opt_flag_method_backup=1;; - l ) echo "List manifest entries."; echo; opt_flag_list_manifest_entries=1;; - h ) echo "Usage `basename $0` options -cblh" - echo -e "-c CONFIG_FILE\tSpecify optional config file." - echo -e "-b\tUse backup method." - echo -e "-l\tList manifest entries." - echo -e "-h\tShow this help." - exit 0;; - #n | o ) echo "Scenario #2: option -$Option- [OPTIND=${OPTIND}]";; - - #q ) echo "Scenario #4: option -q-\ - # with argument \"$OPTARG\" [OPTIND=${OPTIND}]";; - # Note that option 'q' must have an associated argument, - #+ otherwise it falls through to the default. - #r | s ) echo "Scenario #5: option -$Option-";; - * ) echo "Unimplemented option chosen.";; # Default. + c) + echo "Using \"$OPTARG\" as optional config file." + echo + opt_config_file="$OPTARG" + opt_flag_config_file=1 + ;; + b) + echo "MySQL backup method invoked." + echo + opt_flag_method_backup=1 + ;; + l) + echo "List manifest entries." + echo + opt_flag_list_manifest_entries=1 + ;; + h) + echo "Usage $(basename $0) options -cblh" + echo -e "-c CONFIG_FILE\tSpecify optional config file." + echo -e "-b\tUse backup method." + echo -e "-l\tList manifest entries." + echo -e "-h\tShow this help." + exit 0 + ;; + #n | o ) echo "Scenario #2: option -$Option- [OPTIND=${OPTIND}]";; + + #q ) echo "Scenario #4: option -q-\ + # with argument \"$OPTARG\" [OPTIND=${OPTIND}]";; + # Note that option 'q' must have an associated argument, + #+ otherwise it falls through to the default. + #r | s ) echo "Scenario #5: option -$Option-";; + *) echo "Unimplemented option chosen." ;; # Default. esac done -(( $opt_flag_method_backup )) && method_backup -(( $opt_flag_list_manifest_entries )) && method_list_manifest_entries +(($opt_flag_method_backup)) && method_backup +(($opt_flag_list_manifest_entries)) && method_list_manifest_entries shift $(($OPTIND - 1)) # Decrements the argument pointer so it points to next argument. @@ -2289,10 +2388,12 @@ shift $(($OPTIND - 1)) # For backward compatibility. If no option items are present and only one non-option item is there, we expect it # to be the optional config file and invoke the backup method. -opt_flags=( "${!opt_flag_@}" ) # array of all set variables starting with opt_flag_ -if (( $# == 1 )) && (( ${#opt_flags[@]} == 0 )); then - opt_config_file="$1"; opt_flag_config_file=1; method_backup -elif (( $# == 0 )) && (( ${#opt_flags[@]} == 0 )); then +opt_flags=("${!opt_flag_@}") # array of all set variables starting with opt_flag_ +if (($# == 1)) && ((${#opt_flags[@]} == 0)); then + opt_config_file="$1" + opt_flag_config_file=1 + method_backup +elif (($# == 0)) && ((${#opt_flags[@]} == 0)); then method_backup fi From 28255b2b8f107b408deace049914b23adbeb64e8 Mon Sep 17 00:00:00 2001 From: Christoffer Lindahl Date: Sun, 11 Sep 2022 18:30:45 +0200 Subject: [PATCH 5/5] Option to dump users and grants. --- automysqlbackup | 144 +++++++++++++++++++++++++++++++++++++++---- automysqlbackup.conf | 3 + 2 files changed, 135 insertions(+), 12 deletions(-) diff --git a/automysqlbackup b/automysqlbackup index 39818c7..5e1e9d2 100755 --- a/automysqlbackup +++ b/automysqlbackup @@ -41,6 +41,7 @@ let "E_no_basedir=0x400" let "E_config_backupdir_not_writable=0x800" let "E_dump_status_failed=0x1000" let "E_dump_fullschema_failed=0x2000" +let "E_dump_users_failed=0x4000" let "N_config_file_missing=0x01" let "N_arg_conffile_parsed=0x02" @@ -87,6 +88,7 @@ load_default_config() { CONFIG_mysql_dump_login_path_file='automysqldump' CONFIG_mysql_dump_config_extra_file='' CONFIG_mysql_dump_encrypted_login='no' + CONFIG_mysql_dump_users='no' CONFIG_backup_local_files=() CONFIG_db_names=() CONFIG_db_month_names=() @@ -113,23 +115,27 @@ fi mysql_commands() { VERSION=$(mysql -V | grep -oE "[0-9]+\.[0-9]+\.[0-9]+") NODOT_VER=$(echo $VERSION | sed $OS_sed_option 's/\.//g') + LOGIN_ARGS="" if [ "${CONFIG_mysql_dump_encrypted_login}" = "yes" ]; then if [ "${CONFIG_mysql_dump_config_extra_file}" != "" ]; then - export MYSQLDUMP="mysqldump --defaults-extra-file=$CONFIG_mysql_dump_config_extra_file" - export MYSQLSHOW="mysqlshow --defaults-extra-file=$CONFIG_mysql_dump_config_extra_file" - export MYSQL="mysql --defaults-extra-file=$CONFIG_mysql_dump_config_extra_file" + LOGIN_ARGS="--defaults-extra-file=$CONFIG_mysql_dump_config_extra_file" else - export MYSQLDUMP="mysqldump --login-path=$CONFIG_mysql_dump_login_path" - export MYSQLSHOW="mysqlshow --login-path=$CONFIG_mysql_dump_login_path" - export MYSQL="mysql --login-path=$CONFIG_mysql_dump_login_path" - if [ -n "${CONFIG_mysql_dump_login_path_file}" ]; then - export MYSQL_TEST_LOGIN_FILE=$CONFIG_mysql_dump_login_path_file - fi + LOGIN_ARGS="--login-path=$CONFIG_mysql_dump_login_path" fi else - export MYSQLDUMP="mysqldump --user=${CONFIG_mysql_dump_username} --password=${CONFIG_mysql_dump_password} --host=${CONFIG_mysql_dump_host}" - export MYSQLSHOW="mysqlshow --user=${CONFIG_mysql_dump_username} --password=${CONFIG_mysql_dump_password} --host=${CONFIG_mysql_dump_host}" - export MYSQL="mysql --user=${CONFIG_mysql_dump_username} --password=${CONFIG_mysql_dump_password} --host=${CONFIG_mysql_dump_host}" + LOGIN_ARGS="--user=${CONFIG_mysql_dump_username} --password=${CONFIG_mysql_dump_password} --host=${CONFIG_mysql_dump_host}" + fi + + export MYSQLDUMP="mysqldump $LOGIN_ARGS" + export MYSQLSHOW="mysqlshow $LOGIN_ARGS" + export MYSQL="mysql $LOGIN_ARGS" + + if [ "${CONFIG_mysql_dump_users}" = "yes" ]; then + if command -v mysqlpump &> /dev/null; then + export MYSQLUSERS="mysqlpump $LOGIN_ARGS --exclude-databases=% --users" + elif command -v mariadb-dump &> /dev/null; then + export MYSQLUSERS="mariadb-dump $LOGIN_ARGS --system=users" + fi fi } @@ -660,6 +666,49 @@ fullschema() { fi } +# @info: Backup of the database users. +# @args: filename to save data to +# @return: true in case of dry-run, otherwise the return value of mysqlpump +# @deps: load_default_config, parse_configuration +users() { + if (($CONFIG_dryrun)); then + case "${CONFIG_mysql_dump_compression}" in + 'gzip') + echo "dry-running: $MYSQLUSERS | gzip_compression > ${1}${suffix}" + ;; + 'bzip2') + echo "dry-running: $MYSQLUSERS | bzip2_compression > ${1}${suffix}" + ;; + *) + echo "dry-running: $MYSQLUSERS > ${1}${suffix}" + ;; + esac + return 0 + else + log_usersfile="/tmp/${CONFIG_mysql_dump_host}-$(date +%N)-users.log" + case "${CONFIG_mysql_dump_compression}" in + 'gzip') + $MYSQLUSERS 2>"$log_usersfile" | gzip_compression >"${1}${suffix}" + ;; + 'bzip2') + $MYSQLUSERS 2>"$log_usersfile" | bzip2_compression >"${1}${suffix}" + ;; + *) + $MYSQLUSERS 2>"$log_usersfile" >"${1}${suffix}" + ;; + esac + + firstline=$(head -n 1 "$log_usersfile") + if ! [[ $firstline =~ ^"Dump completed"* ]]; then + cat "$log_usersfile" >> "$log_errfile" + fi + + rm -f "$log_usersfile" + + return $? + fi +} + # @info: Process a single db. # @args: subfolder, prefix, midfix, extension, rotation, rotation_divisor, rotation_string, 0/1 (db/dbs), db[, db ...] process_dbs() { @@ -911,6 +960,7 @@ directory_checks_enable_logging() { checkdirs=("${CONFIG_backup_dir}"/{daily,weekly,monthly,latest,tmp}) [[ "${CONFIG_backup_local_files[@]}" ]] && { checkdirs=("${checkdirs[@]}" "${CONFIG_backup_dir}/backup_local_files"); } [[ "${CONFIG_mysql_dump_full_schema}" = 'yes' ]] && { checkdirs=("${checkdirs[@]}" "${CONFIG_backup_dir}/fullschema"); } + [[ "${CONFIG_mysql_dump_users}" = 'yes' ]] && { checkdirs=("${checkdirs[@]}" "${CONFIG_backup_dir}/users"); } [[ "${CONFIG_mysql_dump_dbstatus}" = 'yes' ]] && { checkdirs=("${checkdirs[@]}" "${CONFIG_backup_dir}/status"); } tmp_permcheck=0 @@ -1907,6 +1957,76 @@ method_backup() { fi # <- dump full schema + # -> dump users + if [[ "${CONFIG_mysql_dump_users}" = 'yes' ]]; then + echo "======================================================================" + echo "Dump users." + echo + + # monthly + if (( ${CONFIG_do_monthly} != 0 && (${date_day_of_month} == ${CONFIG_do_monthly} || $date_day_of_month == $date_lastday_of_this_month && $date_lastday_of_this_month < ${CONFIG_do_monthly}) )) && (shopt -s nullglob dotglob; f=("${CONFIG_backup_dir}/users/users_monthly_${date_stamp}_"[0-9][0-9]"h"[0-9][0-9]"m_${date_month}.sql${suffix}"); ((! ${#f[@]}))); then + users "${CONFIG_backup_dir}/users/users_monthly_${datetimestamp}_${date_month}.sql" + if (($? == 0)); then + echo "Rotating $((${CONFIG_rotation_monthly} / 31)) month backups for ${mdb}" + if (($CONFIG_dryrun)); then + find "${CONFIG_backup_dir}/users" -mtime +"${CONFIG_rotation_monthly}" -type f -name 'users_monthly*' -exec echo "dry-running: rm" {} \; + else + find "${CONFIG_backup_dir}/users" -mtime +"${CONFIG_rotation_monthly}" -type f -name 'users_monthly*' -exec rm {} \; + fi + files_postprocessing "${CONFIG_backup_dir}/users/users_monthly_${datetimestamp}_${date_month}.sql${suffix}" + tmp_flags=$? + var= + (($tmp_flags & $flags_files_postprocessing_success_encrypt)) && var=.enc + backupfiles=("${backupfiles[@]}" "${CONFIG_backup_dir}/users/users_monthly_${datetimestamp}_${date_month}.sql${suffix}${var}") + else + let "E |= $E_dump_users_failed" + fi + fi + + # weekly + if [[ ${CONFIG_do_weekly} != 0 && ${date_dayno_of_week} = ${CONFIG_do_weekly} ]] && (shopt -s nullglob dotglob; f=("${CONFIG_backup_dir}/users/users_weekly_${date_stamp}_"[0-9][0-9]"h"[0-9][0-9]"m_${date_weekno}.sql${suffix}"); ((! ${#f[@]}))); then + users "${CONFIG_backup_dir}/users/users_weekly_${datetimestamp}_${date_weekno}.sql" + if (($? == 0)); then + echo "Rotating $((${CONFIG_rotation_monthly} / 31)) month backups for ${mdb}" + if (($CONFIG_dryrun)); then + find "${CONFIG_backup_dir}/users" -mtime +"${CONFIG_rotation_weekly}" -type f -name 'users_weekly*' -exec echo "dry-running: rm" {} \; + else + find "${CONFIG_backup_dir}/users" -mtime +"${CONFIG_rotation_weekly}" -type f -name 'users_weekly*' -exec rm {} \; + fi + files_postprocessing "${CONFIG_backup_dir}/users/users_weekly_${datetimestamp}_${date_weekno}.sql${suffix}" + tmp_flags=$? + var= + (($tmp_flags & $flags_files_postprocessing_success_encrypt)) && var=.enc + backupfiles=("${backupfiles[@]}" "${CONFIG_backup_dir}/users/users_weekly_${datetimestamp}_${date_weekno}.sql${suffix}${var}") + else + let "E |= $E_dump_users_failed" + fi + fi + + # daily + users "${CONFIG_backup_dir}/users/users_daily_${datetimestamp}_${date_day_of_week}.sql" + if (($? == 0)); then + echo "Rotating $((${CONFIG_rotation_monthly} / 31)) month backups for ${mdb}" + if (($CONFIG_dryrun)); then + find "${CONFIG_backup_dir}/users" -mtime +"${CONFIG_rotation_daily}" -type f -name 'users_daily*' -exec echo "dry-running: rm" {} \; + else + find "${CONFIG_backup_dir}/users" -mtime +"${CONFIG_rotation_daily}" -type f -name 'users_daily*' -exec rm {} \; + fi + files_postprocessing "${CONFIG_backup_dir}/users/users_daily_${datetimestamp}_${date_day_of_week}.sql${suffix}" + tmp_flags=$? + var= + (($tmp_flags & $flags_files_postprocessing_success_encrypt)) && var=.enc + backupfiles=("${backupfiles[@]}" "${CONFIG_backup_dir}/users/users_daily_${datetimestamp}_${date_day_of_week}.sql${suffix}${var}") + else + let "E |= $E_dump_users_failed" + fi + echo + echo "======================================================================" + echo + + fi + # <- dump users + # -> dump status if [[ "${CONFIG_mysql_dump_dbstatus}" = 'yes' ]]; then echo "======================================================================" diff --git a/automysqlbackup.conf b/automysqlbackup.conf index d4fb50e..2d6c7a5 100644 --- a/automysqlbackup.conf +++ b/automysqlbackup.conf @@ -41,6 +41,9 @@ # if unset or empty (default) will use CONFIG_mysql_dump_host instead #CONFIG_mysql_dump_host_friendly='' +# Should users and their grants be exported? +#CONFIG_mysql_dump_users='no' + # Backup directory location e.g /backups #CONFIG_backup_dir='/var/backup/db'