From a0dda4efb05f816c9fc42eaefbb5b89fc2ae9606 Mon Sep 17 00:00:00 2001 From: "jerome.gagnon" Date: Thu, 28 Sep 2023 19:22:22 +0000 Subject: [PATCH 1/9] fixes tests --- molecule/default/converge.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/molecule/default/converge.yml b/molecule/default/converge.yml index 394383f..b34299f 100644 --- a/molecule/default/converge.yml +++ b/molecule/default/converge.yml @@ -5,11 +5,13 @@ - name: Set ssh server package name for non-Archlinux ansible_os_family set_fact: openssh_package: "openssh-server" + pip3_extra_args: "" when: ansible_os_family != "Archlinux" - - name: Set ssh server package name for Archlinux ansible_os_family + - name: Set ssh server package name and pip3 argument for Archlinux ansible_os_family set_fact: openssh_package: "openssh" + pip3_extra_args: "--break-system-packages" when: ansible_os_family == "Archlinux" - name: Install openssh @@ -46,3 +48,5 @@ pip: name: yamllint executable: pip3 + extra_args: "{{ pip3_extra_args }}" + From 54f42b1515be0b5bdcdac627404dfa02f3d60c66 Mon Sep 17 00:00:00 2001 From: "jerome.gagnon" Date: Sun, 8 Oct 2023 23:20:54 +0000 Subject: [PATCH 2/9] fixes apk python package name --- molecule/default/Dockerfile.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/molecule/default/Dockerfile.j2 b/molecule/default/Dockerfile.j2 index 0b331c5..5c80668 100644 --- a/molecule/default/Dockerfile.j2 +++ b/molecule/default/Dockerfile.j2 @@ -18,6 +18,6 @@ RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y pyth elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install /usr/bin/python3 /usr/bin/python3-config /usr/bin/dnf-3 sudo bash iproute && dnf clean all; \ elif [ $(command -v yum) ]; then yum makecache fast && yum install -y /usr/bin/python /usr/bin/python2-config sudo yum-plugin-ovl bash iproute && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml iproute2 && zypper clean -a; \ - elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \ + elif [ $(command -v apk) ]; then apk update && apk add --no-cache python3 sudo bash ca-certificates; \ elif [ $(command -v pacman) ]; then pacman --noconfirm -Suy python python-pip sudo openssh; \ elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates iproute2 && xbps-remove -O; fi From d9dfd5c6b5ca4ca8602fc43cec7ff2055a9fd936 Mon Sep 17 00:00:00 2001 From: "jerome.gagnon" Date: Sun, 8 Oct 2023 23:46:51 +0000 Subject: [PATCH 3/9] fixes some linting and ansible warnings --- .ansible-lint | 1 + molecule/default/verify.yml | 3 +++ 2 files changed, 4 insertions(+) diff --git a/.ansible-lint b/.ansible-lint index 2aea330..fc718c3 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -1,3 +1,4 @@ --- skip_list: - fqcn-builtins + - var-naming[no-role-prefix] \ No newline at end of file diff --git a/molecule/default/verify.yml b/molecule/default/verify.yml index f816e3f..a46eb2c 100644 --- a/molecule/default/verify.yml +++ b/molecule/default/verify.yml @@ -4,10 +4,13 @@ tasks: - name: Ensure Borgmatic is installed correctly command: borgmatic --version + changed_when: false - name: Ensure Borg is installed correctly command: borgmatic borg --version + changed_when: false - name: Ensure produced YAML is valid command: | yamllint -d "{extends: relaxed, rules: {line-length: {max: 120}}}" /etc/borgmatic/config.yaml + changed_when: false From d8c51011e72adb9c25d7b8f8b2a144f1ad047ee5 Mon Sep 17 00:00:00 2001 From: "jerome.gagnon" Date: Mon, 9 Oct 2023 00:18:54 +0000 Subject: [PATCH 4/9] bypasses tasks not relevent to docker install method --- tasks/01_install.yml | 1 + tasks/02_user_management.yml | 38 +++++++++++---------- tasks/03_create_key.yml | 66 ++++++++++++++++++++++++------------ tasks/05_configure.yml | 2 ++ tasks/07_install_timer.yml | 1 + 5 files changed, 69 insertions(+), 39 deletions(-) diff --git a/tasks/01_install.yml b/tasks/01_install.yml index e4478d9..5844b7a 100644 --- a/tasks/01_install.yml +++ b/tasks/01_install.yml @@ -19,6 +19,7 @@ - "{{ ansible_lsb.id }}.yml" - name: Install general dependencies (openssh) + when: borg_install_method != "docker" ansible.builtin.package: name: "{{ borg_dep_packages }}" state: present diff --git a/tasks/02_user_management.yml b/tasks/02_user_management.yml index fb396d1..e1ba985 100644 --- a/tasks/02_user_management.yml +++ b/tasks/02_user_management.yml @@ -1,25 +1,29 @@ --- # So in different positions in that role we need the user home # Since we cannot be sure that this FSH is compatible we will determine it. -- name: Get home dir +- name: User management when: - - borg_user == "root" + - borg_install_method != "docker" block: - - name: Get home if borg_user == "root" - ansible.builtin.user: - name: "{{ borg_user }}" - state: present - register: user_info - changed_when: false - check_mode: true # Important, otherwise user will be created + - name: Get home dir + when: + - borg_user == "root" + block: + - name: Get home if borg_user == "root" + ansible.builtin.user: + name: "{{ borg_user }}" + state: present + register: user_info + changed_when: false + check_mode: true # Important, otherwise user will be created - - name: Save the user_info, we need them for the home_dir - ansible.builtin.set_fact: - backup_user_info: "{{ user_info }}" + - name: Save the user_info, we need them for the home_dir + ansible.builtin.set_fact: + backup_user_info: "{{ user_info }}" -- name: Create user if borg_user != "root" - when: - - borg_user != "root" - ansible.builtin.include_tasks: - file: noauto_create_backup_user_and_group.yml + - name: Create user if borg_user != "root" + when: + - borg_user != "root" + ansible.builtin.include_tasks: + file: noauto_create_backup_user_and_group.yml ... diff --git a/tasks/03_create_key.yml b/tasks/03_create_key.yml index 3827d77..9ceffbf 100644 --- a/tasks/03_create_key.yml +++ b/tasks/03_create_key.yml @@ -1,28 +1,50 @@ --- -- name: Create SSH key (if neeeded) for {{ borg_user }} +- name: Create ssh key + when: + - borg_install_method != "docker" block: - - name: Ensure directory exist - ansible.builtin.file: - path: "{{ backup_user_info.home }}/.ssh/" - state: directory - mode: "0700" - owner: "{{ borg_user }}" - group: "{{ borg_group }}" + - name: Create SSH key (if needed) for {{ borg_user }} + block: + - name: Ensure directory exist + ansible.builtin.file: + path: "{{ backup_user_info.home }}/.ssh/" + state: directory + mode: "0700" + owner: "{{ borg_user }}" + group: "{{ borg_group }}" - - name: Generate an OpenSSH keypair - community.crypto.openssh_keypair: - path: "{{ borg_ssh_key_file_path }}" - mode: "0600" - type: "{{ borg_ssh_key_type }}" - owner: "{{ borg_user }}" - group: "{{ borg_group }}" + - name: Generate an OpenSSH keypair + when: not borg_ssh_private_key + community.crypto.openssh_keypair: + path: "{{ borg_ssh_key_file_path }}" + mode: "0600" + type: "{{ borg_ssh_key_type }}" + owner: "{{ borg_user }}" + group: "{{ borg_group }}" - - name: Read SSH key - ansible.builtin.slurp: - src: "{{ borg_ssh_key_file_path }}.pub" - register: backup_local_ssh_key + - name: Generate public key from private key content # Also ensure priv key content is sound + when: borg_ssh_private_key + delegate_to: localhost + community.crypto.openssl_publickey: + path: "{{ borg_ssh_key_file_path }}.pub" + privatekey_content: "{{ borg_ssh_private_key }}" + register: public_key - - name: Print key - ansible.builtin.debug: - msg: "The generated key is: {{ backup_local_ssh_key['content'] | b64decode }}" + - name: Copy provided OpenSSH private key + when: public_key.succeeded + ansible.builtin.copy: + content: "{{ borg_ssh_private_key }}" + dest: "{{ borg_ssh_key_file_path }}" + mode: "0600" + owner: "{{ borg_user }}" + group: "{{ borg_group }}" + + - name: Read SSH key + ansible.builtin.slurp: + src: "{{ borg_ssh_key_file_path }}.pub" + register: backup_local_ssh_key + + - name: Print key + ansible.builtin.debug: + msg: "The generated key is: {{ backup_local_ssh_key['content'] | b64decode }}" ... diff --git a/tasks/05_configure.yml b/tasks/05_configure.yml index e055c20..c517458 100755 --- a/tasks/05_configure.yml +++ b/tasks/05_configure.yml @@ -1,5 +1,7 @@ --- - name: Add Borgmatic config file + when: + - borg_install_method != "docker" block: - name: Ensure /etc/borgmatic exists ansible.builtin.file: diff --git a/tasks/07_install_timer.yml b/tasks/07_install_timer.yml index 4ca6afa..18e499d 100644 --- a/tasks/07_install_timer.yml +++ b/tasks/07_install_timer.yml @@ -2,6 +2,7 @@ - name: Install timer to run Borgmatic when: - borgmatic_timer is defined and borgmatic_timer | length > 0 + - borg_install_method != "docker" block: - name: Start timer install script ansible.builtin.include_tasks: From 28dde00c89975bdc2e444f7b7a037172749ea705 Mon Sep 17 00:00:00 2001 From: "jerome.gagnon" Date: Mon, 9 Oct 2023 00:22:30 +0000 Subject: [PATCH 5/9] implements docker install method --- .github/workflows/main.yml | 2 + defaults/main.yml | 4 + molecule/docker/Dockerfile.j2 | 23 +++++ molecule/docker/INSTALL.rst | 22 ++++ molecule/docker/converge.yml | 96 ++++++++++++++++++ molecule/docker/molecule.yml | 21 ++++ molecule/docker/vault.pw | 1 + molecule/docker/verify.yml | 33 ++++++ tasks/noauto_install_docker.yml | 172 ++++++++++++++++++++++++++++++++ templates/Dockerfile.j2 | 15 +++ templates/ansible_entry.sh.j2 | 9 ++ vars/Alpine.yml | 6 ++ vars/Archlinux.yml | 3 + vars/Debian.yml | 3 + vars/Fedora.yml | 3 + vars/ManjaroLinux.yml | 3 + vars/RedHat-8.yml | 3 + vars/RedHat-9.yml | 3 + vars/RedHat.yml | 3 + 19 files changed, 425 insertions(+) create mode 100644 molecule/docker/Dockerfile.j2 create mode 100644 molecule/docker/INSTALL.rst create mode 100644 molecule/docker/converge.yml create mode 100644 molecule/docker/molecule.yml create mode 100644 molecule/docker/vault.pw create mode 100644 molecule/docker/verify.yml create mode 100644 tasks/noauto_install_docker.yml create mode 100644 templates/Dockerfile.j2 create mode 100644 templates/ansible_entry.sh.j2 create mode 100644 vars/Alpine.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ef664fc..d86146b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,3 +19,5 @@ jobs: # uses: mxschmitt/action-tmate@v3.5 - name: Test using Molecule run: molecule test + - name: Test Docker scenario using Molecule + run: molecule test -s docker \ No newline at end of file diff --git a/defaults/main.yml b/defaults/main.yml index 02e0a65..e53f23e 100755 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -46,4 +46,8 @@ borgmatic_version: ">=1.7.11" borg_venv_path: "/opt/borgmatic" borg_user: "root" borg_group: "root" + +borgmatic_docker_image_name: "ansible_borgmatic" +borgmatic_docker_container_name: "ansible_borgmatic" +borgmatic_docker_timezone: "UTC" ... diff --git a/molecule/docker/Dockerfile.j2 b/molecule/docker/Dockerfile.j2 new file mode 100644 index 0000000..5c80668 --- /dev/null +++ b/molecule/docker/Dockerfile.j2 @@ -0,0 +1,23 @@ +# Molecule managed + +{% if item.registry is defined %} +FROM {{ item.registry.url }}/{{ item.image }} +{% else %} +FROM {{ item.image }} +{% endif %} + +{% if item.env is defined %} +{% for var, value in item.env.items() %} +{% if value %} +ENV {{ var }} {{ value }} +{% endif %} +{% endfor %} +{% endif %} + +RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python3 python3-pip sudo bash ca-certificates iproute2 python3-apt aptitude && apt-get clean; \ + elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install /usr/bin/python3 /usr/bin/python3-config /usr/bin/dnf-3 sudo bash iproute && dnf clean all; \ + elif [ $(command -v yum) ]; then yum makecache fast && yum install -y /usr/bin/python /usr/bin/python2-config sudo yum-plugin-ovl bash iproute && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \ + elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml iproute2 && zypper clean -a; \ + elif [ $(command -v apk) ]; then apk update && apk add --no-cache python3 sudo bash ca-certificates; \ + elif [ $(command -v pacman) ]; then pacman --noconfirm -Suy python python-pip sudo openssh; \ + elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates iproute2 && xbps-remove -O; fi diff --git a/molecule/docker/INSTALL.rst b/molecule/docker/INSTALL.rst new file mode 100644 index 0000000..6a44bde --- /dev/null +++ b/molecule/docker/INSTALL.rst @@ -0,0 +1,22 @@ +******* +Docker driver installation guide +******* + +Requirements +============ + +* Docker Engine + +Install +======= + +Please refer to the `Virtual environment`_ documentation for installation best +practices. If not using a virtual environment, please consider passing the +widely recommended `'--user' flag`_ when invoking ``pip``. + +.. _Virtual environment: https://virtualenv.pypa.io/en/latest/ +.. _'--user' flag: https://packaging.python.org/tutorials/installing-packages/#installing-to-the-user-site + +.. code-block:: bash + + $ pip install 'molecule[docker]' diff --git a/molecule/docker/converge.yml b/molecule/docker/converge.yml new file mode 100644 index 0000000..f1596e8 --- /dev/null +++ b/molecule/docker/converge.yml @@ -0,0 +1,96 @@ +--- +- name: Converge + hosts: all + pre_tasks: + - name: Set ssh server package name for non-Archlinux ansible_os_family + set_fact: + openssh_package: "openssh-server" + pip3_extra_args: "" + when: ansible_os_family != "Archlinux" + + - name: Set ssh server package name and pip3 argument for Archlinux ansible_os_family + set_fact: + openssh_package: "openssh" + pip3_extra_args: "--break-system-packages" + when: ansible_os_family == "Archlinux" + + - name: Install openssh + package: + name: "{{ openssh_package }}" + state: present + + - name: Define borg_source_directories + ansible.builtin.set_fact: + borg_source_directories_tmp: + - /srv/www + - /var/lib/automysqlbackup + + - name: Create backup source folders on the docker host + ansible.builtin.file: + path: "{{ item }}" + mode: "0777" + state: directory + with_items: "{{ borg_source_directories_tmp }}" + + - name: Define borg_repository + ansible.builtin.set_fact: + borg_repository_tmp: + - m5vz9gp4@m5vz9gp4.repo.borgbase.com:repo + - /local_borg_repo + + - name: Create local repository folders on the docker host + ansible.builtin.file: + path: "{{ item }}" + mode: "0777" + state: directory + with_items: "{{ borg_repository_tmp }}" + when: item[0] == "/" + + roles: + - role: borgbase.ansible_role_borgbackup + borg_install_method: docker + borgmatic_timer: cron + borg_repository: "{{ borg_repository_tmp }}" + borg_encryption_passphrase: CHANGEME + borg_source_directories: "{{ borg_source_directories_tmp }}" + borg_exclude_patterns: + - /srv/www/old-sites + borg_retention_policy: + keep_hourly: 3 + keep_daily: 7 + keep_weekly: 4 + keep_monthly: 6 + borgmatic_hooks: + before_backup: + - echo "`date` - Starting backup." + postgresql_databases: + - name: users + hostname: database1.example.org + port: 5433 + borg_ssh_private_key: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 65373636303732303236313234666230386333636233313631663135323734626265616532633064 + 3163346333616539663732636366626535326238623761320a336130633135643735613433636538 + 33333336656238306163303431393562303863633137646337633861346265353131396434393531 + 6564386438356330380a373138353364316535653338396164383861396538333336666436663832 + 32613439616136313331333336636232323231623363633661656632316237653633363466313734 + 35316262653366373137393761393835643166666436333635383334643636616436623030376234 + 33343565363863613161373561616237313138633765376263656536303565363838376163313963 + 37656431316335663030336236633663313937353362653639303836366436383334373132666334 + 39313562316330613131383738613136616631336461626362313764313637356233373437613962 + 31363564643266353737656261613232366336386230333963393935353763343236333564376462 + 36653538363131616133653463613633343036363931316334613136653265636262313235366434 + 31306562363034336431373535393364346435323130386265346431343836613135353430366534 + 61323861653464313763303261656430393930623664396630666133383038313939303030396362 + 34363435316434656462366339346637396134623337633133386638646463633063363133656164 + 35396237366363383637333662366437633361356466616137623362623439323433656562636238 + 66633964323831386435306163343566666533663363343262346332373764366635643961333130 + 63346431326432313234653132383664396165313538346161316264653235616161353833633234 + 31343663346434633863393934653631376334346666346437366639613032343632356635613932 + 62306361343336386435653939386339343066366531356632643730643330353931663239326130 + 39346364363263363332363637616133323761636437313138633630363237383363393432386362 + 33633330323536346430636234373032346663336630623334363363393661376531376337313066 + 64626434356535346461326339376435643738353463343035306433343630653335643635613939 + 37323564323130356338643237383966313539663132656533656434626166373839653435343835 + 62373131393235333934356133643963613665626532643164343063666632626561666330373930 + 6132 diff --git a/molecule/docker/molecule.yml b/molecule/docker/molecule.yml new file mode 100644 index 0000000..c726e3e --- /dev/null +++ b/molecule/docker/molecule.yml @@ -0,0 +1,21 @@ +--- +dependency: + name: galaxy +driver: + name: docker +platforms: + - name: docker-dind # Based on Alpine + image: docker:dind + privileged: True # required to have access to the dind + command: /usr/local/bin/dockerd-entrypoint.sh # Starts docker daemon +provisioner: + name: ansible + config_options: + defaults: + vault_password_file: "${MOLECULE_SCENARIO_DIRECTORY}/vault.pw" +verifier: + name: ansible +lint: | + set -e + yamllint . + ansible-lint . \ No newline at end of file diff --git a/molecule/docker/vault.pw b/molecule/docker/vault.pw new file mode 100644 index 0000000..7aa311a --- /dev/null +++ b/molecule/docker/vault.pw @@ -0,0 +1 @@ +password \ No newline at end of file diff --git a/molecule/docker/verify.yml b/molecule/docker/verify.yml new file mode 100644 index 0000000..7d6a26a --- /dev/null +++ b/molecule/docker/verify.yml @@ -0,0 +1,33 @@ +--- +- name: Verify + hosts: all + tasks: + - name: Set docker_cmd + ansible.builtin.set_fact: + docker_cmd: docker exec -i ansible_borgmatic + + - name: Ensure Borgmatic is installed correctly + command: "{{ docker_cmd }} borgmatic --version" + changed_when: false + + - name: Ensure Borg is installed correctly + command: "{{ docker_cmd }} borgmatic borg --version" + changed_when: false + + - name: Ensure produced YAML is valid + ansible.builtin.shell: | + {{ docker_cmd }} pip3 install yamllint && \ + {{ docker_cmd }} yamllint --list-files -d "{extends: relaxed, rules: {line-length: {max: 120}}}" /etc/borgmatic/config.yaml + changed_when: false + + - name: Ensure modified source path name is present in the config instead of the original + command: "{{ docker_cmd }} grep /sources/var/lib/automysqlbackup /etc/borgmatic/config.yaml" + changed_when: false + + - name: Ensure modified local repo path name is present in the config instead of the original + command: "{{ docker_cmd }} grep /repositories/local_borg_repo /etc/borgmatic/config.yaml" + changed_when: false + + - name: Ensure supercronic is running + command: "{{ docker_cmd }} pgrep supercronic" + changed_when: false diff --git a/tasks/noauto_install_docker.yml b/tasks/noauto_install_docker.yml new file mode 100644 index 0000000..e69eeef --- /dev/null +++ b/tasks/noauto_install_docker.yml @@ -0,0 +1,172 @@ +--- +- name: Install borgbackup with Docker + block: + + - name: Create temp directory for Docker build + ansible.builtin.tempfile: + state: directory + register: build_dir + changed_when: false + + - name: Install build dependencies + ansible.builtin.package: + name: "{{ borg_docker_packages }}" + state: present + + - name: Define Docker image tag based on borgmatic --version + ansible.builtin.set_fact: + borgmatic_docker_tag: "{{ borgmatic_version | regex_search('\\d+\\.\\d+(\\.\\d+){0,1}') }}" + + - name: Debug borgmatic_docker_tag + debug: + var: borgmatic_docker_tag + + - name: Assert source path exists + ansible.builtin.stat: + path: "{{ item }}" + register: source_path + failed_when: not source_path.stat.exists or (source_path.stat.exists and not source_path.stat.isdir) + with_items: "{{ borg_source_directories }}" + + - name: Assert local repository path exists + ansible.builtin.stat: + path: "{{ item }}" + register: source_path + failed_when: not source_path.stat.exists or (source_path.stat.exists and not source_path.stat.isdir) + when: item[0] == "/" + with_items: "{{ borg_repository }}" + + - name: Assert user and group are repositories + ansible.builtin.assert: + that: + - borg_user == "root" + - borg_group == "root" + fail_msg: For docker deployment, only "root" is supported for borg_user and borg_group + + - name: Build volume list from borg_source_directories and borg_repository + ansible.builtin.set_fact: + volumes: >- + {%- set volumes = [] -%} + {%- for dir in borg_source_directories -%} + {%- set _ = volumes.append(dir + ":/sources" + dir + ":ro") -%} + {%- endfor -%} + {%- for dir in borg_repository -%} + {%- if dir[0] == "/" -%} + {%- set _ = volumes.append(dir + ":/repositories" + dir) -%} + {%- endif -%} + {%- endfor -%} + {{ volumes }} + + - name: Debug volume list + ansible.builtin.debug: + var: volumes + verbosity: 1 + + - name: Modify borg_source_directories to reflect path in container + ansible.builtin.set_fact: + borg_source_directories: >- + {%- set sources = [] -%} + {%- for source in borg_source_directories -%} + {%- set _ = sources.append("/sources" + source) -%} + {%- endfor -%} + {{ sources }} + + - name: Debug borg_source_directories + ansible.builtin.debug: + var: borg_source_directories + verbosity: 1 + + - name: Modify borg_repository to reflect path in container + ansible.builtin.set_fact: + borg_repository: >- + {%- set repositories = [] -%} + {%- for repo in borg_repository -%} + {%- if repo[0] == "/" -%} + {%- set _ = repositories.append("/repositories" + repo) -%} + {%- else -%} + {%- set _ = repositories.append(repo) -%} + {%- endif -%} + {%- endfor -%} + {{ repositories }} + borg_repository_flat: "{{ borg_repository | join('|') }}" + + - name: Debug borg_repository + ansible.builtin.debug: + var: borg_repository + verbosity: 1 + + - name: Check if ssh repo in the list + when: + - not borg_ssh_private_key + - borg_repository_flat is match('|[^/]') + ansible.builtin.set_fact: + has_ssh_repo: true + + - name: Test if private key was provided + when: + - not borg_ssh_private_key + - has_ssh_repo + ansible.builtin.fail: + msg: "Private key content must be provided when using docker" + + - name: Copy private key + when: borg_ssh_private_key + changed_when: false + ansible.builtin.copy: + dest: "{{ build_dir.path }}/{{ borg_ssh_key_name }}" + mode: 0600 + content: "{{ borg_ssh_private_key }}" + validate: ssh-keygen -yf %s # Also ensure priv key content is sound + + - name: Generate public key from private key + when: borg_ssh_private_key + changed_when: false + failed_when: not public_key.stdout.startswith("ssh") + register: public_key + ansible.builtin.command: "ssh-keygen -yf {{ build_dir.path }}/{{ borg_ssh_key_name }}" + + - name: Copy other files to build folder for docker build + changed_when: false + ansible.builtin.template: + dest: "{{ build_dir.path }}/{{ item | basename | regex_replace('\\.j2$', '') }}" + src: "{{ item }}" + mode: 0600 + with_items: + - Dockerfile.j2 + - config.yaml.j2 + - ansible_entry.sh.j2 + + - name: Build docker image + changed_when: false # will make the idempotency test fail otherwise + community.docker.docker_image: + name: "{{ borgmatic_docker_image_name }}:{{ borgmatic_docker_tag }}" + source: build + state: present + force_source: true + build: + path: "{{ build_dir.path }}" + pull: true + rm: false + args: + PUBLIC_KEY: "{{ public_key }}" + PRIVATE_KEY: "{{ borg_ssh_private_key }}" + + - name: Start container + changed_when: false # will make the idempotency test fail otherwise + community.docker.docker_container: + name: "{{ borgmatic_docker_container_name }}" + image: "{{ borgmatic_docker_image_name }}:{{ borgmatic_docker_tag }}" + volumes: "{{ volumes }}" + restart_policy: unless-stopped + labels: + ansible_borgmatic_managed: "1" + env: + BACKUP_CRON: "{{ borgmatic_timer_minute }} {{ borgmatic_timer_hour }} * * * borgmatic -c /etc/borgmatic/{{ borgmatic_config_name }}" + TZ: "{{ borgmatic_docker_timezone }}" + + always: + - name: Delete build folder + ansible.builtin.file: + path: "{{ build_dir.path }}" + state: absent + changed_when: false \ No newline at end of file diff --git a/templates/Dockerfile.j2 b/templates/Dockerfile.j2 new file mode 100644 index 0000000..b15f60f --- /dev/null +++ b/templates/Dockerfile.j2 @@ -0,0 +1,15 @@ +FROM ghcr.io/borgmatic-collective/borgmatic:{{ borgmatic_docker_tag }} + +LABEL "ansible_borgmatic_managed"="1" + +COPY config.yaml /etc/borgmatic/{{ borgmatic_config_name }} + +# Those keys will be copied at /root/.ssh at runtime. This is required because of the anom volumes defined in the upstream image +ARG PUBLIC_KEY="" +ARG PRIVATE_KEY="" +RUN if [ ! -z "$PUBLIC_KEY" ]; then echo "$PUBLIC_KEY" > /{{ borg_ssh_key_name}}.pub; fi +RUN if [ ! -z "$PRIVATE_KEY" ]; then echo "$PRIVATE_KEY" > /{{ borg_ssh_key_name}}; fi +COPY ansible_entry.sh / +RUN chmod 700 /ansible_entry.sh + +ENTRYPOINT [ "/ansible_entry.sh" ] \ No newline at end of file diff --git a/templates/ansible_entry.sh.j2 b/templates/ansible_entry.sh.j2 new file mode 100644 index 0000000..c54b49b --- /dev/null +++ b/templates/ansible_entry.sh.j2 @@ -0,0 +1,9 @@ +#!/bin/sh + +# We need to copy ssh keys at runtime because of the built-in volumes in the upstream Docker image definition +if [ -f "/{{ borg_ssh_key_name }}.pub" ]; then mv /{{ borg_ssh_key_name }}.pub /root/.ssh; fi +if [ -f "/{{ borg_ssh_key_name }}" ]; then mv /{{ borg_ssh_key_name }} /root/.ssh; fi + +echo "$BACKUP_CRON" > /etc/borgmatic.d/crontab.txt + +exec env SUPERCRONIC_EXTRA_FLAGS=-debug /entry.sh "$@" \ No newline at end of file diff --git a/vars/Alpine.yml b/vars/Alpine.yml new file mode 100644 index 0000000..31969fb --- /dev/null +++ b/vars/Alpine.yml @@ -0,0 +1,6 @@ +--- +borg_docker_packages: + - py3-docker-py + +python_bin: python3 +pip_bin: pip3 diff --git a/vars/Archlinux.yml b/vars/Archlinux.yml index 48c3688..ebc1d54 100644 --- a/vars/Archlinux.yml +++ b/vars/Archlinux.yml @@ -10,6 +10,9 @@ borg_pip_packages: - python-pip - python-setuptools +borg_docker_packages: + - python-docker + borg_distro_packages: - borg - borgmatic diff --git a/vars/Debian.yml b/vars/Debian.yml index c5fb44b..44ee54d 100644 --- a/vars/Debian.yml +++ b/vars/Debian.yml @@ -16,6 +16,9 @@ borg_pip_packages: - python3-msgpack - python3-venv +borg_docker_packages: + - python3-docker + borg_distro_packages: - borgbackup - borgmatic diff --git a/vars/Fedora.yml b/vars/Fedora.yml index a5583ec..977d5ee 100644 --- a/vars/Fedora.yml +++ b/vars/Fedora.yml @@ -16,6 +16,9 @@ borg_pip_packages: - python3-setuptools - python3-Cython +borg_docker_packages: + - python3-docker + borg_distro_packages: - borgbackup - borgmatic diff --git a/vars/ManjaroLinux.yml b/vars/ManjaroLinux.yml index fd4a65b..7e5b179 100644 --- a/vars/ManjaroLinux.yml +++ b/vars/ManjaroLinux.yml @@ -16,6 +16,9 @@ borg_pip_packages: # untested - python3-msgpack - python3-venv +borg_docker_packages: + - python3-docker + borg_distro_packages: - borg - borgmatic diff --git a/vars/RedHat-8.yml b/vars/RedHat-8.yml index 4497b8d..6936ec6 100644 --- a/vars/RedHat-8.yml +++ b/vars/RedHat-8.yml @@ -16,6 +16,9 @@ borg_pip_packages: - python3-setuptools - python3-virtualenv +borg_docker_packages: + - python3-docker + borg_distro_packages: - borgbackup - borgmatic diff --git a/vars/RedHat-9.yml b/vars/RedHat-9.yml index 2b900f4..e0faa41 100644 --- a/vars/RedHat-9.yml +++ b/vars/RedHat-9.yml @@ -16,6 +16,9 @@ borg_pip_packages: - python3-setuptools # - python3-virtualenv +borg_docker_packages: + - python3-docker + borg_distro_packages: - borgbackup - borgmatic diff --git a/vars/RedHat.yml b/vars/RedHat.yml index 3115a75..ebf6423 100644 --- a/vars/RedHat.yml +++ b/vars/RedHat.yml @@ -15,6 +15,9 @@ borg_pip_packages: - python36-devel - python-setuptools +borg_docker_packages: + - python36-docker + borg_distro_packages: - borgbackup - borgmatic From 23aae2006f718470af4c01266e31e3067d8a85a9 Mon Sep 17 00:00:00 2001 From: "jerome.gagnon" Date: Mon, 9 Oct 2023 00:23:21 +0000 Subject: [PATCH 6/9] updates doc, argument specs and meta --- README.md | 26 +++++++++++++++++++++++--- meta/arguments_specs.yml | 25 +++++++++++++++++++++++-- meta/main.yml | 1 + 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index dd417ba..c348414 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,23 @@ Works great with [BorgBase.com](https://www.borgbase.com) - Simple and Secure Ho keep_monthly: 6 ``` +## Example playbook using Docker +``` +- hosts: all + roles: + - role: borgbase.ansible_role_borgbackup + borg_install_method: docker + borgmatic_timer: cron + borg_repository: ssh://xxxxxx@xxxxxx.repo.borgbase.com/./repo + borg_encryption_passphrase: CHANGEME + borg_source_directories: + - /var/www + borg_ssh_private_key: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 65373636303732303236313234666230386333636233313631663135323734626265616532633064 + 316334...truncated +``` ## Installation @@ -87,7 +103,7 @@ $ git clone https://github.com/borgbase/ansible-role-borgbackup.git roles/ansibl - `borg_encryption_passphrase`: Password to use for repokey or keyfile. Empty if repo is unencrypted. - `borg_exclude_from`: Read exclude patterns from one or more separate named files, one pattern per line. - `borg_exclude_patterns`: Paths or patterns to exclude from backup. See [official documentation](https://borgbackup.readthedocs.io/en/stable/usage/help.html#borg-help-patterns) for more. -- `borg_install_method`: By default `pip` is used to install borgmatic. To install via your distributions package manager set this to `package` and (if needed) overwrite the `borg_distro_packages` variable to contain your distributions package names required to install borgmatic. Note that many distributions ship outdated versions of borgbackup and borgmatic; use at your own risk. +- `borg_install_method`: By default `pip` is used to install borgmatic. To install via your distributions package manager set this to `package` and (if needed) overwrite the `borg_distro_packages` variable to contain your distributions package names required to install borgmatic. Note that many distributions ship outdated versions of borgbackup and borgmatic; use at your own risk. To install via a Docker container, set this to "docker". Docker must be installed on target host. - `borg_require_epel`: When using `borg_install_method: package` on RHEL-based distributions, the EPEL repo is required. To disable the check (e.g. when using a custom mirror instead of the `epel-release` package), set this to `false`. Defaults to `{{ ansible_os_family == 'RedHat' and ansible_distribution != 'Fedora' }}` (i.e. `true` on Enterprise Linux-based distros). - `borg_lock_wait_time`: Config maximum seconds to wait for acquiring a repository/cache lock. Defaults to 5 seconds. - `borg_one_file_system`: Don't cross file-system boundaries. Defaults to `true` @@ -99,6 +115,7 @@ $ git clone https://github.com/borgbase/ansible-role-borgbackup.git roles/ansibl - `borg_ssh_key_name`: Name of the SSH public and pivate key. Default `id_ed25519` - `borg_ssh_key_file_path`: SSH-key to be used. Default `~/.ssh/{{ borg_ssh_key_name }}` - `borg_ssh_key_type`: The algorithm used to generate the SSH private key. Choose: `rsa`, `dsa`, `rsa1`, `ecdsa`, `ed25519`. Default: `ed25519` +- `borg_ssh_private_key`: Content of the ssh private key, may you want to provide it. Only keys without passphrase is supported. Most useful for Docker deployments. IMPORTANT! Be sure to provide the content of this variable via an Ansible Vault. - `borg_ssh_command`: Command to use instead of just "ssh". This can be used to specify SSH options. - `borg_version`: Force a specific borg version to be installed - `borg_venv_path`: Path to store the venv for `borg(backup)` and `borgmatic` @@ -115,9 +132,12 @@ $ git clone https://github.com/borgbase/ansible-role-borgbackup.git roles/ansibl - `borgmatic_store_ctime`: Store ctime into archive. Defaults to `true` - `borgmatic_version`: Force a specific borgmatic version to be installed -- `borg_user`: Name of the User to create Backups (service account) -- `borg_group`: Name of the Group to create Backups (service account) +- `borg_user`: Name of the User to create Backups (service account). When using Docker, must be root. +- `borg_group`: Name of the Group to create Backups (service account). When using Docker, must be root. +- `borgmatic_docker_image_name`: When using borg_install_method=docker, name docker image to build. Defaults to `ansible_borgmatic` +- `borgmatic_docker_container_name`: When using borg_install_method=docker, name of the docker container. Defaults to `ansible_borgmatic` +- `borgmatic_docker_timezone`: Timezone to use when using borg_install_method=docker. Defaults to `UTC` ## Contributing diff --git a/meta/arguments_specs.yml b/meta/arguments_specs.yml index 2908b4f..dfae5b1 100644 --- a/meta/arguments_specs.yml +++ b/meta/arguments_specs.yml @@ -30,12 +30,14 @@ argument_specs: type: str required: false default: pip + choices: [pip, package, docker] description: | By default pip is used to install borgmatic. To install via your distributions package manager set this to package and (if needed) overwrite the borg_distro_packages variable to contain your distributions package names required to install borgmatic. Note that many distributions ship outdated versions of borgbackup and borgmatic; use at your own risk. + To install via a Docker container, set this to "docker". Docker must be installed on target host borgmatic_config_name: type: str required: false @@ -44,11 +46,11 @@ argument_specs: borg_user: type: str default: root - description: Name of the User to create Backups (Service Account) + description: Name of the User to create Backups (Service Account). When using Docker, must be root. borg_group: type: str default: root - description: Name of the Group to create Backups (Service Account) + description: Name of the Group to create Backups (Service Account). When using Docker, must be root. borg_source_directories: type: List default: "/etc/hostname" @@ -105,6 +107,10 @@ argument_specs: type: str required: false description: Path to ssh-key + borg_ssh_private_key: + type: str + required: false + description: Content of the ssh private key, may you want to provide it. Only keys without passphrase is supported. Most useful for Docker deployments. IMPORTANT! Be sure to provide the content of this variable via an Ansible Vault. borg_ssh_command: type: str description: Command to use instead of just ssh. This can be used to specify ssh options. @@ -180,3 +186,18 @@ argument_specs: type: str required: false description: Name of the SSH public and private key + borgmatic_docker_image_name: + type: str + required: false + default: ansible_borgmatic + description: When using borg_install_method=docker, name docker image to build + borgmatic_docker_container_name: + type: str + required: false + default: ansible_borgmatic + description: When using borg_install_method=docker, name of the docker container + borgmatic_docker_timezone: + type: str + required: false + default: UTC + description: Timezone to use when using borg_install_method=docker diff --git a/meta/main.yml b/meta/main.yml index 0438447..aa17767 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -21,6 +21,7 @@ galaxy_info: - name: ArchLinux versions: - all + - name: Docker galaxy_tags: - backup - cloud From 4bbe1ec9fef110a7590da6f4a4a043385018b740 Mon Sep 17 00:00:00 2001 From: "jerome.gagnon" Date: Mon, 9 Oct 2023 01:24:52 +0000 Subject: [PATCH 7/9] fixes broken ssh key generation --- tasks/03_create_key.yml | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/tasks/03_create_key.yml b/tasks/03_create_key.yml index 9ceffbf..d3e7e4e 100644 --- a/tasks/03_create_key.yml +++ b/tasks/03_create_key.yml @@ -14,7 +14,7 @@ group: "{{ borg_group }}" - name: Generate an OpenSSH keypair - when: not borg_ssh_private_key + when: borg_ssh_private_key is undefined community.crypto.openssh_keypair: path: "{{ borg_ssh_key_file_path }}" mode: "0600" @@ -22,16 +22,8 @@ owner: "{{ borg_user }}" group: "{{ borg_group }}" - - name: Generate public key from private key content # Also ensure priv key content is sound - when: borg_ssh_private_key - delegate_to: localhost - community.crypto.openssl_publickey: - path: "{{ borg_ssh_key_file_path }}.pub" - privatekey_content: "{{ borg_ssh_private_key }}" - register: public_key - - name: Copy provided OpenSSH private key - when: public_key.succeeded + when: borg_ssh_private_key is defined ansible.builtin.copy: content: "{{ borg_ssh_private_key }}" dest: "{{ borg_ssh_key_file_path }}" @@ -39,6 +31,22 @@ owner: "{{ borg_user }}" group: "{{ borg_group }}" + - name: Generate public key from private key + when: borg_ssh_private_key is defined + changed_when: false + failed_when: not public_key.stdout.startswith("ssh") + register: public_key + ansible.builtin.command: "ssh-keygen -yf {{ borg_ssh_key_file_path }}" + + - name: Copy provided OpenSSH public key + when: borg_ssh_private_key is defined + ansible.builtin.copy: + content: "{{ public_key.stdout }}" + dest: "{{ borg_ssh_key_file_path }}.pub" + mode: "0666" + owner: "{{ borg_user }}" + group: "{{ borg_group }}" + - name: Read SSH key ansible.builtin.slurp: src: "{{ borg_ssh_key_file_path }}.pub" From ad35507f7b0b451d5f9694a5026d1eafa0f35db7 Mon Sep 17 00:00:00 2001 From: "jerome.gagnon" Date: Mon, 9 Oct 2023 01:41:27 +0000 Subject: [PATCH 8/9] removes debug leftovers --- tasks/noauto_install_docker.yml | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/tasks/noauto_install_docker.yml b/tasks/noauto_install_docker.yml index e69eeef..7823cd0 100644 --- a/tasks/noauto_install_docker.yml +++ b/tasks/noauto_install_docker.yml @@ -17,10 +17,6 @@ ansible.builtin.set_fact: borgmatic_docker_tag: "{{ borgmatic_version | regex_search('\\d+\\.\\d+(\\.\\d+){0,1}') }}" - - name: Debug borgmatic_docker_tag - debug: - var: borgmatic_docker_tag - - name: Assert source path exists ansible.builtin.stat: path: "{{ item }}" @@ -57,11 +53,6 @@ {%- endfor -%} {{ volumes }} - - name: Debug volume list - ansible.builtin.debug: - var: volumes - verbosity: 1 - - name: Modify borg_source_directories to reflect path in container ansible.builtin.set_fact: borg_source_directories: >- @@ -71,11 +62,6 @@ {%- endfor -%} {{ sources }} - - name: Debug borg_source_directories - ansible.builtin.debug: - var: borg_source_directories - verbosity: 1 - - name: Modify borg_repository to reflect path in container ansible.builtin.set_fact: borg_repository: >- @@ -90,11 +76,6 @@ {{ repositories }} borg_repository_flat: "{{ borg_repository | join('|') }}" - - name: Debug borg_repository - ansible.builtin.debug: - var: borg_repository - verbosity: 1 - - name: Check if ssh repo in the list when: - not borg_ssh_private_key From 99909dc87c73e91c7c71c010fe5894116229d4bb Mon Sep 17 00:00:00 2001 From: Jerome Gagnon Date: Sun, 11 Feb 2024 22:15:35 -0500 Subject: [PATCH 9/9] fixes borg_source_directories and borg_repository not being overwritten by set_fact --- tasks/noauto_install_docker.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tasks/noauto_install_docker.yml b/tasks/noauto_install_docker.yml index 7823cd0..a0522f8 100644 --- a/tasks/noauto_install_docker.yml +++ b/tasks/noauto_install_docker.yml @@ -55,16 +55,20 @@ - name: Modify borg_source_directories to reflect path in container ansible.builtin.set_fact: - borg_source_directories: >- + borg_source_directories_tmp: >- {%- set sources = [] -%} {%- for source in borg_source_directories -%} {%- set _ = sources.append("/sources" + source) -%} {%- endfor -%} {{ sources }} + - name: Overwrite borg_source_directories fact + ansible.builtin.set_fact: + borg_source_directories: "{{ borg_source_directories_tmp }}" + - name: Modify borg_repository to reflect path in container ansible.builtin.set_fact: - borg_repository: >- + borg_repository_tmp: >- {%- set repositories = [] -%} {%- for repo in borg_repository -%} {%- if repo[0] == "/" -%} @@ -76,6 +80,10 @@ {{ repositories }} borg_repository_flat: "{{ borg_repository | join('|') }}" + - name: Overwrite borg_repository fact + ansible.builtin.set_fact: + borg_repository: "{{ borg_repository_tmp }}" + - name: Check if ssh repo in the list when: - not borg_ssh_private_key