From c1ee8c8f5eb3e499d4ff1fd98772969f5d7b315d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20H=C3=B6sel?= Date: Sun, 31 Mar 2024 19:49:27 +0200 Subject: [PATCH] borgmatic: Rework for Borgmatic 1.7 and 1.8 (#176) borgmatic: rework for borgmatic 1.8 This patch simplifies the borgmatic role by using a single config variable "borgmatic_config" instead of individual variables. This allows the user to specify either a section-style borgmatic config or a newer unified configuration (1.8 or later). We also use this opportunity to bring other role parts up-to-date, allowing the role to be more decoupled from borgmatics specifics. BREAKING CHANGES The format for borgmatic configuration variables is now a single dict "borgmatic_config" instead of individual entries. New SSH keys managed by this role now use ed25519 instead of RSA. This does not affect existing keys. The scheduled check job feature has been removed, please see the README for how to migrate to borgmatics builtin check timer feature. --- roles/borgmatic/README.md | 230 ++++++++++-------- roles/borgmatic/defaults/main.yml | 11 +- roles/borgmatic/molecule/default/converge.yml | 24 +- roles/borgmatic/molecule/default/molecule.yml | 11 + .../remote-repo-unmanaged-ssh/converge.yml | 17 +- .../remote-repo-unmanaged-ssh/molecule.yml | 14 ++ .../molecule/remote-repo/converge.yml | 18 +- .../molecule/remote-repo/molecule.yml | 14 ++ roles/borgmatic/molecule/verify.yml | 7 - roles/borgmatic/tasks/check.yml | 52 ++++ roles/borgmatic/tasks/config.yml | 20 -- roles/borgmatic/tasks/config_schedule.yml | 18 ++ roles/borgmatic/tasks/config_ssh.yml | 38 +++ .../tasks/config_ssh_known_hosts.yml | 38 +++ roles/borgmatic/tasks/install.yml | 14 -- roles/borgmatic/tasks/main.yml | 78 ++++-- roles/borgmatic/tasks/schedule.yml | 46 ---- roles/borgmatic/tasks/ssh.yml | 37 --- roles/borgmatic/tasks/ssh_known_hosts.yml | 30 --- .../templates/borgmatic-check.service.j2 | 51 ---- .../templates/borgmatic-check.timer.j2 | 11 - roles/borgmatic/templates/config.yaml.j2 | 16 +- roles/borgmatic/vars/main.yml | 6 - 23 files changed, 401 insertions(+), 400 deletions(-) create mode 100644 roles/borgmatic/tasks/check.yml delete mode 100644 roles/borgmatic/tasks/config.yml create mode 100644 roles/borgmatic/tasks/config_schedule.yml create mode 100644 roles/borgmatic/tasks/config_ssh.yml create mode 100644 roles/borgmatic/tasks/config_ssh_known_hosts.yml delete mode 100644 roles/borgmatic/tasks/install.yml delete mode 100644 roles/borgmatic/tasks/schedule.yml delete mode 100644 roles/borgmatic/tasks/ssh.yml delete mode 100644 roles/borgmatic/tasks/ssh_known_hosts.yml delete mode 100644 roles/borgmatic/templates/borgmatic-check.service.j2 delete mode 100644 roles/borgmatic/templates/borgmatic-check.timer.j2 diff --git a/roles/borgmatic/README.md b/roles/borgmatic/README.md index b7e7ddd..f9dbd1d 100644 --- a/roles/borgmatic/README.md +++ b/roles/borgmatic/README.md @@ -2,40 +2,40 @@ Install borg+borgmatic and setup a system backup job. -This role installs borg and [borgmatic](https://torsion.org/borgmatic/) (a configuration-driven wrapper around borg), -then optionally sets up a scheduled backup job. More specifically, this role will: +This role can perform the following actions for you: -- Create a custom borgmatic configuration based on simple Ansible variables -- Manage the SSH client keys and known_hosts file for you -- Setup a schedule using a systemd timer -- And more! See the role variables below +- Install borg and [borgmatic](https://torsion.org/borgmatic/) (a configuration-driven wrapper around borg) +- Set up a scheduled backup job +- Create a custom borgmatic configuration based on Ansible variables +- Manage the SSH client keys and known_hosts file for your remote repositories +- Setup a schedules using systemd timers ## Requirements -- The following distributions are currently supported: - - Ubuntu: 20.04 LTS, 22.04 LTS - - Debian: 11 - - There are no plans to support CentOS/RHEL-based distros right now -- This role requires root access. Make sure to run this role with `become: yes` or equivalent -- Supported Borgmatic versions: 1.5 and up +- A distribution with `borgbackup` and `borgmatic` packages in its repo + - This role supports Borgmatic versions `1.5` and newer + - We test the following distributions, other distros may work but are unsupported: + - Ubuntu 20.04 LTS, 22.04 LTS + - Debian 11, 12 +- **This role requires root access**. Make sure to run this role with `become: yes` or as the root user ## Role Variables -### Limit executed Components +### Select Tasks to Run -To limit execution of this role to specific components, use the variables below. +You can disable individual parts of this role if you don't have a use for them. By default, all components are executed. | Variable | Component/Description | Notes | |-----------|----------|-------| -| `borgmatic_install` | Installation of borg + borgmatic | | -| `borgmatic_setup_backup` | Configuration of the backup job/environment | Disabling this variable will cause all steps below to be skipped. -| `borgmatic_init_repos` | Initialization of repositories defined in `borgmatic_location_repositories`, both remote and local -| `borgmatic_ssh_manage_key` | Management of a client SSH key for borgmatic (see [below](#ssh-management)) | -| `borgmatic_ssh_manage_known_hosts` | Management of the borgmatic-specific known_hosts file | -| `borgmatic_manage_config` | Generation of the borgmatic config file | -| `borgmatic_manage_schedule` | Generation and configuration of the systemd timer + service used for regular backups | Disabling this variable will also disable the check job -| `borgmatic_schedule_check_job` | Generation of a separate job for running backup repo checks | +| `borgmatic_install` | Install of borg + borgmatic | | +| `borgmatic_setup_backup` | Configure the backup job/environment | **Disabling this variable will cause all the steps listed below to be skipped.** +| `borgmatic_init_repos` | Initialize the repositories defined in `borgmatic_location_repositories` +| `borgmatic_ssh_manage_key` | Setup borgmatic to use a dedicated SSH private key for remote repositories (see [below](#ssh-management)) | +| `borgmatic_ssh_manage_known_hosts` | Setup borgmatic to use a custom known_hosts file for remote repositories | +| `borgmatic_manage_config` | Generate the borgmatic config file | +| `borgmatic_manage_schedule` | Generate a the systemd timer + service for regular backups | **Disabling this variable will also disable the check job.** +| `borgmatic_schedule_check_job` | Generate of a separate job for running backup repo checks | ### General @@ -49,17 +49,43 @@ By default, all components are executed. - Default: `repokey` ##### `borgmatic_run_backup` -- Schedule a backup job when this role finishes execution +- Run the backup job at the end of this roles execution - Note that enabling this will cause the role be non-idempotent (it will report changed tasks on every run) - Default: `false` -### SSH Management +#### `borgmatic_run_backup_no_block` +- Whether to wait for the backup job to complete when launched through `borgmatic_run_backup` +- If enabled, the backup job will be launched in the background and Ansible execution can continue +- Default: `true` -If you are backing up to a remote host via SSH, this role can manage the client ssh key and known_hosts file for you. +### Configuration + +The [borgmatic configuration](https://torsion.org/borgmatic/docs/reference/configuration/) is directly from the `borgmatic_config` variable. +To set a configuration option for borgmatic, just add the corresponding key to `borgmatic_config`. +See [above](#example-playbook) for an example. + +There are a few special parameters that require extra attention, listed below: + +- `source_directories`: + - Required for borgmatic to work +- `repositories`: + - Required for borgmatic to work + - Read by the role for scanning remote hosts into the `known_hosts` file if `borgmatic_ssh_manage_known_hosts` is enabled +- `keep_` + - At least one `keep_` option is required for borgmatic to work +- `encryption_passphrase` + - Required for initializing the repository if `borgmatic_init_repos` is enabled (default: `true`) +- `ssh_command`: + - If `borgmatic_ssh_manage_key` is enabled, `" -i {{ borgmatic_ssh_key_path }}"` will be appended to point SSH to the borgmatic private key + - If you want to supply your own private key to borgmatic, please disable `borgmatic_ssh_manage_key` + - If `borgmatic_ssh_manage_known_hosts` is enabled, `" -o UserKnownHostsFile={{ borgmatic_ssh_known_hosts_file }}"` + - If you want to supply your own known_hosts file (or use the system default), please disable `borgmatic_ssh_manage_known_hosts` -This role always executes backups as the root user, but it uses its own ssh user configuration as to -not interfere with other processes on the system running as root. -You can disable this behavior if you want to use the default root ssh settings. +### SSH Management + +Unless configured otherwise, borgmatic will use the users (always `root` in case of this role) default SSH config/key for connecting to remote repositories. +By default, this role instead configures a separate SSH environment for borgmatic, so that there is no interference with other services running as root. +You can customize this behavior with the variables below ##### `borgmatic_ssh_manage_key` - If set to `true`, the role will setup borg to use a custom ssh key found at `{{ borgmatic_ssh_key_path }}`. @@ -72,6 +98,12 @@ You can disable this behavior if you want to use the default root ssh settings. - You can set this to an already existing ssh key if you don't want to use the one generated by this role - Default: `{{ borgmatic_config_path }}/id_rsa` +##### `borgmatic_ssh_key_gen_options` +- Set the `ssh-keygen` options to use when generating the key, such as algorithm and key length + - For RSA, use `-t rsa -b <2048/4096>` +- The following options are added by the role: `-f {{ borgmatic_ssh_key_path }} -N '' -q` +- Default: `-t ed25519` + ##### `borgmatic_ssh_manage_known_hosts` - If set to `true`, the role will create and a borgmatic-specific known_hosts file that borg will then save all remote server fingerprints to. - If set to `false`, borgmatic will use the default root known_hosts file instead. @@ -82,47 +114,6 @@ You can disable this behavior if you want to use the default root ssh settings. - You can set this to an existing known_hosts file if you don't want to use the one generated by this role - Default: `{{ borgmatic_config_path }}/known_hosts` -### Backup settings - -This role supports all borgmatic configuration parameters as variables. - -The following settings correspond to the configuration options for [borgmatic](https://torsion.org/borgmatic/docs/reference/configuration/). -See the official page for more details. These parameters are templated out into the borgmatic config, so borg placeholders and the like should work. - -Aside from the required parameters that are passed to this role (see below), *all other parameters are optional and left undefined by default.*. This means that unless you specify them, borgmatic will just use its internal defaults for these values. - -The format for variable names is: `borgmatic_{section}_{parameter}`. For example, a borgmatic configuration like this: - -```yaml -location: - source_directories: - - /home - - /etc - - /var -storage: - encryption_passcommand: secret-tool lookup borg-repository repo-name -``` - -can be represented like so: - -```yaml -borgmatic_location_source_directories: - - /home - - /etc - - /var -borgmatic_storage_encryption_passcomand: secret-tool lookup borg-repository repo-name - -``` - -The following parameters are either required or need extra attention: - -| Name | Description | Required | Default | -|------|-------------|:--------:|---------| -| `location_source_directories` | List of directories to backup | X | `["/etc", "/home", "/var"]` | -| `location_repositories` | Paths to target repositories, local or remote | X | undefined | -| `storage_encryption_passphrase` | Passphrase with which to encrypt the repository | If `borgmatic_init_repos` is set to `true` | undefined -| `retention_keep_*` | At least one `keep_` parameter is required for retention to work properly | X | undefined - ### Schedule settings These variables control the systemd timer and service used to run borgmatic periodically. @@ -138,57 +129,82 @@ The prefix for all variables in this section is: `borgmatic_schedule_` | `persistent` | Whether to immoderately run the backup job if the host "missed" its last run (the random delay still applies) | | `false` | | `wakeup` | Whether to wake the system for the backup job if it is in standby. May or may not be supported | | `false` | +#### Scheduling A Separate Check Job -### Scheduling A Separate Check Job +Previous versions of this role let you set up a separate check job to run at a different time than borgmatic itself, to prevent long backup runs due to checks. +This feature has been superseded by a built-in Borgmatic config option, please see the documentation [here](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/#check-frequency) for how to enable it. -On larger repositories, checks may take a very long time to complete. -If you do not want your checks to run on every backup, there are two options: +If you previously used this role to configure a separate job, please manually delete the old check job before adjusting your config: -For borgmatic 1.6.2 and newer, you can use the "Check Frequency" feature described [here](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/#check-frequency). This method is preferred if possible as it prevents conflicts and only requires a few configuration parameters. -For older borgmatic versions, you can use this roles capability to setup a separate check job described below. -This check job can be configured to perform one or more of the following actions: `repository`, `archives`, `data` (implies archive), `extract`. -Please see the [borg documentation](https://borgbackup.readthedocs.io/en/stable/usage/check.html) for more information about these options. +1. Disable the check timer: `systemctl disable --now borgmatic-check.timer` +2. Stop the check service if it is running: `systemctl stop borgmatic-check.service` +3. Delete the following unit files: + - `/etc/systemd/system/borgmatic-check.timer` + - `/etc/systemd/system/borgmatic-check.service` +4. Reload systemd `systemctl daemon-reload` ---- -**NOTE** +## Example Playbooks -Make sure that the check and backup job timers don't run at the same time! If they do, it is very likely that one of them will fail due to a missing lock on the borg repository. +This will setup a backup job to archive important directories onto a remote server that has a repository configured ---- +For Borgmatic 1.8 and newer: -Note that the values configured in [Schedule settings](#schedule-settings) will affect the check job (except for `backup` time of course). +```yaml +- hosts: all + become: yes # this role requires root! + tasks: + - name: Setup backups with borgmatic + ansible.builtin.include_role: maxhoesel.borgbackup.borgmatic + vars: + borgmatic_config: + # What do we want to backup? + source_directories: + - /home + - /etc + - /var -##### `borgmatic_schedule_check_job` -- Whether to enable or disable the separate check job -- Default: `false` + # Where do we want to backup to? + repositories: + - path: ssh://borg@remote-backup-server.local/./my_repository_name + label: backupserver -##### `borgmatic_schedule_check_job_time` -- [systemd time expression](https://www.freedesktop.org/software/systemd/man/systemd.time.html) defining when the job should run -- Make sure this time is far enough apart from the backup job to prevent errors due to conflict locks. -- Default: `Mon 12:00` + # What's our encryption password? + # This is required if `borgmatic_init_repos` is set to true (default) + encryption_passphrase: "mysupersecretpassphrase" -##### `borgmatic_schedule_check_job_checks` -- List of checks to run (see above) -- Default: `['repository', 'archive']` + # How many backups to we want to keep? + keep_daily: 7 +``` -## Example Playbooks +For Borgmatic 1.7 and older: ```yaml - hosts: all - roles: - - role: maxhoesel.borgbackup.borgmatic - become: yes + become: yes # this role requires root! + tasks: + - name: Setup backups with borgmatic + ansible.builtin.include_role: maxhoesel.borgbackup.borgmatic vars: - # Always Required*Most parameters - borgmatic_location_source_directories: - - /home - - /etc - - /var - # Always Required - borgmatic_location_repositories: - - borg@remote-backup-server.my.domain:{fqdn} - # Required if borgmatic_init_repos is set to true (default) - borgmatic_storage_encryption_passphrase: "mysupersecretpassphrase" - # At least one retention_keep_ parameter is required - borgmatic_retention_keep_daily: 5 + borgmatic_config: + location: + # What do we want to backup? + source_directories: + - /home + - /etc + - /var + + # Where do we want to backup to? + # For borgmatic <1.7, `borgmatic_repositories` is just a list of strings + repositories: + - path: ssh://borg@remote-backup-server.local/./my_repository_name + label: backupserver + + storage: + # What's our encryption password? + # This is required if `borgmatic_init_repos` is set to true (default) + encryption_passphrase: "mysupersecretpassphrase" + + retention: + # How many backups to we want to keep? + keep_daily: 7 ``` diff --git a/roles/borgmatic/defaults/main.yml b/roles/borgmatic/defaults/main.yml index c54bbc6..5eaf7c9 100644 --- a/roles/borgmatic/defaults/main.yml +++ b/roles/borgmatic/defaults/main.yml @@ -11,12 +11,12 @@ borgmatic_init_encryption: repokey borgmatic_ssh_manage_key: yes borgmatic_ssh_key_path: "{{ borgmatic_config_path }}/id_rsa" +borgmatic_ssh_key_gen_options: "-t ed25519" borgmatic_ssh_manage_known_hosts: yes borgmatic_ssh_known_hosts_file: "{{ borgmatic_config_path }}/known_hosts" borgmatic_manage_config: yes -#borgmatic_location_source_directories: -#borgmatic_location_repositories Required +borgmatic_config: {} borgmatic_manage_schedule: yes borgmatic_schedule_on: daily @@ -26,10 +26,5 @@ borgmatic_schedule_require_ac_power: no borgmatic_schedule_persistent: no borgmatic_schedule_wakeup: no -borgmatic_schedule_check_job: no -borgmatic_schedule_check_job_time: "Mon 12:00" -borgmatic_schedule_check_job_checks: - - repository - - archives - borgmatic_run_backup: no +borgmatic_run_backup_no_block: true diff --git a/roles/borgmatic/molecule/default/converge.yml b/roles/borgmatic/molecule/default/converge.yml index 7dd134f..30e487a 100644 --- a/roles/borgmatic/molecule/default/converge.yml +++ b/roles/borgmatic/molecule/default/converge.yml @@ -7,12 +7,17 @@ name: "borgmatic" vars: borgmatic_run_backup: yes - borgmatic_location_source_directories: - - /etc - borgmatic_location_repositories: - - /tmp/myrepo - borgmatic_storage_encryption_passphrase: molecule-test - borgmatic_retention_keep_daily: 3 + borgmatic_run_backup_no_block: false + borgmatic_config: + location: + source_directories: + - /etc + repositories: + - /tmp/myrepo + storage: + encryption_passphrase: molecule-test + retention: + keep_daily: 3 borgmatic_schedule_check_job: yes # Don't actually trigger the timer, we run the check job manually borgmatic_schedule_check_job_time: "2099-12-31" @@ -22,10 +27,3 @@ path: "/tmp/myrepo/lock.roster" state: absent timeout: 120 - - - name: Manually run check job - systemd: - name: borgmatic-check - state: started - tags: - - molecule-idempotence-notest diff --git a/roles/borgmatic/molecule/default/molecule.yml b/roles/borgmatic/molecule/default/molecule.yml index 4458c3e..b1ff869 100644 --- a/roles/borgmatic/molecule/default/molecule.yml +++ b/roles/borgmatic/molecule/default/molecule.yml @@ -22,6 +22,17 @@ platforms: cgroupns_mode: host privileged: true + - name: borgmatic-debian-12 + groups: + - debian + image: "docker.io/geerlingguy/docker-debian12-ansible" + override_command: false + pre_build_image: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + cgroupns_mode: host + privileged: true + - name: borgmatic-debian-11 groups: - debian diff --git a/roles/borgmatic/molecule/remote-repo-unmanaged-ssh/converge.yml b/roles/borgmatic/molecule/remote-repo-unmanaged-ssh/converge.yml index 614b301..2a52135 100644 --- a/roles/borgmatic/molecule/remote-repo-unmanaged-ssh/converge.yml +++ b/roles/borgmatic/molecule/remote-repo-unmanaged-ssh/converge.yml @@ -7,12 +7,17 @@ name: "borgmatic" vars: borgmatic_run_backup: yes - borgmatic_location_source_directories: - - /etc - borgmatic_location_repositories: - - borg@borgmatic-server:molecule - borgmatic_storage_encryption_passphrase: molecule-test - borgmatic_retention_keep_daily: 3 + borgmatic_run_backup_no_block: false + borgmatic_config: + location: + source_directories: + - /etc + repositories: + - ssh://borg@borgmatic-server/./molecule + storage: + encryption_passphrase: molecule-test + retention: + keep_daily: 3 # setup is done manually in "prepare" borgmatic_ssh_manage_key: false borgmatic_ssh_key_path: /root/.ssh/id_rsa diff --git a/roles/borgmatic/molecule/remote-repo-unmanaged-ssh/molecule.yml b/roles/borgmatic/molecule/remote-repo-unmanaged-ssh/molecule.yml index 964650b..2682f5d 100644 --- a/roles/borgmatic/molecule/remote-repo-unmanaged-ssh/molecule.yml +++ b/roles/borgmatic/molecule/remote-repo-unmanaged-ssh/molecule.yml @@ -28,6 +28,20 @@ platforms: networks: - name: molecule-borgmatic + - name: borgmatic-debian-12 + groups: + - debian + - clients + image: "docker.io/geerlingguy/docker-debian12-ansible" + override_command: false + pre_build_image: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + cgroupns_mode: host + privileged: true + networks: + - name: molecule-borgmatic + - name: borgmatic-debian-11 groups: - debian diff --git a/roles/borgmatic/molecule/remote-repo/converge.yml b/roles/borgmatic/molecule/remote-repo/converge.yml index 27385e4..a0842c3 100644 --- a/roles/borgmatic/molecule/remote-repo/converge.yml +++ b/roles/borgmatic/molecule/remote-repo/converge.yml @@ -7,10 +7,16 @@ name: "borgmatic" vars: borgmatic_run_backup: true - borgmatic_location_source_directories: - - /etc - borgmatic_location_repositories: - - borg@borgmatic-server:molecule - borgmatic_storage_encryption_passphrase: molecule-test - borgmatic_retention_keep_daily: 3 + borgmatic_run_backup_no_block: false + borgmatic_config: + location: + source_directories: + - /etc + repositories: + - borg@borgmatic-server:molecule + storage: + encryption_passphrase: molecule-test + retention: + keep_daily: 3 + # use pre-generated ssh key so that we can accept it on the server borgmatic_ssh_key_path: /tmp/borgmatic_id_rsa diff --git a/roles/borgmatic/molecule/remote-repo/molecule.yml b/roles/borgmatic/molecule/remote-repo/molecule.yml index 964650b..2682f5d 100644 --- a/roles/borgmatic/molecule/remote-repo/molecule.yml +++ b/roles/borgmatic/molecule/remote-repo/molecule.yml @@ -28,6 +28,20 @@ platforms: networks: - name: molecule-borgmatic + - name: borgmatic-debian-12 + groups: + - debian + - clients + image: "docker.io/geerlingguy/docker-debian12-ansible" + override_command: false + pre_build_image: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + cgroupns_mode: host + privileged: true + networks: + - name: molecule-borgmatic + - name: borgmatic-debian-11 groups: - debian diff --git a/roles/borgmatic/molecule/verify.yml b/roles/borgmatic/molecule/verify.yml index 0edf49c..d9fad8d 100644 --- a/roles/borgmatic/molecule/verify.yml +++ b/roles/borgmatic/molecule/verify.yml @@ -13,13 +13,6 @@ assert: that: not borgmatic_timer_enabled.changed - - name: Wait for backup to finish - wait_for: - path: "/var/borg-server/{{ ansible_fqdn }}/molecule/lock.roster" - state: absent - timeout: 120 - delegate_to: borgmatic-server - - name: Get backup information command: borgmatic list --json changed_when: no diff --git a/roles/borgmatic/tasks/check.yml b/roles/borgmatic/tasks/check.yml new file mode 100644 index 0000000..f9101ea --- /dev/null +++ b/roles/borgmatic/tasks/check.yml @@ -0,0 +1,52 @@ +#code: language=ansible +- name: Verify that source and repository parameters are set + ansible.builtin.assert: + that: + - config.source_directories is defined + - config.source_directories | length > 0 + - config.repositories is defined + - config.repositories | length > 0 + when: borgmatic_manage_config + vars: + config: "{{ (_borgmatic_config_with_sections | bool) | ternary(borgmatic_config.location, borgmatic_config) }}" # compat with borgmatic < 1.8 + +- name: Verify that at least one keep_ option is set + ansible.builtin.assert: + that: > + config.keep_within|d("") | length > 0 or + config.keep_secondly|d(0) > 0 or + config.keep_minutely|d(0) > 0 or + config.keep_hourly|d(0) > 0 or + config.keep_daily|d(0) > 0 or + config.keep_weekly|d(0) > 0 or + config.keep_monthly|d(0) > 0 or + config.keep_yearly|d(0) > 0 + when: borgmatic_manage_config + vars: + config: "{{ (_borgmatic_config_with_sections | bool) | ternary(borgmatic_config.retention, borgmatic_config) }}" # compat with borgmatic < 1.8 + +- name: borgmatic_ssh_manage_key | Check that ssh_command does not contain custom private key + ansible.builtin.assert: + that: + - '" -i " not in config.ssh_command|d("")' + fail_msg: > + borgmatic_ssh_manage_key is enabled, but provided ssh_command points to a different private key. + Please remove the private key ("-i") from ssh_command or disable borgmatic_ssh_manage_key if you want to manage the key yourself. + when: + - borgmatic_manage_config + - borgmatic_ssh_manage_key + vars: + config: "{{ (_borgmatic_config_with_sections | bool) | ternary(borgmatic_config.storage, borgmatic_config) }}" # compat with borgmatic < 1.8 + +- name: borgmatic_ssh_manage_known_hosts | Check that ssh_command does not contain custom known_hosts_file + ansible.builtin.assert: + that: + - '" -o UserKnownHostsFile=" not in config.ssh_command|d("")' + fail_msg: > + borgmatic_ssh_manage_key is enabled, but provided ssh_command points to a different known_hosts file. + Please remove "-o UserKnownHostsFile=" from ssh_command or disable borgmatic_ssh_manage_key if you want to manage the key yourself. + when: + - borgmatic_manage_config + - borgmatic_ssh_manage_known_hosts + vars: + config: "{{ (_borgmatic_config_with_sections | bool) | ternary(borgmatic_config.storage, borgmatic_config) }}" # compat with borgmatic < 1.8 diff --git a/roles/borgmatic/tasks/config.yml b/roles/borgmatic/tasks/config.yml deleted file mode 100644 index 3081d9a..0000000 --- a/roles/borgmatic/tasks/config.yml +++ /dev/null @@ -1,20 +0,0 @@ -- name: Generate config variables # noqa var-spacing - ansible.builtin.set_fact: - # -> dict of config var name and actual value - _borgmatic_configvars: "{{ _borgmatic_configvars | d({}) | combine({ item: lookup('vars', item )} ) }}" - # -> List of all variable names that we need to lookup for our config - loop: "{{ query('varnames', '^borgmatic_(' + borgmatic_config_sections | join('|') + ')_') }}" - -- name: Config is installed - template: - src: config.yaml.j2 - dest: "{{ borgmatic_config_path }}/config.yaml" - owner: root - group: root - mode: "640" - -- name: Initialize repos - command: "borgmatic init --encryption {{ borgmatic_init_encryption }}" - register: _borgmatic_init_command - changed_when: _borgmatic_init_command.stdout | length > 0 - when: borgmatic_init_repos diff --git a/roles/borgmatic/tasks/config_schedule.yml b/roles/borgmatic/tasks/config_schedule.yml new file mode 100644 index 0000000..f927591 --- /dev/null +++ b/roles/borgmatic/tasks/config_schedule.yml @@ -0,0 +1,18 @@ +#code: language=ansible +- name: Systemd service and timer are installed + template: + src: "{{ item }}.j2" + dest: "/etc/systemd/system/{{ item }}" + owner: root + group: root + mode: "644" + loop: + - borgmatic.service + - borgmatic.timer + +- name: Systemd timer is enabled and running + systemd: + daemon_reload: yes + name: borgmatic.timer + enabled: yes + state: started diff --git a/roles/borgmatic/tasks/config_ssh.yml b/roles/borgmatic/tasks/config_ssh.yml new file mode 100644 index 0000000..67e9d1c --- /dev/null +++ b/roles/borgmatic/tasks/config_ssh.yml @@ -0,0 +1,38 @@ +#code: language=ansible +- name: Setup borgmatic ssh key + block: + - name: Generate SSH keypair + command: "ssh-keygen {{ borgmatic_ssh_key_gen_options }} -f {{ borgmatic_ssh_key_path }} -q -N ''" + args: + creates: "{{ borgmatic_ssh_key_path }}" + when: borgmatic_ssh_manage_key + +# needs to be a separate task due to https://github.com/maxhoesel-ansible/ansible-collection-borgbackup/issues/135 +# the solution proposed in the issue breaks idempotency on some systems +- name: Setup known_hosts + ansible.builtin.include_tasks: config_ssh_known_hosts.yml + when: borgmatic_ssh_manage_known_hosts + +# Append the parameters required for custom known_hosts/key to the ssh command, while making sure not to insert any duplicates. +- name: Set ssh_command + block: + - name: Generate ssh_command string + set_fact: + _borgmatic_ssh_command: >- + {{ config.ssh_command | d('ssh') }} + {% if borgmatic_ssh_manage_key and _borgmatic_ssh_key_string not in config.ssh_command | d('ssh') %}{{ _borgmatic_ssh_key_string }}{% endif %} + {% if borgmatic_ssh_manage_known_hosts and _borgmatic_ssh_known_hosts_string not in config.ssh_command | d('ssh') %}{{ _borgmatic_ssh_known_hosts_string }}{% endif %} + vars: + _borgmatic_ssh_key_string: " -i {{ borgmatic_ssh_key_path }}" + _borgmatic_ssh_known_hosts_string: " -o UserKnownHostsFile={{ borgmatic_ssh_known_hosts_file }}" + config: "{{ (_borgmatic_config_with_sections | bool) | ternary(borgmatic_config.storage, borgmatic_config) }}" # compat with borgmatic < 1.8 + + - name: Insert ssh_command into borgmatic <1.8 config + ansible.builtin.set_fact: + _borgmatic_effective_config: "{{ borgmatic_config | combine({'storage': {'ssh_command': _borgmatic_ssh_command}}, recursive=True) }}" + when: _borgmatic_config_with_sections + - name: Insert ssh_command into borgmatic >=1.8 config + ansible.builtin.set_fact: + _borgmatic_effective_config: "{{ borgmatic_config | combine({'ssh_command': _borgmatic_ssh_command}, recursive=True) }}" + when: not _borgmatic_config_with_sections + when: borgmatic_ssh_manage_key or borgmatic_ssh_manage_known_hosts diff --git a/roles/borgmatic/tasks/config_ssh_known_hosts.yml b/roles/borgmatic/tasks/config_ssh_known_hosts.yml new file mode 100644 index 0000000..176627a --- /dev/null +++ b/roles/borgmatic/tasks/config_ssh_known_hosts.yml @@ -0,0 +1,38 @@ +#code: language=ansible + +- name: Get repository paths from config + ansible.builtin.set_fact: + # compat with borgmatic <1.7 + _borgmatic_repositories: "{{ (borgmatic_config.repositories.0.path is defined | bool) | ternary(config.repositories | map(attribute='path') | list, config.repositories) }}" + vars: + config: "{{ (_borgmatic_config_with_sections | bool) | ternary(borgmatic_config.location, borgmatic_config) }}" # compat with borgmatic < 1.8 + +- name: Generate list of remote hosts + ansible.builtin.set_fact: + _borgmatic_remote_hosts: "{{ _borgmatic_repositories | map('regex_search', '(?:ssh:\/\/)?(?:[^@]+@)([^\/:]+)', '\\1') | select | list | flatten | d([]) | unique }}" + +# ssh-keyscan returns a multiline string, with each line containing one known_hosts entry +# We call the known_hosts task once for each remote host, with the task itself +# then splitting the lines into individual entries that ansible.builtin.known_hosts +# can understand. I'm sure there is an easier way to do this, but this works for now +# and doesn't throw up and idempotency issues +- name: Get remote server ssh fingerprints + command: "ssh-keyscan {{ item }}" + changed_when: no + check_mode: no + retries: 5 + delay: 5 + # ensure we actually gather *all* ssh keys from the remote host + until: > + borgmatic_ssh_fingerprints.rc == 0 and + "Connection closed" not in borgmatic_ssh_fingerprints.stderr and + "Connection reset" not in borgmatic_ssh_fingerprints.stderr + register: borgmatic_ssh_fingerprints + loop: "{{ _borgmatic_remote_hosts }}" + +- name: known_hosts are registered + known_hosts: + name: "{{ item.0.item }}" + key: "{{ item.1 }}" + path: "{{ borgmatic_ssh_known_hosts_file }}" + loop: "{{ borgmatic_ssh_fingerprints.results | subelements('stdout_lines') }}" diff --git a/roles/borgmatic/tasks/install.yml b/roles/borgmatic/tasks/install.yml deleted file mode 100644 index 8ff950d..0000000 --- a/roles/borgmatic/tasks/install.yml +++ /dev/null @@ -1,14 +0,0 @@ -- name: Borg and borgmatic are installed - apt: - name: - - borgmatic - - borgbackup - update_cache: yes - -- name: Borgmatic config path is present - file: - path: "{{ borgmatic_config_path }}" - state: directory - mode: "750" - owner: root - group: root diff --git a/roles/borgmatic/tasks/main.yml b/roles/borgmatic/tasks/main.yml index e46a2e6..59c737f 100644 --- a/roles/borgmatic/tasks/main.yml +++ b/roles/borgmatic/tasks/main.yml @@ -1,29 +1,61 @@ --- # tasks file for borgmatic +- name: Determine if old-style sectioned Borgmatic config (<1.8) is in use + ansible.builtin.set_fact: + # location contains source_directories and repositories and must thus always exist + _borgmatic_config_with_sections: "{{ borgmatic_config.location is defined }}" -- name: Verify that required parameters are set - assert: - that: - - borgmatic_location_source_directories is defined - - borgmatic_location_source_directories | length > 0 - - borgmatic_location_repositories is defined - - borgmatic_location_repositories | length > 0 - when: borgmatic_manage_config -- name: Verify that at least one keep option is set - assert: - that: query('varnames', '^borgmatic_retention_keep_') | length > 0 - when: borgmatic_manage_config - -- ansible.builtin.include_tasks: install.yml +- name: Perform variable checks + ansible.builtin.include_tasks: check.yml + +- name: Copy config to effective config map for further modification + set_fact: + _borgmatic_effective_config: "{{ borgmatic_config }}" + +- name: Borg and borgmatic are installed + ansible.builtin.package: + name: + - borgmatic + - borgbackup when: borgmatic_install -- block: # noqa unnamed-task - - name: Configure SSH - ansible.builtin.include_tasks: ssh.yml - - name: Generate Borgmatic Config - ansible.builtin.include_tasks: config.yml - when: borgmatic_manage_config - - name: Create Schedule - ansible.builtin.include_tasks: schedule.yml - when: borgmatic_manage_schedule +- name: Configure Borgmatic + block: + - name: Borgmatic config path is present + file: + path: "{{ borgmatic_config_path }}" + state: directory + mode: "750" + owner: root + group: root + + - name: Configure SSH + ansible.builtin.include_tasks: config_ssh.yml + + - name: Config is installed + template: + src: config.yaml.j2 + dest: "{{ borgmatic_config_path }}/config.yaml" + owner: root + group: root + mode: "640" + + - name: Initialize repos + command: "borgmatic init --encryption {{ borgmatic_init_encryption }}" + register: _borgmatic_init_command + changed_when: _borgmatic_init_command.stdout | length > 0 + when: borgmatic_init_repos + + - name: Create Schedule + ansible.builtin.include_tasks: config_schedule.yml + when: borgmatic_manage_schedule when: borgmatic_setup_backup + +- name: Run backup job + systemd: + name: borgmatic.service + state: started + no_block: "{{ borgmatic_run_backup_no_block }}" + when: borgmatic_run_backup + tags: + - molecule-idempotence-notest diff --git a/roles/borgmatic/tasks/schedule.yml b/roles/borgmatic/tasks/schedule.yml deleted file mode 100644 index 86c1353..0000000 --- a/roles/borgmatic/tasks/schedule.yml +++ /dev/null @@ -1,46 +0,0 @@ -- name: Systemd service and timer are installed - template: - src: "{{ item }}.j2" - dest: "/etc/systemd/system/{{ item }}" - owner: root - group: root - mode: "644" - loop: - - borgmatic.service - - borgmatic.timer - -- name: Check timer and service is installed - template: - src: "{{ item }}.j2" - dest: "/etc/systemd/system/{{ item }}" - owner: root - group: root - mode: "644" - loop: - - borgmatic-check.service - - borgmatic-check.timer - when: borgmatic_schedule_check_job - -- name: Systemd timer is enabled and running - systemd: - daemon_reload: yes - name: borgmatic.timer - enabled: yes - state: started - -- name: Systemd check timer is enabled and running - systemd: - daemon_reload: yes - name: borgmatic-check.timer - enabled: yes - state: started - when: borgmatic_schedule_check_job - -- name: Run backup job - systemd: - name: borgmatic.service - state: started - no_block: yes - when: borgmatic_run_backup - tags: - - molecule-idempotence-notest diff --git a/roles/borgmatic/tasks/ssh.yml b/roles/borgmatic/tasks/ssh.yml deleted file mode 100644 index 6c1b1c4..0000000 --- a/roles/borgmatic/tasks/ssh.yml +++ /dev/null @@ -1,37 +0,0 @@ -- block: # noqa unnamed-task - - name: Verify that ssh_key_path is present in custom ssh_command - assert: - that: "'-i ' + borgmatic_ssh_key_path in borgmatic_storage_ssh_command" - fail_msg: > - custom_ssh_command does not include ssh_key_path and ssh_manage_key is enabled. - To use the ssh key management capabilities of this role, please add this to your custom - ssh command: "-i {{ borgmatic_ssh_key_path }}" - when: borgmatic_storage_ssh_command is defined - - - name: Look for existing ssh key - stat: - path: "{{ borgmatic_ssh_key_path }}" - register: borgmatic_ssh_keys_current - - name: Generate SSH keypair - command: "ssh-keygen -b 2048 -t rsa -f {{ borgmatic_ssh_key_path }} -q -N ''" - when: not borgmatic_ssh_keys_current.stat.exists - when: borgmatic_ssh_manage_key - -# needs to be a separate task due to https://github.com/maxhoesel-ansible/ansible-collection-borgbackup/issues/135 -# the solution proposed in the issue breaks idempotency on some systems -- name: Setup known_hosts - ansible.builtin.include_tasks: ssh_known_hosts.yml - when: borgmatic_ssh_manage_known_hosts - -# Append the parameters required for custom known_hosts/key to the ssh command, while making sure not to insert any duplicates. -# Required to preserve compatibility with older versions of this role that asked the user to manually adjust borgmatic_storage_ssh_command -- name: Generate ssh command - set_fact: - borgmatic_storage_ssh_command: >- - {{ borgmatic_storage_ssh_command | d('ssh') }} - {% if borgmatic_ssh_manage_key and _borgmatic_ssh_key_string not in borgmatic_storage_ssh_command | d('') %}{{ _borgmatic_ssh_key_string }}{% endif %} - {% if borgmatic_ssh_manage_known_hosts and _borgmatic_ssh_known_hosts_string not in borgmatic_storage_ssh_command | d('') %}{{ _borgmatic_ssh_known_hosts_string }}{% endif %} - when: borgmatic_ssh_manage_key or borgmatic_ssh_manage_known_hosts - vars: - _borgmatic_ssh_key_string: " -i {{ borgmatic_ssh_key_path }}" - _borgmatic_ssh_known_hosts_string: " -o UserKnownHostsFile={{ borgmatic_ssh_known_hosts_file }}" diff --git a/roles/borgmatic/tasks/ssh_known_hosts.yml b/roles/borgmatic/tasks/ssh_known_hosts.yml deleted file mode 100644 index fea2e7b..0000000 --- a/roles/borgmatic/tasks/ssh_known_hosts.yml +++ /dev/null @@ -1,30 +0,0 @@ -- name: Verify that ssh_known_hosts_file is present in custom ssh_command - assert: - that: "'-o UserKnownHostsFile=' + borgmatic_ssh_known_hosts_file in borgmatic_storage_ssh_command" - fail_msg: > - custom_ssh_command does not include ssh_known_hosts_file and ssh_manage_key is enabled. - To use the ssh key management capabilities of this role, please add this to your custom - ssh command: "-o UserKnownHostsFile={{ borgmatic_ssh_known_hosts_file }}" - when: borgmatic_storage_ssh_command is defined - -# ssh-keyscan returns a multiline string, with each line containing one known_hosts entry -# We call the known_hosts task once for each remote host, with the task itself -# then splitting the lines into individual entries that ansible.builtin.known_hosts -# can understand. I'm sure there is an easier way to do this, but this works for now -# and doesn't throw up and idempotency issues -- name: Get remote server ssh fingerprints - command: "ssh-keyscan -H {{ (item.split('@').1).split(':').0 }}" # get the actual fqdn from the repo definition - changed_when: no - check_mode: no - retries: 5 - delay: 5 - until: borgmatic_ssh_fingerprints.rc == 0 - register: borgmatic_ssh_fingerprints - loop: "{{ borgmatic_location_repositories | select('match', '.*@.*:.*') | list }}" # only match remote ssh hosts -- name: known_hosts are registered - known_hosts: - name: "{{ (item.0.item.split('@').1).split(':').0 }}" - key: "{{ item.1 }}" - path: "{{ borgmatic_ssh_known_hosts_file }}" - loop: "{{ borgmatic_ssh_fingerprints.results | subelements('stdout_lines') }}" - no_log: yes # reduce log spam a little bit diff --git a/roles/borgmatic/templates/borgmatic-check.service.j2 b/roles/borgmatic/templates/borgmatic-check.service.j2 deleted file mode 100644 index d2aa114..0000000 --- a/roles/borgmatic/templates/borgmatic-check.service.j2 +++ /dev/null @@ -1,51 +0,0 @@ -[Unit] -Description=borgmatic backup check job -Wants=network-online.target -After=network-online.target -{% if borgmatic_schedule_require_ac_power %} -ConditionACPower=true -{% endif %} - -[Service] -Type=oneshot -{% if borgmatic_schedule_harden %} - -# Security settings for systemd running as root, optional but recommended to improve security. You -# can disable individual settings if they cause problems for your use case. For more details, see -# the systemd manual: https://www.freedesktop.org/software/systemd/man/systemd.exec.html -LockPersonality=true -# Certain borgmatic features like Healthchecks integration need MemoryDenyWriteExecute to be off. -# But you can try setting it to "yes" for improved security if you don't use those features. -NoNewPrivileges=yes -PrivateDevices=yes -ProtectClock=yes -ProtectControlGroups=yes -ProtectHostname=yes -ProtectKernelLogs=yes -ProtectKernelModules=yes -ProtectKernelTunables=yes -RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK -RestrictNamespaces=yes -RestrictRealtime=yes -RestrictSUIDSGID=yes -SystemCallArchitectures=native -SystemCallFilter=@system-service -SystemCallErrorNumber=EPERM -CapabilityBoundingSet=CAP_DAC_READ_SEARCH CAP_NET_RAW -{% endif %} - -# Lower CPU and I/O priority. -Nice=19 -CPUSchedulingPolicy=batch -IOSchedulingClass=best-effort -IOSchedulingPriority=7 -IOWeight=100 - -Restart=no -# Prevent rate limiting of borgmatic log events. If you are using an older version of systemd that -# doesn't support this (pre-240 or so), you may have to remove this option. -LogRateLimitIntervalSec=0 - -# Delay start to prevent backups running during boot. Note that systemd-inhibit requires dbus and -# dbus-user-session to be installed. -ExecStart=/usr/bin/borgmatic check --syslog-verbosity 1{% for check in borgmatic_schedule_check_job_checks %} --only {{ check }}{% endfor %} diff --git a/roles/borgmatic/templates/borgmatic-check.timer.j2 b/roles/borgmatic/templates/borgmatic-check.timer.j2 deleted file mode 100644 index 893c6d5..0000000 --- a/roles/borgmatic/templates/borgmatic-check.timer.j2 +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=run borgmatic consistency checks - -[Timer] -OnCalendar={{ borgmatic_schedule_check_job_time }} -Persistent={{ borgmatic_schedule_persistent }} -WakeSystem={{ borgmatic_schedule_wakeup }} -RandomizedDelaySec={{ borgmatic_schedule_max_random_delay }} - -[Install] -WantedBy=timers.target diff --git a/roles/borgmatic/templates/config.yaml.j2 b/roles/borgmatic/templates/config.yaml.j2 index 65d3392..75869d8 100644 --- a/roles/borgmatic/templates/config.yaml.j2 +++ b/roles/borgmatic/templates/config.yaml.j2 @@ -3,18 +3,4 @@ # Borgmatic main configuration file # {{ ansible_managed }} -{%- set config = {} %} -{%- for section in borgmatic_config_sections %} - {%- set section_cfg = {} %} - {%- for entry in _borgmatic_configvars %} - {%- if entry is match('^borgmatic_' + section + '_') %} - {%- set entry_name = entry.replace('borgmatic_' + section + '_', '') %} - {#- Really nasty way to update a dict in jinja, but i don't know of a cleaner way to accomplish this #} - {%- set _ = section_cfg.update({entry_name: _borgmatic_configvars[entry]}) %} - {%- endif %} - {%- endfor %} - {%- if section_cfg | length > 0 %} - {%- set _ = config.update({section: section_cfg}) %} - {%- endif %} -{%- endfor %} -{{ config | to_nice_yaml(indent=2, width=120) }} +{{ _borgmatic_effective_config | to_nice_yaml(indent=2, width=120) }} diff --git a/roles/borgmatic/vars/main.yml b/roles/borgmatic/vars/main.yml index f4c219d..e69de29 100644 --- a/roles/borgmatic/vars/main.yml +++ b/roles/borgmatic/vars/main.yml @@ -1,6 +0,0 @@ -borgmatic_config_sections: - - location - - storage - - retention - - consistency - - hooks