added azure as backend for storage (#31)

Add azure as backend for storage

src/config.yaml

@@ -9,14 +9,23 @@ options:
     type: string
     default: ""
     description: |
-      Accepted values are s3 | scp | sftp | ftp | rsync | file
+      Accepted values are s3 | scp | sftp | ftp | rsync | file | azure
       An empty string will disable backups.
   remote_backup_url:
     type: string
     default: ""
     description: |
       URL to the remote server and its local path to be used as the
-      backup destination.
+      backup destination. 
+      While using azure as a backend:
+      Duplicity will take care to create the container 
+      when performing the backup.  Do not create it manually before.
+      If the container is already created on Azure, 
+      it will start putting data inside it without any issues. However, 
+      if you explicitly set the configuration with a container name 
+      (e.g., remote_backup_url = azure:///some-dir), it will throw an error,
+      because Duplicity appends the unit name to the container name, 
+      Therefore, avoid explicitly setting the container name in the configuration.
 
       Backends and their URL formats:
         file:      'file:///some_dir'
@@ -26,6 +35,7 @@ options:
                     'other.host[:port]//absolute_path'
         s3:         's3:other.host[:port]/bucket_name[/prefix]'
                     's3+http://bucket_name[/prefix]'
+        azure:      'azure://'
         scp:        'other.host[:port]/some_dir'
         sftp:       'other.host[:port]/some_dir'
   aws_access_key_id:
@@ -42,6 +52,13 @@ options:
       Secret access key for the AWS IMA user. The user must have a policy that
       grants it privileges to upload to the S3 bucket. This value is required
       when backend='s3'.
+  azure_connection_string:
+    type: string
+    default: ""
+    description: |
+      connection string for the AZURE storage account. This value is required
+      when backend='azure'.
+      check - https://learn.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string#configure-a-connection-string-for-an-azure-storage-account
   remote_user:
     type: string
     default: ""
@@ -109,4 +126,4 @@ options:
     default: ''
     description:
       base64 encoded private SSH key for SSH authentication from duplicity
-      application unit and the remote backup host.
+      application unit and the remote backup host.

src/lib/lib_duplicity.py

@@ -88,7 +88,7 @@ def _backup_url(self):
                 else:
                     url += "{}@".format(user)
             url += remote_path.replace(prefix, "")
-        elif backend in ["s3", "file"]:
+        elif backend in ["s3", "file", "azure"]:
             url = remote_path.replace(prefix, "")
         else:
             return None
@@ -108,6 +108,12 @@ def _set_environment_vars(self):
         duplicity.
         :return:
         """
+        # Set the Azure Credentials. It doesnt matter if they are used or not
+        os.environ["AZURE_CONNECTION_STRING"] = self.charm_config.get(
+            "azure_connection_string"
+        )
+        os.environ["PASSPHRASE"] = self.charm_config.get("encryption_passphrase")
+
         # Set the Aws Credentials. It doesnt matter if they are used or not
         os.environ["AWS_SECRET_ACCESS_KEY"] = self.charm_config.get(
             "aws_secret_access_key"

src/reactive/duplicity.py

@@ -12,6 +12,7 @@
 import base64
 import binascii
 import os
+import subprocess
 from re import fullmatch
 
 from charmhelpers import fetch
@@ -40,6 +41,37 @@
 config = hookenv.config()
 
 
+class PipPackageInstallError(RuntimeError):
+    """Error during package installation with pip."""
+
+    pass
+
+
+def install_in_system_python(package_to_install):
+    """Install dependency in system python.
+
+    The charm use subprocess to call duplicity,
+    which is installed by apt.
+    So the azure-storage-blob has to be installed n system level python
+    See https://duplicity.nongnu.org/vers7/duplicity.1.html#sect10
+    There is no maintained package in apt repositories,
+    so we have to install it with pip.
+    """
+    command = ["sudo", "pip", "install", package_to_install]
+    output = subprocess.run(
+        command,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+    if output.returncode != 0:
+        raise PipPackageInstallError(
+            "Failed to install a package using '{}' resulting '{}'".format(
+                " ".join(command),
+                output.stderr.decode(),
+            )
+        )
+
+
 @when_not("duplicity.installed")
 def install_duplicity():
     """Apt install duplicity's dependencies.
@@ -56,14 +88,25 @@ def install_duplicity():
     fetch.apt_install("python-paramiko")
     fetch.apt_install("python-boto")
     fetch.apt_install("lftp")
+    install_in_system_python("azure-storage-blob")
     hookenv.status_set("active", "")
     set_flag("duplicity.installed")
 
 
+@hook("upgrade-charm")
+def upgrade_duplicity():
+    """Install packages on charm upgrade.
+
+    This ensures that any new packages added in recent releases will be installed.
+    """
+    install_duplicity()
+
+
 @when_any(
     "config.changed.backend",
     "config.changed.aws_access_key_id",
     "config.changed.aws_secret_access_key" "config.changed.known_host_key",
+    "config.changed.azure_connection_string",
     "config.changed.remote_password",
     "config.changed.private_ssh_key",
 )
@@ -72,10 +115,11 @@ def validate_backend():
 
     Validates that the config value for 'backend' is something that duplicity
     can use (see config description for backend for the accepted types). For S3
-    only, check that the AWS IMA credentials are also set.
+    only, check that the AWS IMA credentials are set. For AZURE check connections
+    string is also set.
     """
     backend = config.get("backend").lower()
-    if backend in ["s3", "scp", "sftp", "ftp", "rsync", "file"]:
+    if backend in ["s3", "scp", "sftp", "ftp", "rsync", "file", "azure"]:
         clear_flag("duplicity.invalid_backend")
     else:
         set_flag("duplicity.invalid_backend")
@@ -86,6 +130,12 @@ def validate_backend():
         else:
             set_flag("duplicity.invalid_aws_creds")
             return
+    elif backend == "azure":
+        if config.get("azure_connection_string"):
+            clear_flag("duplicity.invalid_azure_creds")
+        else:
+            set_flag("duplicity.invalid_azure_creds")
+            return
     elif backend == "rsync":
         if config.get("private_ssh_key"):
             clear_flag("duplicity.invalid_rsync_key")
@@ -258,6 +308,12 @@ def assess_status():  # pylint: disable=C901
             'and "aws_secret_access_key" to be set',
         )
         return
+    if is_flag_set("duplicity.invalid_azure_creds"):
+        hookenv.status_set(
+            workload_state="blocked",
+            message='Azure backups require "azure_connection_string" ',
+        )
+        return
     if is_flag_set("duplicity.invalid_secure_backend_opts"):
         hookenv.status_set(
             workload_state="blocked",

src/templates/periodic_backup

@@ -1 +1 @@
-{{ frequency }} root /usr/bin/juju-run {{ unit_name }} {{ charm_dir }}/scripts/periodic_backup.py
+{{ frequency }} root /usr/bin/juju-run {{ unit_name }} {{ charm_dir }}/scripts/periodic_backup.py

src/templates/periodic_deletion

@@ -1 +1 @@
-{{ frequency }} root /usr/bin/juju-run {{ unit_name }} {{ charm_dir }}/scripts/periodic_deletion.py
+{{ frequency }} root /usr/bin/juju-run {{ unit_name }} {{ charm_dir }}/scripts/periodic_deletion.py

src/tests/unit/test_reactive_duplicity.py

@@ -1,5 +1,6 @@
 """Unit tests for reactive hooks."""
 
+from unittest import TestCase
 from unittest.mock import ANY, MagicMock, call, mock_open, patch
 
 from croniter import CroniterBadCronError, CroniterBadDateError, CroniterNotAlphaError
@@ -11,19 +12,57 @@
         import duplicity
 
 
-@patch("duplicity.hookenv")
-@patch("duplicity.fetch")
-@patch("duplicity.set_flag")
-def test_install_duplicity(mock_set_flag, mock_fetch, mock_hookenv):
-    """Verify install hook."""
-    hookenv_calls = [call("maintenance", "Installing duplicity"), call("active", "")]
-    fetch_calls = [
-        call(x) for x in ["duplicity", "python-paramiko", "python-boto", "lftp"]
-    ]
-    duplicity.install_duplicity()
-    mock_hookenv.status_set.assert_has_calls(hookenv_calls)
-    mock_fetch.apt_install.assert_has_calls(fetch_calls)
-    mock_set_flag.assert_called_with("duplicity.installed")
+class TestInstallDuplicity(TestCase):
+    """Verify charm installation."""
+
+    @patch("duplicity.hookenv")
+    @patch("duplicity.fetch")
+    @patch("duplicity.set_flag")
+    def test_install_duplicity(self, mock_set_flag, mock_fetch, mock_hookenv):
+        """Verify install hook."""
+        hookenv_calls = [
+            call("maintenance", "Installing duplicity"),
+            call("active", ""),
+        ]
+        fetch_calls = [
+            call(x) for x in ["duplicity", "python-paramiko", "python-boto", "lftp"]
+        ]
+        duplicity.install_duplicity()
+        mock_hookenv.status_set.assert_has_calls(hookenv_calls)
+        mock_fetch.apt_install.assert_has_calls(fetch_calls)
+        mock_set_flag.assert_called_with("duplicity.installed")
+
+    @patch("subprocess.run")
+    def test_install_duplicity_raises_when_pip_fails(self, mock_subprocess_run):
+        """Verify that charm installation fails when pip fails."""
+        mock_subprocess_run.return_value = MagicMock(
+            returncode=1, stderr=b"failed to install package with pip"
+        )
+        with self.assertRaises(duplicity.PipPackageInstallError):
+            duplicity.install_duplicity()
+
+    @patch("duplicity.hookenv")
+    @patch("duplicity.fetch")
+    @patch("duplicity.set_flag")
+    @patch("subprocess.run")
+    def test_install_duplicity_succeeds_when_pip_suceeds(
+        self, mock_subprocess_run, mock_set_flag, mock_fetch, mock_hookenv
+    ):
+        """Verify that charm installation fails when pip fails."""
+        hookenv_calls = [
+            call("maintenance", "Installing duplicity"),
+            call("active", ""),
+        ]
+        fetch_calls = [
+            call(x) for x in ["duplicity", "python-paramiko", "python-boto", "lftp"]
+        ]
+        mock_subprocess_run.return_value = MagicMock(
+            returncode=0, stdout=b"package installed!"
+        )
+        duplicity.install_duplicity()
+        mock_hookenv.status_set.assert_has_calls(hookenv_calls)
+        mock_fetch.apt_install.assert_has_calls(fetch_calls)
+        mock_set_flag.assert_called_with("duplicity.installed")
 
 
 class TestValidateBackend:
@@ -162,6 +201,43 @@ def test_invalid_backend_bad_backend(
         mock_set_flag.assert_called_with("duplicity.invalid_backend")
         mock_clear_flag.assert_not_called()
 
+    @patch("duplicity.set_flag")
+    @patch("duplicity.clear_flag")
+    @patch("duplicity.config")
+    def test_validate_backend_success_azure(
+        self, mock_config, mock_clear_flag, mock_set_flag
+    ):
+        """Verify valid azure backend."""
+        backend = "azure"
+        mock_config.get.side_effect = [backend, "azure_conn_string"]
+        expected_calls = [
+            call("duplicity.invalid_backend"),
+            call("duplicity.invalid_azure_creds"),
+        ]
+        duplicity.validate_backend()
+        mock_clear_flag.assert_has_calls(calls=expected_calls)
+        mock_set_flag.assert_not_called()
+
+    @pytest.mark.parametrize("conn_string", ["some_string", ""])
+    @patch("duplicity.set_flag")
+    @patch("duplicity.clear_flag")
+    @patch("duplicity.config")
+    def test_invalid_backend_azure(
+        self, mock_config, mock_clear_flag, mock_set_flag, conn_string
+    ):
+        """Verify invalid azure backend."""
+        backend = "azure"
+        side_effects = [backend, conn_string]
+        mock_config.get.side_effect = side_effects
+        duplicity.validate_backend()
+        mock_clear_flag.assert_any_call("duplicity.invalid_backend")
+
+        if conn_string:
+            mock_clear_flag.assert_called_with("duplicity.invalid_azure_creds")
+            mock_set_flag.assert_not_called()
+        else:
+            mock_set_flag.assert_called_with("duplicity.invalid_azure_creds")
+
 
 @pytest.mark.parametrize(
     "backup_dir,path_exists", [("my dir", False), ("my dir", True), ("", True)]
@@ -435,43 +511,48 @@ def test_check_status(self, mock_hookenv):
                 'and "aws_secret_access_key" to be set',
                 2,
             ),
+            (
+                "duplicity.invalid_azure_creds",
+                'Azure backups require "azure_connection_string" ',
+                3,
+            ),
             (
                 "duplicity.invalid_secure_backend_opts",
                 "{} backend requires known_host_key "
                 'and either "remote_password" or "private_ssh_key" to be set',
-                3,
+                4,
             ),
             (
                 "duplicity.invalid_rsync_key",
                 "rsync backend requires private_ssh_key. remote_password auth "
                 "not supported",
-                4,
+                5,
             ),
             (
                 "duplicity.invalid_encryption_method",
                 "Must set either an encryption passphrase, "
                 "GPG public key, or disable encryption",
-                5,
+                6,
             ),
             (
                 "duplicity.invalid_private_ssh_key",
                 "Invalid private_ssh_key. ensure that key is base64 encoded",
-                6,
+                7,
             ),
             (
                 "duplicity.invalid_backup_frequency",
                 'Invalid value "{}" for "backup_frequency"',
-                7,
+                8,
             ),
             (
                 "duplicity.invalid_retention_period",
                 'Invalid value "{}" for "retention_period"',
-                8,
+                9,
             ),
             (
                 "duplicity.invalid_deletion_frequency",
                 'Invalid value "{}" for "deletion_frequency"',
-                9,
+                10,
             ),
         ],
     )