wip: fix CI configuration and storage_class access

.ci/ansible/settings.py.j2

@@ -27,34 +27,55 @@ API_ROOT = {{ api_root | repr }}
 {% endif %}
 
 {% if s3_test | default(false) %}
-STORAGES__default = "storages.backends.s3boto3.S3Boto3Storage"
 MEDIA_ROOT = ""
-AWS_ACCESS_KEY_ID = "{{ minio_access_key }}"
-AWS_SECRET_ACCESS_KEY = "{{ minio_secret_key }}"
-AWS_S3_REGION_NAME = "eu-central-1"
-AWS_S3_ADDRESSING_STYLE = "path"
+STORAGES = {
+    "default": {
+        "BACKEND": "storages.backends.s3boto3.S3Boto3Storage",
+        "OPTIONS": {
+            "access_key": "{{ minio_access_key }}",
+            "secret_key": "{{ minio_secret_key }}",
+            "region_name": "eu-central-1",
+            "addressing_style": "path",
+            "signature_version": "s3v4",
+            "bucket_name": "pulp3",
+            "endpoint_url": "http://minio:9000",
+            "default_acl": "@none None",
+        }
+    }
+}
+# TODO: this looks deprecated. Its a bit hard to find info but:
+# a) it shows in boto repository (deprecated/archived)
+# b) it doesnt show in boto3 repository: https://github.com/search?q=repo%3Aboto%2Fboto3%20USE_SIGV4&type=code
 S3_USE_SIGV4 = True
-AWS_S3_SIGNATURE_VERSION = "s3v4"
-AWS_STORAGE_BUCKET_NAME = "pulp3"
-AWS_S3_ENDPOINT_URL = "http://minio:9000"
-AWS_DEFAULT_ACL = "@none None"
 {% endif %}
 
 {% if azure_test | default(false) %}
-STORAGES__default = "storages.backends.azure_storage.AzureStorage"
 MEDIA_ROOT = ""
-AZURE_ACCOUNT_KEY = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
-AZURE_ACCOUNT_NAME = "devstoreaccount1"
-AZURE_CONTAINER =  "pulp-test"
-AZURE_LOCATION = "pulp3"
-AZURE_OVERWRITE_FILES = True
-AZURE_URL_EXPIRATION_SECS = 120
-AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;'
+STORAGES = {
+    "default": {
+        "BACKEND": "storages.backends.azure_storage.AzureStorage",
+        "OPTIONS": {
+            "account_key": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==",
+            "account_name": "devstoreaccount1",
+            "azure_container": "pulp-test",
+            "location": "pulp3",
+            "overwrite_files": True,
+            "expiration_secs": 120,
+            "connection_string": 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;',
+        }
+    }
+}
 {% endif %}
 
 {% if gcp_test | default(false) %}
-STORAGES__default = "storages.backends.gcloud.GoogleCloudStorage"
-MEDIA_ROOT = ""
-GS_BUCKET_NAME = "gcppulp"
-GS_CUSTOM_ENDPOINT = "http://ci-gcp:4443"
+STORAGES = {
+    "default": {
+        "BACKEND": "storages.backends.gcloud.GoogleCloudStorage",
+        "OPTIONS": {
+            "media_root": "",
+            "gs_bucket_name": "gcppulp",
+            "gs_custom_endpoint": "http://ci-gcp:4443",
+        }
+    }
+}
 {% endif %}

CHANGES/plugin_api/5404.removal

@@ -0,0 +1 @@
+Removed `DEFAULT_FILE_STORAGE` setting in favour of the new `STORAGES` options introduced in Django 4.2.

docs/admin/reference/settings.md

@@ -115,7 +115,7 @@ Overview of the integration with Pulp:
 ### REDIRECT_TO_OBJECT_STORAGE
 
 When set to `True` access to artifacts is redirected to the corresponding Cloud storage
-configured in `STORAGES['default']` using pre-authenticated URLs. When set to `False`
+configured in `STORAGES['default']['BACKEND']` using pre-authenticated URLs. When set to `False`
 artifacts are always served by the content app instead.
 
 Defaults to `True`; ignored for local file storage.
@@ -124,7 +124,7 @@ Defaults to `True`; ignored for local file storage.
 
 The location where Pulp will store files. By default this is `/var/lib/pulp/media`.
 
-This only affects storage location when `STORAGES['default']` is set to
+This only affects storage location when `STORAGES['default']['BACKEND']` is set to
 `pulpcore.app.models.storage.FileSystem`.
 
 See the [storage documentation](site:pulpcore/docs/admin/guides/configure-pulp/configure-storages.md) for more info.
@@ -222,7 +222,7 @@ It should have permissions of:
 ### CHUNKED_UPLOAD_DIR
 
 A relative path inside the DEPLOY_ROOT directory used exclusively for uploaded chunks. The
-uploaded chunks are stored in the default storage specified by `STORAGES['default']`. This
+uploaded chunks are stored in the default storage specified by `STORAGES['default']['BACKEND']`. This
 option allows users to customize the actual place where chunked uploads should be stored within
 the declared storage. The default, `upload`, is sufficient for most use cases. A change to
 this setting only applies to uploads created after the change.

pulp_file/tests/functional/api/test_download_policies.py

@@ -54,7 +54,7 @@ def test_download_policy(
     download_policy,
 ):
     """Test that "on_demand" and "streamed" download policies work as expected."""
-    if download_policy == "on_demand" and "SFTP" in settings.STORAGES["default"]:
+    if download_policy == "on_demand" and "SFTP" in settings.STORAGES["default"]["BACKEND"]:
         pytest.skip("This storage technology is not properly supported.")
 
     remote = file_remote_ssl_factory(
@@ -148,7 +148,7 @@ def test_download_policy(
     assert expected_checksum == actual_checksum
     if (
         download_policy == "immediate"
-        and settings.STORAGES["default"] != "pulpcore.app.models.storage.FileSystem"
+        and settings.STORAGES["default"]["BACKEND"] != "pulpcore.app.models.storage.FileSystem"
         and settings.REDIRECT_TO_OBJECT_STORAGE
     ):
         content_disposition = downloaded_file.response_obj.headers.get("Content-Disposition")
@@ -187,7 +187,7 @@ def test_download_policy(
     content_unit = expected_files_list[4]
     content_unit_url = urljoin(distribution.base_url, content_unit[0])
     # The S3 test API project doesn't handle invalid Range values correctly
-    if settings.STORAGES["default"] == "pulpcore.app.models.storage.FileSystem":
+    if settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem":
         with pytest.raises(ClientResponseError) as exc:
             range_header = {"Range": "bytes=-1-11"}
             download_file(content_unit_url, headers=range_header)

pulpcore/app/apps.py

@@ -321,11 +321,11 @@ def _ensure_default_domain(sender, **kwargs):
         if (
             settings.HIDE_GUARDED_DISTRIBUTIONS != default.hide_guarded_distributions
             or settings.REDIRECT_TO_OBJECT_STORAGE != default.redirect_to_object_storage
-            or settings.STORAGES["default"] != default.storage_class
+            or settings.STORAGES["default"]["BACKEND"] != default.storage_class
         ):
             default.hide_guarded_distributions = settings.HIDE_GUARDED_DISTRIBUTIONS
             default.redirect_to_object_storage = settings.REDIRECT_TO_OBJECT_STORAGE
-            default.storage_class = settings.STORAGES["default"]
+            default.storage_class = settings.STORAGES["default"]["BACKEND"]
             default.save(skip_hooks=True)
 
 
@@ -392,7 +392,7 @@ def _get_permission(perm):
 
 def _populate_artifact_serving_distribution(sender, apps, verbosity, **kwargs):
     if (
-        settings.STORAGES["default"] == "pulpcore.app.models.storage.FileSystem"
+        settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem"
         or not settings.REDIRECT_TO_OBJECT_STORAGE
     ):
         try:

pulpcore/app/checks.py

@@ -38,7 +38,7 @@ def secret_key_check(app_configs, **kwargs):
 def storage_paths(app_configs, **kwargs):
     warnings = []
 
-    if settings.STORAGES["default"] == "pulpcore.app.models.storage.FileSystem":
+    if settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem":
         try:
             media_root_dev = Path(settings.MEDIA_ROOT).stat().st_dev
         except OSError:

pulpcore/app/importexport.py

@@ -114,7 +114,7 @@ def export_artifacts(export, artifact_pks):
     with ProgressReport(**data) as pb:
         pb.BATCH_INTERVAL = 5000
 
-        if settings.STORAGES["default"] != "pulpcore.app.models.storage.FileSystem":
+        if settings.STORAGES["default"]["BACKEND"] != "pulpcore.app.models.storage.FileSystem":
             with tempfile.TemporaryDirectory(dir=".") as temp_dir:
                 for offset in range(0, len(artifact_pks), EXPORT_BATCH_SIZE):
                     batch = artifact_pks[offset : offset + EXPORT_BATCH_SIZE]

pulpcore/app/settings.py

@@ -376,7 +376,7 @@
 )
 storage_validator.messages["combined"] = (
     "'REDIRECT_TO_OBJECT_STORAGE=True' is only supported with the local file, S3, GCP or Azure"
-    "storage backend configured in STORAGES['default']."
+    "storage backend configured in STORAGES['default']['BACKEND']."
 )
 
 cache_enabled_validator = Validator("CACHE_ENABLED", eq=True)

pulpcore/app/tasks/export.py

@@ -70,7 +70,7 @@ def _export_to_file_system(path, relative_paths_to_artifacts, method=FS_EXPORT_M
         ValidationError: When path is not in the ALLOWED_EXPORT_PATHS setting
     """
     using_filesystem_storage = (
-        settings.STORAGES["default"] == "pulpcore.app.models.storage.FileSystem"
+        settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem"
     )
 
     if method != FS_EXPORT_METHODS.WRITE and not using_filesystem_storage:

pulpcore/app/util.py

@@ -536,7 +536,7 @@ def get_artifact_url(artifact, headers=None, http_method=None):
         if settings.DOMAIN_ENABLED:
             loc = f"domain {artifact_domain.name}.storage_class"
         else:
-            loc = "settings.STORAGES['default']"
+            loc = "settings.STORAGES['default']['BACKEND']"
 
         raise NotImplementedError(
             f"The value {loc}={artifact_domain.storage_class} does not allow redirecting."
@@ -582,7 +582,9 @@ def get_default_domain():
         try:
             default_domain = Domain.objects.get(name="default")
         except Domain.DoesNotExist:
-            default_domain = Domain(name="default", storage_class=settings.STORAGES["default"])
+            default_domain = Domain(
+                name="default", storage_class=settings.STORAGES["default"]["BACKEND"]
+            )
             default_domain.save(skip_hooks=True)
 
     return default_domain

pulpcore/pytest_plugin.py

@@ -621,7 +621,7 @@ def _settings_factory(storage_class=None, storage_settings=None):
             "AZURE_CONNECTION_STRING",
         ]
         settings = storage_settings or dict()
-        backend = storage_class or settings.STORAGES["default"]
+        backend = storage_class or settings.STORAGES["default"]["BACKEND"]
         for key in keys[backend]:
             if key not in settings:
                 settings[key] = getattr(pulp_settings, key, None)

pulpcore/tests/functional/api/test_artifact_distribution.py

@@ -29,7 +29,7 @@ def test_artifact_distribution(random_artifact):
     hasher = sha256()
     hasher.update(response.content)
     assert hasher.hexdigest() == random_artifact.sha256
-    if settings.STORAGES["default"] in OBJECT_STORAGES:
+    if settings.STORAGES["default"]["BACKEND"] in OBJECT_STORAGES:
         content_disposition = response.headers.get("Content-Disposition")
         assert content_disposition is not None
         filename = artifact_uuid

pulpcore/tests/functional/api/test_crd_artifacts.py

@@ -148,7 +148,7 @@ def test_upload_mixed_attrs(pulpcore_bindings, pulpcore_random_file):
 def test_delete_artifact(pulpcore_bindings, pulpcore_random_file, gen_user):
     """Verify that the deletion of artifacts is prohibited for both regular users and
     administrators."""
-    if settings.STORAGES["default"] != "pulpcore.app.models.storage.FileSystem":
+    if settings.STORAGES["default"]["BACKEND"] != "pulpcore.app.models.storage.FileSystem":
         pytest.skip("this test only works for filesystem storage")
     media_root = settings.MEDIA_ROOT
 

pulpcore/tests/functional/api/test_crud_domains.py

@@ -59,7 +59,7 @@ def test_default_domain(pulpcore_bindings):
     # Read the default domain, ensure storage is set to default
     default_domain = domains.results[0]
     assert default_domain.name == "default"
-    assert default_domain.storage_class == settings.STORAGES["default"]
+    assert default_domain.storage_class == settings.STORAGES["default"]["BACKEND"]
     assert default_domain.redirect_to_object_storage == settings.REDIRECT_TO_OBJECT_STORAGE
     assert default_domain.hide_guarded_distributions == settings.HIDE_GUARDED_DISTRIBUTIONS
 

pulpcore/tests/functional/api/test_status.py

@@ -142,7 +142,7 @@ def verify_get_response(status, expected_schema):
     assert status["content_settings"]["content_path_prefix"] is not None
 
     assert status["storage"]["used"] is not None
-    if settings.STORAGES["default"] != "pulpcore.app.models.storage.FileSystem":
+    if settings.STORAGES["default"]["BACKEND"] != "pulpcore.app.models.storage.FileSystem":
         assert status["storage"]["free"] is None
         assert status["storage"]["total"] is None
     else:

pulpcore/tests/functional/api/using_plugin/test_orphans.py

@@ -73,7 +73,7 @@ def test_orphans_delete(
     content_unit = file_bindings.ContentFilesApi.read(file_random_content_unit.pulp_href)
     artifact = pulpcore_bindings.ArtifactsApi.read(random_artifact.pulp_href)
 
-    if settings.STORAGES["default"] == "pulpcore.app.models.storage.FileSystem":
+    if settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem":
         # Verify that the artifacts are on disk
         relative_path = pulpcore_bindings.ArtifactsApi.read(content_unit.artifact).file
         artifact_path1 = os.path.join(settings.MEDIA_ROOT, relative_path)
@@ -89,7 +89,7 @@ def test_orphans_delete(
         with pytest.raises(file_bindings.ApiException) as exc:
             file_bindings.ContentFilesApi.read(file_random_content_unit.pulp_href)
         assert exc.value.status == 404
-        if settings.STORAGES["default"] == "pulpcore.app.models.storage.FileSystem":
+        if settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem":
             assert os.path.exists(artifact_path1) is False
             assert os.path.exists(artifact_path2) is False
 
@@ -108,7 +108,7 @@ def test_orphans_cleanup(
     content_unit = file_bindings.ContentFilesApi.read(file_random_content_unit.pulp_href)
     artifact = pulpcore_bindings.ArtifactsApi.read(random_artifact.pulp_href)
 
-    if settings.STORAGES["default"] == "pulpcore.app.models.storage.FileSystem":
+    if settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem":
         # Verify that the artifacts are on disk
         relative_path = pulpcore_bindings.ArtifactsApi.read(content_unit.artifact).file
         artifact_path1 = os.path.join(settings.MEDIA_ROOT, relative_path)
@@ -123,7 +123,7 @@ def test_orphans_cleanup(
     with pytest.raises(file_bindings.ApiException) as exc:
         file_bindings.ContentFilesApi.read(file_random_content_unit.pulp_href)
     assert exc.value.status == 404
-    if settings.STORAGES["default"] == "pulpcore.app.models.storage.FileSystem":
+    if settings.STORAGES["default"]["BACKEND"] == "pulpcore.app.models.storage.FileSystem":
         assert os.path.exists(artifact_path1) is False
         assert os.path.exists(artifact_path2) is False
 

pulpcore/tests/unit/models/test_content.py

@@ -43,7 +43,7 @@ def test_create_read_delete_content(tmp_path):
 
 @pytest.mark.django_db
 def test_storage_location(tmp_path, settings):
-    if settings.STORAGES["default"] != "pulpcore.app.models.storage.FileSystem":
+    if settings.STORAGES["default"]["BACKEND"] != "pulpcore.app.models.storage.FileSystem":
         pytest.skip("Skipping test for nonlocal storage.")
 
     tf = tmp_path / "ab"