Add --all-repos argument to deletion task [RHELDST-13644]

There are ocassions where we would like to remove a package from all
repos its associated with. This can be tedious as every repo has to be
listed out. This change adds a convenient --all-repos argument to
address this issue.

src/pubtools/_pulp/tasks/delete.py

@@ -27,6 +27,7 @@
 
 MODULEMD_REGEX = re.compile(r"^[-.+\w]+:[-.+\w]+:\d+(:[-.+\w]+){0,2}$")
 
+ALL_REPOS_INDICATOR = "*"
 
 @attr.s
 class ClearedRepo(object):
@@ -78,7 +79,10 @@ def add_args(self):
 
         self.parser.add_argument(
             "--repo",
-            help="remove content from these comma-seperated repositories ",
+            help="remove content from these comma-seperated repositories. If "
+                 "'%s' is used, the package will be removed from all repos, "
+                 "excluding all-rpm-content-* repos. These may be added "
+                 "separately." % ALL_REPOS_INDICATOR,
             type=str,
             action=SplitAndExtend,
             split_on=",",
@@ -564,22 +568,30 @@ def map_to_repo(self, units, repos, unit_attr):
         repo_map = {}
         unit_map = {}
         repos = sorted(repos)
-
         for unit in sorted(units):
             unit_name = getattr(unit, unit_attr)
             unit_map.setdefault(unit_name, RemoveUnitItem(unit=unit, repos=[]))
-            for repo in repos:
-                if repo not in unit.repository_memberships:
-                    LOG.warning(
-                        "%s is not present in %s",
-                        unit_name,
-                        repo,
-                    )
-                else:
+            if ALL_REPOS_INDICATOR in repos:
+                for repo in unit.repository_memberships:
+                    if re.match("all-rpm-content-.*", repo) and repo not in repos:
+                        continue
                     repo_map.setdefault(repo, []).append(unit)
                     unit_map.get(unit_name).repos.append(repo)
+            else:
+                for repo in repos:
+                    if repo not in unit.repository_memberships:
+                        LOG.warning(
+                            "%s is not present in %s",
+                            unit_name,
+                            repo,
+                        )
+                    else:
+                        repo_map.setdefault(repo, []).append(unit)
+                        unit_map.get(unit_name).repos.append(repo)
 
         missing = set(repos) - set(repo_map.keys())
+        if ALL_REPOS_INDICATOR in missing:
+            missing.remove(ALL_REPOS_INDICATOR)
         if missing:
             missing = ", ".join(sorted(list(missing)))
             LOG.warning("No units to remove from %s", missing)

tests/delete/test_delete_packages.py

@@ -801,3 +801,187 @@ def test_delete_rpms_skip_publish(command_tester, fake_collector, monkeypatch):
     # All the files exist on Pulp
     files_search = list(client.search_content(criteria1).result())
     assert len(files_search) == 2
+
+
+def test_delete_rpms_with_all_repos(command_tester, fake_collector, monkeypatch):
+    """Deleting RPMs from repos succeeds"""
+
+    repo1 = YumRepository(
+        id="some-yumrepo", relative_url="some/publish/url", mutable_urls=["repomd.xml"]
+    )
+    repo2 = YumRepository(
+        id="other-yumrepo",
+        relative_url="other/publish/url",
+        mutable_urls=["repomd.xml"],
+    )
+    arc_repo_1 = YumRepository(
+        id="all-rpm-content-gg",
+        relative_url="other/publish/url",
+        mutable_urls=["repomd.xml"],
+    )
+    arc_repo_2 = YumRepository(
+        id="all-rpm-content-xp",
+        relative_url="other/publish/url",
+        mutable_urls=["repomd.xml"],
+    )
+
+    files1 = [
+        RpmUnit(
+            name="bash",
+            version="1.23",
+            release="1.test8",
+            arch="x86_64",
+            filename="bash-1.23-1.test8_x86_64.rpm",
+            sha256sum="a" * 64,
+            md5sum="b" * 32,
+            signing_key="aabbcc",
+            unit_id="file1_rpm1",
+        ),
+        RpmUnit(
+            name="dash",
+            version="2.25",
+            release="1.test8",
+            arch="x86_64",
+            filename="dash-2.25-1.test8_x86_64.rpm",
+            sha256sum="a" * 64,
+            md5sum="b" * 32,
+            signing_key="aabbcc",
+            unit_id="file1_rpm2",
+        ),
+    ]
+
+    files2 = [
+        RpmUnit(
+            name="crash",
+            version="3.30",
+            release="1.test8",
+            arch="s390x",
+            filename="crash-3.30-1.test8_s390x.rpm",
+            sha256sum="a" * 64,
+            md5sum="b" * 32,
+            signing_key="aabbcc",
+            unit_id="file2_rpm1",
+        )
+    ]
+
+
+    with FakeDeletePackages() as task_instance:
+        task_instance.pulp_client_controller.insert_repository(repo1)
+        task_instance.pulp_client_controller.insert_repository(repo2)
+        task_instance.pulp_client_controller.insert_repository(arc_repo_1)
+        task_instance.pulp_client_controller.insert_repository(arc_repo_2)
+        task_instance.pulp_client_controller.insert_units(repo1, files1)
+        task_instance.pulp_client_controller.insert_units(repo2, files2)
+        task_instance.pulp_client_controller.insert_units(repo2, [files1[0]])
+        task_instance.pulp_client_controller.insert_units(arc_repo_1, files1)
+        task_instance.pulp_client_controller.insert_units(arc_repo_2, files2)
+        task_instance.pulp_client_controller.insert_units(arc_repo_2, [files1[0]])
+
+        # It should run with expected output.
+        command_tester.test(
+            task_instance.main,
+            [
+                "test-delete",
+                "--pulp-url",
+                "https://pulp.example.com/",
+                "--repo",
+                "*",
+                "--repo",
+                "all-rpm-content-xp",
+                "--file",
+                "bash-1.23-1.test8_x86_64.rpm",
+                "--file",
+                "dash-2.25-1.test8_x86_64.rpm",
+                "--signing-key",
+                "aabbcc",
+            ]
+        )
+    # It should record that it removed these push items:
+    assert sorted(fake_collector.items, key=lambda pi: pi["filename"]) == [
+        {
+            "origin": "pulp",
+            "src": None,
+            "dest": "all-rpm-content-xp",
+            "signing_key": None,
+            "filename": "bash-1.23-1.test8.x86_64.rpm",
+            "state": "DELETED",
+            "build": None,
+            "checksums": {"sha256": "a" * 64},
+        },
+        {
+            "origin": "pulp",
+            "src": None,
+            "dest": "other-yumrepo",
+            "signing_key": None,
+            "filename": "bash-1.23-1.test8.x86_64.rpm",
+            "state": "DELETED",
+            "build": None,
+            "checksums": {"sha256": "a" * 64},
+        },
+        {
+            "origin": "pulp",
+            "src": None,
+            "dest": "some-yumrepo",
+            "signing_key": None,
+            "filename": "bash-1.23-1.test8.x86_64.rpm",
+            "state": "DELETED",
+            "build": None,
+            "checksums": {"sha256": "a" * 64},
+        },
+        {
+            "origin": "pulp",
+            "src": None,
+            "dest": "some-yumrepo",
+            "signing_key": None,
+            "filename": "dash-2.25-1.test8.x86_64.rpm",
+            "state": "DELETED",
+            "build": None,
+            "checksums": {"sha256": "a" * 64},
+        },
+    ]
+    #
+    # verify whether files were deleted on Pulp
+    client = task_instance.pulp_client
+
+    # get the repo where the files were deleted
+    repos = sorted(
+        list(
+            client.search_repository(
+                Criteria.with_id(["all-rpm-content-gg", "all-rpm-content-xp", "some-yumrepo", "other-yumrepo"])
+            ).result()
+        ),
+        key=lambda r: r.id,
+    )
+    assert len(repos) == 4
+    r_arc_1, r_arc_2, r2, r1 = repos
+
+    assert r_arc_1.id == arc_repo_1.id
+    assert r_arc_2.id == arc_repo_2.id
+    assert r1.id == repo1.id
+    assert r2.id == repo2.id
+    #
+    # # criteria with the unit_ids
+    # # critera1 for files1 in repo1
+    unit_ids = []
+    for f in files1:
+        unit_ids.append(f.unit_id)
+
+    for f in files2:
+        unit_ids.append(f.unit_id)
+    search_criteria = Criteria.with_field("unit_id", Matcher.in_(unit_ids))
+
+    # No files should be in repo1
+    result1 =list(r1.search_content(search_criteria).result())
+    assert len(result1) == 0
+
+    result2 = list(r2.search_content(search_criteria).result())
+    assert len(result2) == 1
+    assert result2[0].unit_id == files2[0].unit_id
+
+    # --repos * arg should skip over all-rpm-content repos...
+    result3 = list(r_arc_1.search_content(search_criteria).result())
+    assert len(result3) == 2
+
+    # ... unless specified as an extra --repo arg.
+    result4 = list(r_arc_2.search_content(search_criteria).result())
+    assert len(result4) == 1

tests/logs/delete/test_delete_packages/test_delete_rpms_with_all_repos.jsonl

@@ -0,0 +1,22 @@
+{"event": {"type": "delete-files-start"}}
+{"event": {"type": "get-files-start"}}
+{"event": {"type": "get-files-end"}}
+{"event": {"type": "unassociate-files-start"}}
+{"event": {"type": "unassociate-files-end"}}
+{"event": {"type": "record-push-items-start"}}
+{"event": {"type": "record-push-items-end"}}
+{"event": {"type": "delete-files-end"}}
+{"event": {"type": "delete-rpms-start"}}
+{"event": {"type": "get-rpms-start"}}
+{"event": {"type": "get-rpms-end"}}
+{"event": {"type": "unassociate-rpms-start"}}
+{"event": {"type": "unassociate-rpms-end"}}
+{"event": {"type": "record-push-items-start"}}
+{"event": {"type": "record-push-items-end"}}
+{"event": {"type": "delete-rpms-end"}}
+{"event": {"type": "publish-start"}}
+{"event": {"type": "publish-end"}}
+{"event": {"type": "set-cdn_published-start"}}
+{"event": {"type": "set-cdn_published-end"}}
+{"event": {"type": "flush-ud-cache-start"}}
+{"event": {"type": "flush-ud-cache-end"}}

tests/logs/delete/test_delete_packages/test_delete_rpms_with_all_repos.txt

@@ -0,0 +1,36 @@
+[    INFO] Delete files: started
+[    INFO] Get files: started
+[ WARNING] Requested unit(s) don't exist as file: bash-1.23-1.test8_x86_64.rpm, dash-2.25-1.test8_x86_64.rpm
+[    INFO] 0 unit(s) found for deletion
+[    INFO] Get files: finished
+[ WARNING] No units to remove from all-rpm-content-xp
+[    INFO] Unassociate files: started
+[ WARNING] Nothing mapped for removal
+[    INFO] Unassociate files: finished
+[    INFO] Record push items: started
+[    INFO] Record push items: finished
+[    INFO] Delete files: finished
+[    INFO] Delete RPMs: started
+[    INFO] Get RPMs: started
+[    INFO] 2 unit(s) found for deletion
+[    INFO] Get RPMs: finished
+[    INFO] Deleting bash-1.23-1.test8_x86_64.rpm from all-rpm-content-xp, other-yumrepo, some-yumrepo
+[    INFO] Deleting dash-2.25-1.test8_x86_64.rpm from some-yumrepo
+[    INFO] Unassociate RPMs: started
+[    INFO] all-rpm-content-xp: removed 1 rpm(s), tasks: e3e70682-c209-4cac-629f-6fbed82c07cd
+[    INFO] other-yumrepo: removed 1 rpm(s), tasks: 82e2e662-f728-b4fa-4248-5e3a0a5d2f34
+[    INFO] some-yumrepo: removed 2 rpm(s), tasks: d4713d60-c8a7-0639-eb11-67b367a9c378
+[    INFO] Unassociate RPMs: finished
+[    INFO] Record push items: started
+[    INFO] Record push items: finished
+[    INFO] Delete RPMs: finished
+[    INFO] Publish: started
+[    INFO] Publishing all-rpm-content-xp
+[    INFO] Publishing other-yumrepo
+[    INFO] Publishing some-yumrepo
+[    INFO] Publish: finished
+[    INFO] Set cdn_published: started
+[    INFO] Set cdn_published: finished
+[    INFO] Flush UD cache: started
+[    INFO] UD cache flush is not enabled.
+[    INFO] Flush UD cache: finished