From b4d39d7baaff3033290170a31109b49b9cb49ae9 Mon Sep 17 00:00:00 2001 From: Aidan McMahon-Smith Date: Thu, 17 Oct 2024 14:50:25 +0200 Subject: [PATCH] Add --all-repos argument to deletion task [RHELDST-13644] There are ocassions where we would like to remove a package from all repos its associated with. This can be tedious as every repo has to be listed out. This change adds a convenient --all-repos argument to address this issue. --- src/pubtools/_pulp/tasks/delete.py | 17 +- tests/delete/test_delete_packages.py | 184 ++++++++++++++++++ .../test_delete_rpms_with_all_repos.jsonl | 22 +++ .../test_delete_rpms_with_all_repos.txt | 38 ++++ 4 files changed, 259 insertions(+), 2 deletions(-) create mode 100644 tests/logs/delete/test_delete_packages/test_delete_rpms_with_all_repos.jsonl create mode 100644 tests/logs/delete/test_delete_packages/test_delete_rpms_with_all_repos.txt diff --git a/src/pubtools/_pulp/tasks/delete.py b/src/pubtools/_pulp/tasks/delete.py index e2c02369..65059e0a 100644 --- a/src/pubtools/_pulp/tasks/delete.py +++ b/src/pubtools/_pulp/tasks/delete.py @@ -27,6 +27,7 @@ MODULEMD_REGEX = re.compile(r"^[-.+\w]+:[-.+\w]+:\d+(:[-.+\w]+){0,2}$") +ALL_REPOS_INDICATOR = "*" @attr.s class ClearedRepo(object): @@ -78,7 +79,10 @@ def add_args(self): self.parser.add_argument( "--repo", - help="remove content from these comma-seperated repositories ", + help="remove content from these comma-seperated repositories. If " + "'%s' is used, the package will be removed from all repos, " + "excluding all-rpm-content-* repos. These may be added " + "separately." % ALL_REPOS_INDICATOR, type=str, action=SplitAndExtend, split_on=",", @@ -563,8 +567,17 @@ def log_missing_units(self, searched_units, unit_attr, unit_type, unit_names): def map_to_repo(self, units, repos, unit_attr): repo_map = {} unit_map = {} + if ALL_REPOS_INDICATOR in repos: + repos = repos.copy() + for unit in sorted(units): + # If an RPM package is removed from it's all-rpm-content + # repo, it will be an orphan and get garbage collected. + # This saves it from needing to be reuploaded later. + repos += [r for r in unit.repository_memberships + if not re.match("all-rpm-content-.*", r)] + repos.remove(ALL_REPOS_INDICATOR) + repos = list(set(repos)) repos = sorted(repos) - for unit in sorted(units): unit_name = getattr(unit, unit_attr) unit_map.setdefault(unit_name, RemoveUnitItem(unit=unit, repos=[])) diff --git a/tests/delete/test_delete_packages.py b/tests/delete/test_delete_packages.py index a6091b72..e4ac156c 100644 --- a/tests/delete/test_delete_packages.py +++ b/tests/delete/test_delete_packages.py @@ -801,3 +801,187 @@ def test_delete_rpms_skip_publish(command_tester, fake_collector, monkeypatch): # All the files exist on Pulp files_search = list(client.search_content(criteria1).result()) assert len(files_search) == 2 + + +def test_delete_rpms_with_all_repos(command_tester, fake_collector, monkeypatch): + """Deleting RPMs from repos succeeds""" + + repo1 = YumRepository( + id="some-yumrepo", relative_url="some/publish/url", mutable_urls=["repomd.xml"] + ) + repo2 = YumRepository( + id="other-yumrepo", + relative_url="other/publish/url", + mutable_urls=["repomd.xml"], + ) + arc_repo_1 = YumRepository( + id="all-rpm-content-gg", + relative_url="other/publish/url", + mutable_urls=["repomd.xml"], + ) + arc_repo_2 = YumRepository( + id="all-rpm-content-xp", + relative_url="other/publish/url", + mutable_urls=["repomd.xml"], + ) + + files1 = [ + RpmUnit( + name="bash", + version="1.23", + release="1.test8", + arch="x86_64", + filename="bash-1.23-1.test8_x86_64.rpm", + sha256sum="a" * 64, + md5sum="b" * 32, + signing_key="aabbcc", + unit_id="file1_rpm1", + ), + RpmUnit( + name="dash", + version="2.25", + release="1.test8", + arch="x86_64", + filename="dash-2.25-1.test8_x86_64.rpm", + sha256sum="a" * 64, + md5sum="b" * 32, + signing_key="aabbcc", + unit_id="file1_rpm2", + ), + ] + + files2 = [ + RpmUnit( + name="crash", + version="3.30", + release="1.test8", + arch="s390x", + filename="crash-3.30-1.test8_s390x.rpm", + sha256sum="a" * 64, + md5sum="b" * 32, + signing_key="aabbcc", + unit_id="file2_rpm1", + ) + ] + + + with FakeDeletePackages() as task_instance: + task_instance.pulp_client_controller.insert_repository(repo1) + task_instance.pulp_client_controller.insert_repository(repo2) + task_instance.pulp_client_controller.insert_repository(arc_repo_1) + task_instance.pulp_client_controller.insert_repository(arc_repo_2) + task_instance.pulp_client_controller.insert_units(repo1, files1) + task_instance.pulp_client_controller.insert_units(repo2, files2) + task_instance.pulp_client_controller.insert_units(repo2, [files1[0]]) + task_instance.pulp_client_controller.insert_units(arc_repo_1, files1) + task_instance.pulp_client_controller.insert_units(arc_repo_2, files2) + task_instance.pulp_client_controller.insert_units(arc_repo_2, [files1[0]]) + + # It should run with expected output. + command_tester.test( + task_instance.main, + [ + "test-delete", + "--pulp-url", + "https://pulp.example.com/", + "--repo", + "*", + "--repo", + "all-rpm-content-xp", + "--file", + "bash-1.23-1.test8_x86_64.rpm", + "--file", + "dash-2.25-1.test8_x86_64.rpm", + "--signing-key", + "aabbcc", + ] + ) + # It should record that it removed these push items: + assert sorted(fake_collector.items, key=lambda pi: pi["filename"]) == [ + { + "origin": "pulp", + "src": None, + "dest": "all-rpm-content-xp", + "signing_key": None, + "filename": "bash-1.23-1.test8.x86_64.rpm", + "state": "DELETED", + "build": None, + "checksums": {"sha256": "a" * 64}, + }, + { + "origin": "pulp", + "src": None, + "dest": "other-yumrepo", + "signing_key": None, + "filename": "bash-1.23-1.test8.x86_64.rpm", + "state": "DELETED", + "build": None, + "checksums": {"sha256": "a" * 64}, + }, + { + "origin": "pulp", + "src": None, + "dest": "some-yumrepo", + "signing_key": None, + "filename": "bash-1.23-1.test8.x86_64.rpm", + "state": "DELETED", + "build": None, + "checksums": {"sha256": "a" * 64}, + }, + { + "origin": "pulp", + "src": None, + "dest": "some-yumrepo", + "signing_key": None, + "filename": "dash-2.25-1.test8.x86_64.rpm", + "state": "DELETED", + "build": None, + "checksums": {"sha256": "a" * 64}, + }, + ] + # + # verify whether files were deleted on Pulp + client = task_instance.pulp_client + + # get the repo where the files were deleted + repos = sorted( + list( + client.search_repository( + Criteria.with_id(["all-rpm-content-gg", "all-rpm-content-xp", "some-yumrepo", "other-yumrepo"]) + ).result() + ), + key=lambda r: r.id, + ) + assert len(repos) == 4 + r_arc_1, r_arc_2, r2, r1 = repos + + assert r_arc_1.id == arc_repo_1.id + assert r_arc_2.id == arc_repo_2.id + assert r1.id == repo1.id + assert r2.id == repo2.id + # + # # criteria with the unit_ids + # # critera1 for files1 in repo1 + unit_ids = [] + for f in files1: + unit_ids.append(f.unit_id) + + for f in files2: + unit_ids.append(f.unit_id) + search_criteria = Criteria.with_field("unit_id", Matcher.in_(unit_ids)) + + # No files should be in repo1 + result1 =list(r1.search_content(search_criteria).result()) + assert len(result1) == 0 + + result2 = list(r2.search_content(search_criteria).result()) + assert len(result2) == 1 + assert result2[0].unit_id == files2[0].unit_id + + # --repos * arg should skip over all-rpm-content repos... + result3 = list(r_arc_1.search_content(search_criteria).result()) + assert len(result3) == 2 + + # ... unless specified as an extra --repo arg. + result4 = list(r_arc_2.search_content(search_criteria).result()) + assert len(result4) == 1 diff --git a/tests/logs/delete/test_delete_packages/test_delete_rpms_with_all_repos.jsonl b/tests/logs/delete/test_delete_packages/test_delete_rpms_with_all_repos.jsonl new file mode 100644 index 00000000..1f9123c9 --- /dev/null +++ b/tests/logs/delete/test_delete_packages/test_delete_rpms_with_all_repos.jsonl @@ -0,0 +1,22 @@ +{"event": {"type": "delete-files-start"}} +{"event": {"type": "get-files-start"}} +{"event": {"type": "get-files-end"}} +{"event": {"type": "unassociate-files-start"}} +{"event": {"type": "unassociate-files-end"}} +{"event": {"type": "record-push-items-start"}} +{"event": {"type": "record-push-items-end"}} +{"event": {"type": "delete-files-end"}} +{"event": {"type": "delete-rpms-start"}} +{"event": {"type": "get-rpms-start"}} +{"event": {"type": "get-rpms-end"}} +{"event": {"type": "unassociate-rpms-start"}} +{"event": {"type": "unassociate-rpms-end"}} +{"event": {"type": "record-push-items-start"}} +{"event": {"type": "record-push-items-end"}} +{"event": {"type": "delete-rpms-end"}} +{"event": {"type": "publish-start"}} +{"event": {"type": "publish-end"}} +{"event": {"type": "set-cdn_published-start"}} +{"event": {"type": "set-cdn_published-end"}} +{"event": {"type": "flush-ud-cache-start"}} +{"event": {"type": "flush-ud-cache-end"}} diff --git a/tests/logs/delete/test_delete_packages/test_delete_rpms_with_all_repos.txt b/tests/logs/delete/test_delete_packages/test_delete_rpms_with_all_repos.txt new file mode 100644 index 00000000..1835f426 --- /dev/null +++ b/tests/logs/delete/test_delete_packages/test_delete_rpms_with_all_repos.txt @@ -0,0 +1,38 @@ +[ INFO] Delete files: started +[ INFO] Get files: started +[ WARNING] Requested unit(s) don't exist as file: bash-1.23-1.test8_x86_64.rpm, dash-2.25-1.test8_x86_64.rpm +[ INFO] 0 unit(s) found for deletion +[ INFO] Get files: finished +[ WARNING] No units to remove from all-rpm-content-xp +[ INFO] Unassociate files: started +[ WARNING] Nothing mapped for removal +[ INFO] Unassociate files: finished +[ INFO] Record push items: started +[ INFO] Record push items: finished +[ INFO] Delete files: finished +[ INFO] Delete RPMs: started +[ INFO] Get RPMs: started +[ INFO] 2 unit(s) found for deletion +[ INFO] Get RPMs: finished +[ WARNING] dash-2.25-1.test8_x86_64.rpm is not present in all-rpm-content-xp +[ WARNING] dash-2.25-1.test8_x86_64.rpm is not present in other-yumrepo +[ INFO] Deleting bash-1.23-1.test8_x86_64.rpm from all-rpm-content-xp, other-yumrepo, some-yumrepo +[ INFO] Deleting dash-2.25-1.test8_x86_64.rpm from some-yumrepo +[ INFO] Unassociate RPMs: started +[ INFO] all-rpm-content-xp: removed 1 rpm(s), tasks: e3e70682-c209-4cac-629f-6fbed82c07cd +[ INFO] other-yumrepo: removed 1 rpm(s), tasks: 82e2e662-f728-b4fa-4248-5e3a0a5d2f34 +[ INFO] some-yumrepo: removed 2 rpm(s), tasks: d4713d60-c8a7-0639-eb11-67b367a9c378 +[ INFO] Unassociate RPMs: finished +[ INFO] Record push items: started +[ INFO] Record push items: finished +[ INFO] Delete RPMs: finished +[ INFO] Publish: started +[ INFO] Publishing all-rpm-content-xp +[ INFO] Publishing other-yumrepo +[ INFO] Publishing some-yumrepo +[ INFO] Publish: finished +[ INFO] Set cdn_published: started +[ INFO] Set cdn_published: finished +[ INFO] Flush UD cache: started +[ INFO] UD cache flush is not enabled. +[ INFO] Flush UD cache: finished