diff --git a/src/pubtools/_pulp/tasks/delete.py b/src/pubtools/_pulp/tasks/delete.py index e2c02369..b6765eb1 100644 --- a/src/pubtools/_pulp/tasks/delete.py +++ b/src/pubtools/_pulp/tasks/delete.py @@ -78,7 +78,9 @@ def add_args(self): self.parser.add_argument( "--repo", - help="remove content from these comma-seperated repositories ", + help="remove content from these comma-seperated repositories. " + "If '*' is used, the packge will be removed from all repos," + "excluding all-rpm-content repos. These may be added separately.", type=str, action=SplitAndExtend, split_on=",", @@ -564,21 +566,29 @@ def map_to_repo(self, units, repos, unit_attr): repo_map = {} unit_map = {} repos = sorted(repos) - for unit in sorted(units): unit_name = getattr(unit, unit_attr) unit_map.setdefault(unit_name, RemoveUnitItem(unit=unit, repos=[])) - for repo in repos: - if repo not in unit.repository_memberships: - LOG.warning( - "%s is not present in %s", - unit_name, - repo, - ) - else: + if "*" in repos: + for repo in unit.repository_memberships: + # If an RPM package is removed from it's all-rpm-content + # repo, it will be an orphan and get garbage collected. + # This saves it from needing to be reuploaded later. + if re.match("all-rpm-content-.*", repo) and repo not in repos: + continue repo_map.setdefault(repo, []).append(unit) unit_map.get(unit_name).repos.append(repo) - + else: + for repo in repos: + if repo not in unit.repository_memberships: + LOG.warning( + "%s is not present in %s", + unit_name, + repo, + ) + else: + repo_map.setdefault(repo, []).append(unit) + unit_map.get(unit_name).repos.append(repo) missing = set(repos) - set(repo_map.keys()) if missing: missing = ", ".join(sorted(list(missing))) diff --git a/tests/delete/test_delete_packages.py b/tests/delete/test_delete_packages.py index a6091b72..e4ac156c 100644 --- a/tests/delete/test_delete_packages.py +++ b/tests/delete/test_delete_packages.py @@ -801,3 +801,187 @@ def test_delete_rpms_skip_publish(command_tester, fake_collector, monkeypatch): # All the files exist on Pulp files_search = list(client.search_content(criteria1).result()) assert len(files_search) == 2 + + +def test_delete_rpms_with_all_repos(command_tester, fake_collector, monkeypatch): + """Deleting RPMs from repos succeeds""" + + repo1 = YumRepository( + id="some-yumrepo", relative_url="some/publish/url", mutable_urls=["repomd.xml"] + ) + repo2 = YumRepository( + id="other-yumrepo", + relative_url="other/publish/url", + mutable_urls=["repomd.xml"], + ) + arc_repo_1 = YumRepository( + id="all-rpm-content-gg", + relative_url="other/publish/url", + mutable_urls=["repomd.xml"], + ) + arc_repo_2 = YumRepository( + id="all-rpm-content-xp", + relative_url="other/publish/url", + mutable_urls=["repomd.xml"], + ) + + files1 = [ + RpmUnit( + name="bash", + version="1.23", + release="1.test8", + arch="x86_64", + filename="bash-1.23-1.test8_x86_64.rpm", + sha256sum="a" * 64, + md5sum="b" * 32, + signing_key="aabbcc", + unit_id="file1_rpm1", + ), + RpmUnit( + name="dash", + version="2.25", + release="1.test8", + arch="x86_64", + filename="dash-2.25-1.test8_x86_64.rpm", + sha256sum="a" * 64, + md5sum="b" * 32, + signing_key="aabbcc", + unit_id="file1_rpm2", + ), + ] + + files2 = [ + RpmUnit( + name="crash", + version="3.30", + release="1.test8", + arch="s390x", + filename="crash-3.30-1.test8_s390x.rpm", + sha256sum="a" * 64, + md5sum="b" * 32, + signing_key="aabbcc", + unit_id="file2_rpm1", + ) + ] + + + with FakeDeletePackages() as task_instance: + task_instance.pulp_client_controller.insert_repository(repo1) + task_instance.pulp_client_controller.insert_repository(repo2) + task_instance.pulp_client_controller.insert_repository(arc_repo_1) + task_instance.pulp_client_controller.insert_repository(arc_repo_2) + task_instance.pulp_client_controller.insert_units(repo1, files1) + task_instance.pulp_client_controller.insert_units(repo2, files2) + task_instance.pulp_client_controller.insert_units(repo2, [files1[0]]) + task_instance.pulp_client_controller.insert_units(arc_repo_1, files1) + task_instance.pulp_client_controller.insert_units(arc_repo_2, files2) + task_instance.pulp_client_controller.insert_units(arc_repo_2, [files1[0]]) + + # It should run with expected output. + command_tester.test( + task_instance.main, + [ + "test-delete", + "--pulp-url", + "https://pulp.example.com/", + "--repo", + "*", + "--repo", + "all-rpm-content-xp", + "--file", + "bash-1.23-1.test8_x86_64.rpm", + "--file", + "dash-2.25-1.test8_x86_64.rpm", + "--signing-key", + "aabbcc", + ] + ) + # It should record that it removed these push items: + assert sorted(fake_collector.items, key=lambda pi: pi["filename"]) == [ + { + "origin": "pulp", + "src": None, + "dest": "all-rpm-content-xp", + "signing_key": None, + "filename": "bash-1.23-1.test8.x86_64.rpm", + "state": "DELETED", + "build": None, + "checksums": {"sha256": "a" * 64}, + }, + { + "origin": "pulp", + "src": None, + "dest": "other-yumrepo", + "signing_key": None, + "filename": "bash-1.23-1.test8.x86_64.rpm", + "state": "DELETED", + "build": None, + "checksums": {"sha256": "a" * 64}, + }, + { + "origin": "pulp", + "src": None, + "dest": "some-yumrepo", + "signing_key": None, + "filename": "bash-1.23-1.test8.x86_64.rpm", + "state": "DELETED", + "build": None, + "checksums": {"sha256": "a" * 64}, + }, + { + "origin": "pulp", + "src": None, + "dest": "some-yumrepo", + "signing_key": None, + "filename": "dash-2.25-1.test8.x86_64.rpm", + "state": "DELETED", + "build": None, + "checksums": {"sha256": "a" * 64}, + }, + ] + # + # verify whether files were deleted on Pulp + client = task_instance.pulp_client + + # get the repo where the files were deleted + repos = sorted( + list( + client.search_repository( + Criteria.with_id(["all-rpm-content-gg", "all-rpm-content-xp", "some-yumrepo", "other-yumrepo"]) + ).result() + ), + key=lambda r: r.id, + ) + assert len(repos) == 4 + r_arc_1, r_arc_2, r2, r1 = repos + + assert r_arc_1.id == arc_repo_1.id + assert r_arc_2.id == arc_repo_2.id + assert r1.id == repo1.id + assert r2.id == repo2.id + # + # # criteria with the unit_ids + # # critera1 for files1 in repo1 + unit_ids = [] + for f in files1: + unit_ids.append(f.unit_id) + + for f in files2: + unit_ids.append(f.unit_id) + search_criteria = Criteria.with_field("unit_id", Matcher.in_(unit_ids)) + + # No files should be in repo1 + result1 =list(r1.search_content(search_criteria).result()) + assert len(result1) == 0 + + result2 = list(r2.search_content(search_criteria).result()) + assert len(result2) == 1 + assert result2[0].unit_id == files2[0].unit_id + + # --repos * arg should skip over all-rpm-content repos... + result3 = list(r_arc_1.search_content(search_criteria).result()) + assert len(result3) == 2 + + # ... unless specified as an extra --repo arg. + result4 = list(r_arc_2.search_content(search_criteria).result()) + assert len(result4) == 1 diff --git a/tests/logs/delete/test_delete_packages/test_delete_rpms_with_all_repos.jsonl b/tests/logs/delete/test_delete_packages/test_delete_rpms_with_all_repos.jsonl new file mode 100644 index 00000000..1f9123c9 --- /dev/null +++ b/tests/logs/delete/test_delete_packages/test_delete_rpms_with_all_repos.jsonl @@ -0,0 +1,22 @@ +{"event": {"type": "delete-files-start"}} +{"event": {"type": "get-files-start"}} +{"event": {"type": "get-files-end"}} +{"event": {"type": "unassociate-files-start"}} +{"event": {"type": "unassociate-files-end"}} +{"event": {"type": "record-push-items-start"}} +{"event": {"type": "record-push-items-end"}} +{"event": {"type": "delete-files-end"}} +{"event": {"type": "delete-rpms-start"}} +{"event": {"type": "get-rpms-start"}} +{"event": {"type": "get-rpms-end"}} +{"event": {"type": "unassociate-rpms-start"}} +{"event": {"type": "unassociate-rpms-end"}} +{"event": {"type": "record-push-items-start"}} +{"event": {"type": "record-push-items-end"}} +{"event": {"type": "delete-rpms-end"}} +{"event": {"type": "publish-start"}} +{"event": {"type": "publish-end"}} +{"event": {"type": "set-cdn_published-start"}} +{"event": {"type": "set-cdn_published-end"}} +{"event": {"type": "flush-ud-cache-start"}} +{"event": {"type": "flush-ud-cache-end"}} diff --git a/tests/logs/delete/test_delete_packages/test_delete_rpms_with_all_repos.txt b/tests/logs/delete/test_delete_packages/test_delete_rpms_with_all_repos.txt new file mode 100644 index 00000000..3b167a68 --- /dev/null +++ b/tests/logs/delete/test_delete_packages/test_delete_rpms_with_all_repos.txt @@ -0,0 +1,37 @@ +[ INFO] Delete files: started +[ INFO] Get files: started +[ WARNING] Requested unit(s) don't exist as file: bash-1.23-1.test8_x86_64.rpm, dash-2.25-1.test8_x86_64.rpm +[ INFO] 0 unit(s) found for deletion +[ INFO] Get files: finished +[ WARNING] No units to remove from *, all-rpm-content-xp +[ INFO] Unassociate files: started +[ WARNING] Nothing mapped for removal +[ INFO] Unassociate files: finished +[ INFO] Record push items: started +[ INFO] Record push items: finished +[ INFO] Delete files: finished +[ INFO] Delete RPMs: started +[ INFO] Get RPMs: started +[ INFO] 2 unit(s) found for deletion +[ INFO] Get RPMs: finished +[ WARNING] No units to remove from * +[ INFO] Deleting bash-1.23-1.test8_x86_64.rpm from all-rpm-content-xp, other-yumrepo, some-yumrepo +[ INFO] Deleting dash-2.25-1.test8_x86_64.rpm from some-yumrepo +[ INFO] Unassociate RPMs: started +[ INFO] all-rpm-content-xp: removed 1 rpm(s), tasks: e3e70682-c209-4cac-629f-6fbed82c07cd +[ INFO] other-yumrepo: removed 1 rpm(s), tasks: 82e2e662-f728-b4fa-4248-5e3a0a5d2f34 +[ INFO] some-yumrepo: removed 2 rpm(s), tasks: d4713d60-c8a7-0639-eb11-67b367a9c378 +[ INFO] Unassociate RPMs: finished +[ INFO] Record push items: started +[ INFO] Record push items: finished +[ INFO] Delete RPMs: finished +[ INFO] Publish: started +[ INFO] Publishing all-rpm-content-xp +[ INFO] Publishing other-yumrepo +[ INFO] Publishing some-yumrepo +[ INFO] Publish: finished +[ INFO] Set cdn_published: started +[ INFO] Set cdn_published: finished +[ INFO] Flush UD cache: started +[ INFO] UD cache flush is not enabled. +[ INFO] Flush UD cache: finished