diff --git a/src/pubtools/_pulp/tasks/delete.py b/src/pubtools/_pulp/tasks/delete.py index e2c02369..d5bf8faf 100644 --- a/src/pubtools/_pulp/tasks/delete.py +++ b/src/pubtools/_pulp/tasks/delete.py @@ -78,10 +78,18 @@ def add_args(self): self.parser.add_argument( "--repo", - help="remove content from these comma-seperated repositories ", + help="remove content from these comma-seperated repositories", type=str, action=SplitAndExtend, split_on=",", + default=[] + ) + + self.parser.add_argument( + "--all-repos", + help="remove content from all repos it belongs to. " + "If --repo is used, this behaviour will not run.", + action="store_true" ) self.parser.add_argument( @@ -130,8 +138,13 @@ def run(self): if not (self.args.file or self.args.advisory): self.fail("One of --file or --advisory is required") - if self.args.file and not self.args.repo: - self.fail("Repository names in --repo is required") + if self.args.repo and self.args.all_repos: + LOG.warning("Both --repos and --all-repos have been used. " + "Defaulting to the repos specified with --repo.") + self.args.all_repos = False + + if self.args.file and not (self.args.repo or self.args.all_repos): + self.fail("Repository names in --repo or --all-repos is required") if not signing_keys and self.args.allow_unsigned: signing_keys = [None] @@ -564,26 +577,39 @@ def map_to_repo(self, units, repos, unit_attr): repo_map = {} unit_map = {} repos = sorted(repos) - - for unit in sorted(units): - unit_name = getattr(unit, unit_attr) - unit_map.setdefault(unit_name, RemoveUnitItem(unit=unit, repos=[])) - for repo in repos: - if repo not in unit.repository_memberships: - LOG.warning( - "%s is not present in %s", - unit_name, - repo, - ) - else: + if repos: + for unit in sorted(units): + unit_name = getattr(unit, unit_attr) + unit_map.setdefault(unit_name, RemoveUnitItem(unit=unit, repos=[])) + for repo in repos: + if repo not in unit.repository_memberships: + LOG.warning( + "%s is not present in %s", + unit_name, + repo, + ) + else: + repo_map.setdefault(repo, []).append(unit) + unit_map.get(unit_name).repos.append(repo) + + missing = set(repos) - set(repo_map.keys()) + if missing: + missing = ", ".join(sorted(list(missing))) + LOG.warning("No units to remove from %s", missing) + else: + LOG.info("Get repos from units") + LOG.info(list(units)) + for unit in sorted(units): + unit_name = getattr(unit, unit_attr) + unit_map.setdefault(unit_name, RemoveUnitItem(unit=unit, repos=[])) + for repo in unit.repository_memberships: + LOG.info(repo) + if re.match("all-rpm-content-.*", repo): + continue repo_map.setdefault(repo, []).append(unit) unit_map.get(unit_name).repos.append(repo) - - missing = set(repos) - set(repo_map.keys()) - if missing: - missing = ", ".join(sorted(list(missing))) - LOG.warning("No units to remove from %s", missing) - + LOG.info(repo_map) + LOG.info(unit_map) return repo_map, unit_map @step("Unassociate RPMs") diff --git a/tests/delete/test_delete_packages.py b/tests/delete/test_delete_packages.py index a6091b72..4e7e225b 100644 --- a/tests/delete/test_delete_packages.py +++ b/tests/delete/test_delete_packages.py @@ -801,3 +801,250 @@ def test_delete_rpms_skip_publish(command_tester, fake_collector, monkeypatch): # All the files exist on Pulp files_search = list(client.search_content(criteria1).result()) assert len(files_search) == 2 + + +def test_delete_rpms_with_all_repos(command_tester, fake_collector, monkeypatch): + """Deleting RPMs from repos succeeds""" + + repo1 = YumRepository( + id="some-yumrepo", relative_url="some/publish/url", mutable_urls=["repomd.xml"] + ) + repo2 = YumRepository( + id="other-yumrepo", + relative_url="other/publish/url", + mutable_urls=["repomd.xml"], + ) + all_rpm_content_repo = YumRepository( + id="all-rpm-content-gg", + relative_url="other/publish/url", + mutable_urls=["repomd.xml"], + ) + + files1 = [ + RpmUnit( + name="bash", + version="1.23", + release="1.test8", + arch="x86_64", + filename="bash-1.23-1.test8_x86_64.rpm", + sha256sum="a" * 64, + md5sum="b" * 32, + signing_key="aabbcc", + unit_id="file1_rpm1", + ), + RpmUnit( + name="dash", + version="2.25", + release="1.test8", + arch="x86_64", + filename="dash-2.25-1.test8_x86_64.rpm", + sha256sum="a" * 64, + md5sum="b" * 32, + signing_key="aabbcc", + unit_id="file1_rpm2", + ), + ] + + files2 = [ + RpmUnit( + name="crash", + version="3.30", + release="1.test8", + arch="s390x", + filename="crash-3.30-1.test8_s390x.rpm", + sha256sum="a" * 64, + md5sum="b" * 32, + signing_key="aabbcc", + unit_id="file2_rpm1", + ) + ] + + + with FakeDeletePackages() as task_instance: + task_instance.pulp_client_controller.insert_repository(repo1) + task_instance.pulp_client_controller.insert_repository(repo2) + task_instance.pulp_client_controller.insert_repository(all_rpm_content_repo) + task_instance.pulp_client_controller.insert_units(repo1, files1) + task_instance.pulp_client_controller.insert_units(repo2, files2) + task_instance.pulp_client_controller.insert_units(repo2, [files1[0]]) + task_instance.pulp_client_controller.insert_units(all_rpm_content_repo, files1) + task_instance.pulp_client_controller.insert_units(all_rpm_content_repo, files2) + + # It should run with expected output. + command_tester.test( + task_instance.main, + [ + "test-delete", + "--pulp-url", + "https://pulp.example.com/", + "--all-repos", + "--file", + "bash-1.23-1.test8_x86_64.rpm", + "--file", + "dash-2.25-1.test8_x86_64.rpm", + "--signing-key", + "aabbcc", + ] + ) + # It should record that it removed these push items: + assert sorted(fake_collector.items, key=lambda pi: pi["filename"]) == [ + { + "origin": "pulp", + "src": None, + "dest": "other-yumrepo", + "signing_key": None, + "filename": "bash-1.23-1.test8.x86_64.rpm", + "state": "DELETED", + "build": None, + "checksums": {"sha256": "a" * 64}, + }, + { + "origin": "pulp", + "src": None, + "dest": "some-yumrepo", + "signing_key": None, + "filename": "bash-1.23-1.test8.x86_64.rpm", + "state": "DELETED", + "build": None, + "checksums": {"sha256": "a" * 64}, + }, + { + "origin": "pulp", + "src": None, + "dest": "some-yumrepo", + "signing_key": None, + "filename": "dash-2.25-1.test8.x86_64.rpm", + "state": "DELETED", + "build": None, + "checksums": {"sha256": "a" * 64}, + }, + ] + # + # verify whether files were deleted on Pulp + client = task_instance.pulp_client + + # get the repo where the files were deleted + repos = sorted( + list( + client.search_repository( + Criteria.with_id(["all-rpm-content-gg", "some-yumrepo", "other-yumrepo"]) + ).result() + ), + key=lambda r: r.id, + ) + assert len(repos) == 3 + r_all_content, r2, r1 = repos + + assert r_all_content.id == all_rpm_content_repo.id + assert r1.id == repo1.id + assert r2.id == repo2.id + # + # # criteria with the unit_ids + # # critera1 for files1 in repo1 + unit_ids = [] + for f in files1: + unit_ids.append(f.unit_id) + + for f in files2: + unit_ids.append(f.unit_id) + search_criteria = Criteria.with_field("unit_id", Matcher.in_(unit_ids)) + + # No files should be in repo1 + result1 =list(r1.search_content(search_criteria).result()) + assert len(result1) == 0 + + result2 = list(r2.search_content(search_criteria).result()) + assert len(result2) == 1 + assert result2[0].unit_id == files2[0].unit_id + + # --all-repos arg should skip over all-rpm-content repos + result3 = list(r_all_content.search_content(search_criteria).result()) + assert len(result3) == 3 + + +def test_delete_rpms_conflicting_repo_args(command_tester, fake_collector, monkeypatch): + """If --repos and --all-repos are used, we should get a warning and should + default to using just --repos""" + + repo1 = YumRepository( + id="some-yumrepo", relative_url="some/publish/url", mutable_urls=["repomd.xml"] + ) + repo2 = YumRepository( + id="other-yumrepo", + relative_url="other/publish/url", + mutable_urls=["repomd.xml"], + ) + + rpm_file = RpmUnit( + name="bash", + version="1.23", + release="1.test8", + arch="x86_64", + filename="bash-1.23-1.test8_x86_64.rpm", + sha256sum="a" * 64, + md5sum="b" * 32, + signing_key="aabbcc", + unit_id="file1_rpm1", + ) + + with FakeDeletePackages() as task_instance: + task_instance.pulp_client_controller.insert_repository(repo1) + task_instance.pulp_client_controller.insert_repository(repo2) + task_instance.pulp_client_controller.insert_units(repo1, [rpm_file]) + task_instance.pulp_client_controller.insert_units(repo2, [rpm_file]) + + command_tester.test( + task_instance.main, + [ + "test-delete", + "--pulp-url", + "https://pulp.example.com/", + "--repo", + "some-yumrepo", + "--all-repos", + "--file", + "bash-1.23-1.test8_x86_64.rpm", + "--signing-key", + "aabbcc", + ], + ) + + # It should record that it removed these push items: + assert sorted(fake_collector.items, key=lambda pi: pi["filename"]) == [ + { + "origin": "pulp", + "src": None, + "dest": "some-yumrepo", + "signing_key": None, + "filename": "bash-1.23-1.test8.x86_64.rpm", + "state": "DELETED", + "build": None, + "checksums": {"sha256": "a" * 64}, + }, + ] + + # verify whether files were deleted on Pulp + client = task_instance.pulp_client + + # get the repo where the files were deleted + repos = sorted( + list( + client.search_repository( + Criteria.with_id(["some-yumrepo", "other-yumrepo"]) + ).result() + ), + key=lambda r: r.id, + ) + assert len(repos) == 2 + r2, r1 = repos + + criteria1 = Criteria.with_field("unit_id", Matcher.in_([rpm_file.unit_id])) + + # Files were removed as if --all-repos wasn't present. + result1 = list(r1.search_content(criteria1).result()) + assert len(result1) == 0 + + result2 = list(r2.search_content(criteria1).result()) + assert len(result2) == 1 + assert result2[0].unit_id == rpm_file.unit_id + diff --git a/tests/logs/delete/test_delete_packages/test_delete_rpms_conflicting_repo_args.jsonl b/tests/logs/delete/test_delete_packages/test_delete_rpms_conflicting_repo_args.jsonl new file mode 100644 index 00000000..1f9123c9 --- /dev/null +++ b/tests/logs/delete/test_delete_packages/test_delete_rpms_conflicting_repo_args.jsonl @@ -0,0 +1,22 @@ +{"event": {"type": "delete-files-start"}} +{"event": {"type": "get-files-start"}} +{"event": {"type": "get-files-end"}} +{"event": {"type": "unassociate-files-start"}} +{"event": {"type": "unassociate-files-end"}} +{"event": {"type": "record-push-items-start"}} +{"event": {"type": "record-push-items-end"}} +{"event": {"type": "delete-files-end"}} +{"event": {"type": "delete-rpms-start"}} +{"event": {"type": "get-rpms-start"}} +{"event": {"type": "get-rpms-end"}} +{"event": {"type": "unassociate-rpms-start"}} +{"event": {"type": "unassociate-rpms-end"}} +{"event": {"type": "record-push-items-start"}} +{"event": {"type": "record-push-items-end"}} +{"event": {"type": "delete-rpms-end"}} +{"event": {"type": "publish-start"}} +{"event": {"type": "publish-end"}} +{"event": {"type": "set-cdn_published-start"}} +{"event": {"type": "set-cdn_published-end"}} +{"event": {"type": "flush-ud-cache-start"}} +{"event": {"type": "flush-ud-cache-end"}} diff --git a/tests/logs/delete/test_delete_packages/test_delete_rpms_conflicting_repo_args.txt b/tests/logs/delete/test_delete_packages/test_delete_rpms_conflicting_repo_args.txt new file mode 100644 index 00000000..8802ed22 --- /dev/null +++ b/tests/logs/delete/test_delete_packages/test_delete_rpms_conflicting_repo_args.txt @@ -0,0 +1,32 @@ +[ WARNING] Both --repos and --all-repos have been used. Defaulting to the repos specified with --repo. +[ INFO] Delete files: started +[ INFO] Get files: started +[ WARNING] Requested unit(s) don't exist as file: bash-1.23-1.test8_x86_64.rpm +[ INFO] 0 unit(s) found for deletion +[ INFO] Get files: finished +[ WARNING] No units to remove from some-yumrepo +[ INFO] Unassociate files: started +[ WARNING] Nothing mapped for removal +[ INFO] Unassociate files: finished +[ INFO] Record push items: started +[ INFO] Record push items: finished +[ INFO] Delete files: finished +[ INFO] Delete RPMs: started +[ INFO] Get RPMs: started +[ INFO] 1 unit(s) found for deletion +[ INFO] Get RPMs: finished +[ INFO] Deleting bash-1.23-1.test8_x86_64.rpm from some-yumrepo +[ INFO] Unassociate RPMs: started +[ INFO] some-yumrepo: removed 1 rpm(s), tasks: e3e70682-c209-4cac-629f-6fbed82c07cd +[ INFO] Unassociate RPMs: finished +[ INFO] Record push items: started +[ INFO] Record push items: finished +[ INFO] Delete RPMs: finished +[ INFO] Publish: started +[ INFO] Publishing some-yumrepo +[ INFO] Publish: finished +[ INFO] Set cdn_published: started +[ INFO] Set cdn_published: finished +[ INFO] Flush UD cache: started +[ INFO] UD cache flush is not enabled. +[ INFO] Flush UD cache: finished diff --git a/tests/logs/delete/test_delete_packages/test_delete_rpms_with_all_repos.jsonl b/tests/logs/delete/test_delete_packages/test_delete_rpms_with_all_repos.jsonl new file mode 100644 index 00000000..1f9123c9 --- /dev/null +++ b/tests/logs/delete/test_delete_packages/test_delete_rpms_with_all_repos.jsonl @@ -0,0 +1,22 @@ +{"event": {"type": "delete-files-start"}} +{"event": {"type": "get-files-start"}} +{"event": {"type": "get-files-end"}} +{"event": {"type": "unassociate-files-start"}} +{"event": {"type": "unassociate-files-end"}} +{"event": {"type": "record-push-items-start"}} +{"event": {"type": "record-push-items-end"}} +{"event": {"type": "delete-files-end"}} +{"event": {"type": "delete-rpms-start"}} +{"event": {"type": "get-rpms-start"}} +{"event": {"type": "get-rpms-end"}} +{"event": {"type": "unassociate-rpms-start"}} +{"event": {"type": "unassociate-rpms-end"}} +{"event": {"type": "record-push-items-start"}} +{"event": {"type": "record-push-items-end"}} +{"event": {"type": "delete-rpms-end"}} +{"event": {"type": "publish-start"}} +{"event": {"type": "publish-end"}} +{"event": {"type": "set-cdn_published-start"}} +{"event": {"type": "set-cdn_published-end"}} +{"event": {"type": "flush-ud-cache-start"}} +{"event": {"type": "flush-ud-cache-end"}} diff --git a/tests/logs/delete/test_delete_packages/test_delete_rpms_with_all_repos.txt b/tests/logs/delete/test_delete_packages/test_delete_rpms_with_all_repos.txt new file mode 100644 index 00000000..d3eec657 --- /dev/null +++ b/tests/logs/delete/test_delete_packages/test_delete_rpms_with_all_repos.txt @@ -0,0 +1,46 @@ +[ INFO] Delete files: started +[ INFO] Get files: started +[ WARNING] Requested unit(s) don't exist as file: bash-1.23-1.test8_x86_64.rpm, dash-2.25-1.test8_x86_64.rpm +[ INFO] 0 unit(s) found for deletion +[ INFO] Get files: finished +[ INFO] Get repos from units +[ INFO] [] +[ INFO] {} +[ INFO] {} +[ INFO] Unassociate files: started +[ WARNING] Nothing mapped for removal +[ INFO] Unassociate files: finished +[ INFO] Record push items: started +[ INFO] Record push items: finished +[ INFO] Delete files: finished +[ INFO] Delete RPMs: started +[ INFO] Get RPMs: started +[ INFO] 2 unit(s) found for deletion +[ INFO] Get RPMs: finished +[ INFO] Get repos from units +[ INFO] [RpmUnit(name='dash', version='2.25', release='1.test8', arch='x86_64', signing_key='aabbcc', filename='dash-2.25-1.test8_x86_64.rpm', md5sum='bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', sha256sum='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', repository_memberships=['all-rpm-content-gg', 'some-yumrepo'], unit_id='file1_rpm2'), RpmUnit(name='bash', version='1.23', release='1.test8', arch='x86_64', signing_key='aabbcc', filename='bash-1.23-1.test8_x86_64.rpm', md5sum='bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', sha256sum='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', repository_memberships=['all-rpm-content-gg', 'other-yumrepo', 'some-yumrepo'], unit_id='file1_rpm1')] +[ INFO] all-rpm-content-gg +[ INFO] other-yumrepo +[ INFO] some-yumrepo +[ INFO] all-rpm-content-gg +[ INFO] some-yumrepo +[ INFO] {'other-yumrepo': [RpmUnit(name='bash', version='1.23', release='1.test8', arch='x86_64', signing_key='aabbcc', filename='bash-1.23-1.test8_x86_64.rpm', md5sum='bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', sha256sum='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', repository_memberships=['all-rpm-content-gg', 'other-yumrepo', 'some-yumrepo'], unit_id='file1_rpm1')], 'some-yumrepo': [RpmUnit(name='bash', version='1.23', release='1.test8', arch='x86_64', signing_key='aabbcc', filename='bash-1.23-1.test8_x86_64.rpm', md5sum='bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', sha256sum='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', repository_memberships=['all-rpm-content-gg', 'other-yumrepo', 'some-yumrepo'], unit_id='file1_rpm1'), RpmUnit(name='dash', version='2.25', release='1.test8', arch='x86_64', signing_key='aabbcc', filename='dash-2.25-1.test8_x86_64.rpm', md5sum='bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', sha256sum='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', repository_memberships=['all-rpm-content-gg', 'some-yumrepo'], unit_id='file1_rpm2')]} +[ INFO] {'bash-1.23-1.test8_x86_64.rpm': RemoveUnitItem(unit=RpmUnit(name='bash', version='1.23', release='1.test8', arch='x86_64', signing_key='aabbcc', filename='bash-1.23-1.test8_x86_64.rpm', md5sum='bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', sha256sum='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', repository_memberships=['all-rpm-content-gg', 'other-yumrepo', 'some-yumrepo'], unit_id='file1_rpm1'), repos=['other-yumrepo', 'some-yumrepo']), 'dash-2.25-1.test8_x86_64.rpm': RemoveUnitItem(unit=RpmUnit(name='dash', version='2.25', release='1.test8', arch='x86_64', signing_key='aabbcc', filename='dash-2.25-1.test8_x86_64.rpm', md5sum='bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', sha256sum='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', repository_memberships=['all-rpm-content-gg', 'some-yumrepo'], unit_id='file1_rpm2'), repos=['some-yumrepo'])} +[ INFO] Deleting bash-1.23-1.test8_x86_64.rpm from other-yumrepo, some-yumrepo +[ INFO] Deleting dash-2.25-1.test8_x86_64.rpm from some-yumrepo +[ INFO] Unassociate RPMs: started +[ INFO] other-yumrepo: removed 1 rpm(s), tasks: e3e70682-c209-4cac-629f-6fbed82c07cd +[ INFO] some-yumrepo: removed 2 rpm(s), tasks: 82e2e662-f728-b4fa-4248-5e3a0a5d2f34 +[ INFO] Unassociate RPMs: finished +[ INFO] Record push items: started +[ INFO] Record push items: finished +[ INFO] Delete RPMs: finished +[ INFO] Publish: started +[ INFO] Publishing other-yumrepo +[ INFO] Publishing some-yumrepo +[ INFO] Publish: finished +[ INFO] Set cdn_published: started +[ INFO] Set cdn_published: finished +[ INFO] Flush UD cache: started +[ INFO] UD cache flush is not enabled. +[ INFO] Flush UD cache: finished diff --git a/tests/logs/delete/test_delete_packages/test_no_repo_provided.txt b/tests/logs/delete/test_delete_packages/test_no_repo_provided.txt index bea5a620..838c4225 100644 --- a/tests/logs/delete/test_delete_packages/test_no_repo_provided.txt +++ b/tests/logs/delete/test_delete_packages/test_no_repo_provided.txt @@ -1,2 +1,2 @@ -[ ERROR] Repository names in --repo is required +[ ERROR] Repository names in --repo or --all-repos is required # Raised: 30