Skip to content

Commit

Permalink
Draft: add --all-repos argument to deletion task
Browse files Browse the repository at this point in the history
  • Loading branch information
@amcmahon-rh
amcmahon-rh committed Oct 18, 2024
1 parent ef4cbcf commit 7e3dce5
Show file tree
Hide file tree
Showing 7 changed files with 417 additions and 22 deletions.
68 changes: 47 additions & 21 deletions src/pubtools/_pulp/tasks/delete.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,10 +78,18 @@ def add_args(self):

self.parser.add_argument(
"--repo",
help="remove content from these comma-seperated repositories ",
help="remove content from these comma-seperated repositories",
type=str,
action=SplitAndExtend,
split_on=",",
default=[]
)

self.parser.add_argument(
"--all-repos",
help="remove content from all repos it belongs to. "
"If --repo is used, this behaviour will not run.",
action="store_true"
)

self.parser.add_argument(
Expand Down Expand Up @@ -130,8 +138,13 @@ def run(self):
if not (self.args.file or self.args.advisory):
self.fail("One of --file or --advisory is required")

if self.args.file and not self.args.repo:
self.fail("Repository names in --repo is required")
if self.args.repo and self.args.all_repos:
LOG.warning("Both --repos and --all-repos have been used. "
"Defaulting to the repos specified with --repo.")
self.args.all_repos = False

if self.args.file and not (self.args.repo or self.args.all_repos):
self.fail("Repository names in --repo or --all-repos is required")

if not signing_keys and self.args.allow_unsigned:
signing_keys = [None]
Expand Down Expand Up @@ -564,26 +577,39 @@ def map_to_repo(self, units, repos, unit_attr):
repo_map = {}
unit_map = {}
repos = sorted(repos)

for unit in sorted(units):
unit_name = getattr(unit, unit_attr)
unit_map.setdefault(unit_name, RemoveUnitItem(unit=unit, repos=[]))
for repo in repos:
if repo not in unit.repository_memberships:
LOG.warning(
"%s is not present in %s",
unit_name,
repo,
)
else:
if repos:
for unit in sorted(units):
unit_name = getattr(unit, unit_attr)
unit_map.setdefault(unit_name, RemoveUnitItem(unit=unit, repos=[]))
for repo in repos:
if repo not in unit.repository_memberships:
LOG.warning(
"%s is not present in %s",
unit_name,
repo,
)
else:
repo_map.setdefault(repo, []).append(unit)
unit_map.get(unit_name).repos.append(repo)

missing = set(repos) - set(repo_map.keys())
if missing:
missing = ", ".join(sorted(list(missing)))
LOG.warning("No units to remove from %s", missing)
else:
LOG.info("Get repos from units")
LOG.info(list(units))
for unit in sorted(units):
unit_name = getattr(unit, unit_attr)
unit_map.setdefault(unit_name, RemoveUnitItem(unit=unit, repos=[]))
for repo in unit.repository_memberships:
LOG.info(repo)
if re.match("all-rpm-content-.*", repo):
continue
repo_map.setdefault(repo, []).append(unit)
unit_map.get(unit_name).repos.append(repo)

missing = set(repos) - set(repo_map.keys())
if missing:
missing = ", ".join(sorted(list(missing)))
LOG.warning("No units to remove from %s", missing)

LOG.info(repo_map)
LOG.info(unit_map)
return repo_map, unit_map

@step("Unassociate RPMs")
Expand Down
247 changes: 247 additions & 0 deletions tests/delete/test_delete_packages.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -801,3 +801,250 @@ def test_delete_rpms_skip_publish(command_tester, fake_collector, monkeypatch):
# All the files exist on Pulp
files_search = list(client.search_content(criteria1).result())
assert len(files_search) == 2


def test_delete_rpms_with_all_repos(command_tester, fake_collector, monkeypatch):
"""Deleting RPMs from repos succeeds"""

repo1 = YumRepository(
id="some-yumrepo", relative_url="some/publish/url", mutable_urls=["repomd.xml"]
)
repo2 = YumRepository(
id="other-yumrepo",
relative_url="other/publish/url",
mutable_urls=["repomd.xml"],
)
all_rpm_content_repo = YumRepository(
id="all-rpm-content-gg",
relative_url="other/publish/url",
mutable_urls=["repomd.xml"],
)

files1 = [
RpmUnit(
name="bash",
version="1.23",
release="1.test8",
arch="x86_64",
filename="bash-1.23-1.test8_x86_64.rpm",
sha256sum="a" * 64,
md5sum="b" * 32,
signing_key="aabbcc",
unit_id="file1_rpm1",
),
RpmUnit(
name="dash",
version="2.25",
release="1.test8",
arch="x86_64",
filename="dash-2.25-1.test8_x86_64.rpm",
sha256sum="a" * 64,
md5sum="b" * 32,
signing_key="aabbcc",
unit_id="file1_rpm2",
),
]

files2 = [
RpmUnit(
name="crash",
version="3.30",
release="1.test8",
arch="s390x",
filename="crash-3.30-1.test8_s390x.rpm",
sha256sum="a" * 64,
md5sum="b" * 32,
signing_key="aabbcc",
unit_id="file2_rpm1",
)
]


with FakeDeletePackages() as task_instance:
task_instance.pulp_client_controller.insert_repository(repo1)
task_instance.pulp_client_controller.insert_repository(repo2)
task_instance.pulp_client_controller.insert_repository(all_rpm_content_repo)
task_instance.pulp_client_controller.insert_units(repo1, files1)
task_instance.pulp_client_controller.insert_units(repo2, files2)
task_instance.pulp_client_controller.insert_units(repo2, [files1[0]])
task_instance.pulp_client_controller.insert_units(all_rpm_content_repo, files1)
task_instance.pulp_client_controller.insert_units(all_rpm_content_repo, files2)

# It should run with expected output.
command_tester.test(
task_instance.main,
[
"test-delete",
"--pulp-url",
"https://pulp.example.com/",
"--all-repos",
"--file",
"bash-1.23-1.test8_x86_64.rpm",
"--file",
"dash-2.25-1.test8_x86_64.rpm",
"--signing-key",
"aabbcc",
]
)
# It should record that it removed these push items:
assert sorted(fake_collector.items, key=lambda pi: pi["filename"]) == [
{
"origin": "pulp",
"src": None,
"dest": "other-yumrepo",
"signing_key": None,
"filename": "bash-1.23-1.test8.x86_64.rpm",
"state": "DELETED",
"build": None,
"checksums": {"sha256": "a" * 64},
},
{
"origin": "pulp",
"src": None,
"dest": "some-yumrepo",
"signing_key": None,
"filename": "bash-1.23-1.test8.x86_64.rpm",
"state": "DELETED",
"build": None,
"checksums": {"sha256": "a" * 64},
},
{
"origin": "pulp",
"src": None,
"dest": "some-yumrepo",
"signing_key": None,
"filename": "dash-2.25-1.test8.x86_64.rpm",
"state": "DELETED",
"build": None,
"checksums": {"sha256": "a" * 64},
},
]
#
# verify whether files were deleted on Pulp
client = task_instance.pulp_client

# get the repo where the files were deleted
repos = sorted(
list(
client.search_repository(
Criteria.with_id(["all-rpm-content-gg", "some-yumrepo", "other-yumrepo"])
).result()
),
key=lambda r: r.id,
)
assert len(repos) == 3
r_all_content, r2, r1 = repos

assert r_all_content.id == all_rpm_content_repo.id
assert r1.id == repo1.id
assert r2.id == repo2.id
#
# # criteria with the unit_ids
# # critera1 for files1 in repo1
unit_ids = []
for f in files1:
unit_ids.append(f.unit_id)

for f in files2:
unit_ids.append(f.unit_id)
search_criteria = Criteria.with_field("unit_id", Matcher.in_(unit_ids))

# No files should be in repo1
result1 =list(r1.search_content(search_criteria).result())
assert len(result1) == 0

result2 = list(r2.search_content(search_criteria).result())
assert len(result2) == 1
assert result2[0].unit_id == files2[0].unit_id

# --all-repos arg should skip over all-rpm-content repos
result3 = list(r_all_content.search_content(search_criteria).result())
assert len(result3) == 3


def test_delete_rpms_conflicting_repo_args(command_tester, fake_collector, monkeypatch):
"""If --repos and --all-repos are used, we should get a warning and should
default to using just --repos"""

repo1 = YumRepository(
id="some-yumrepo", relative_url="some/publish/url", mutable_urls=["repomd.xml"]
)
repo2 = YumRepository(
id="other-yumrepo",
relative_url="other/publish/url",
mutable_urls=["repomd.xml"],
)

rpm_file = RpmUnit(
name="bash",
version="1.23",
release="1.test8",
arch="x86_64",
filename="bash-1.23-1.test8_x86_64.rpm",
sha256sum="a" * 64,
md5sum="b" * 32,
signing_key="aabbcc",
unit_id="file1_rpm1",
)

with FakeDeletePackages() as task_instance:
task_instance.pulp_client_controller.insert_repository(repo1)
task_instance.pulp_client_controller.insert_repository(repo2)
task_instance.pulp_client_controller.insert_units(repo1, [rpm_file])
task_instance.pulp_client_controller.insert_units(repo2, [rpm_file])

command_tester.test(
task_instance.main,
[
"test-delete",
"--pulp-url",
"https://pulp.example.com/",
"--repo",
"some-yumrepo",
"--all-repos",
"--file",
"bash-1.23-1.test8_x86_64.rpm",
"--signing-key",
"aabbcc",
],
)

# It should record that it removed these push items:
assert sorted(fake_collector.items, key=lambda pi: pi["filename"]) == [
{
"origin": "pulp",
"src": None,
"dest": "some-yumrepo",
"signing_key": None,
"filename": "bash-1.23-1.test8.x86_64.rpm",
"state": "DELETED",
"build": None,
"checksums": {"sha256": "a" * 64},
},
]

# verify whether files were deleted on Pulp
client = task_instance.pulp_client

# get the repo where the files were deleted
repos = sorted(
list(
client.search_repository(
Criteria.with_id(["some-yumrepo", "other-yumrepo"])
).result()
),
key=lambda r: r.id,
)
assert len(repos) == 2
r2, r1 = repos

criteria1 = Criteria.with_field("unit_id", Matcher.in_([rpm_file.unit_id]))

# Files were removed as if --all-repos wasn't present.
result1 = list(r1.search_content(criteria1).result())
assert len(result1) == 0

result2 = list(r2.search_content(criteria1).result())
assert len(result2) == 1
assert result2[0].unit_id == rpm_file.unit_id

22 changes: 22 additions & 0 deletions tests/logs/delete/test_delete_packages/test_delete_rpms_conflicting_repo_args.jsonl
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
{"event": {"type": "delete-files-start"}}
{"event": {"type": "get-files-start"}}
{"event": {"type": "get-files-end"}}
{"event": {"type": "unassociate-files-start"}}
{"event": {"type": "unassociate-files-end"}}
{"event": {"type": "record-push-items-start"}}
{"event": {"type": "record-push-items-end"}}
{"event": {"type": "delete-files-end"}}
{"event": {"type": "delete-rpms-start"}}
{"event": {"type": "get-rpms-start"}}
{"event": {"type": "get-rpms-end"}}
{"event": {"type": "unassociate-rpms-start"}}
{"event": {"type": "unassociate-rpms-end"}}
{"event": {"type": "record-push-items-start"}}
{"event": {"type": "record-push-items-end"}}
{"event": {"type": "delete-rpms-end"}}
{"event": {"type": "publish-start"}}
{"event": {"type": "publish-end"}}
{"event": {"type": "set-cdn_published-start"}}
{"event": {"type": "set-cdn_published-end"}}
{"event": {"type": "flush-ud-cache-start"}}
{"event": {"type": "flush-ud-cache-end"}}
Loading

0 comments on commit 7e3dce5

Please sign in to comment.