From 4f579513375de2e170a0b7baf27dcc310e60d959 Mon Sep 17 00:00:00 2001 From: Nathan Gillett Date: Tue, 12 Mar 2024 02:55:48 -0500 Subject: [PATCH] clean-up: remove dead code, format touched files [RHELDST-141] --- tests/test_ubipop.py | 414 ++++++++++++++++++++++++++++++++++++------- ubipop/__init__.py | 232 +++++++++--------------- 2 files changed, 433 insertions(+), 213 deletions(-) diff --git a/tests/test_ubipop.py b/tests/test_ubipop.py index 65bdf48..beba31a 100644 --- a/tests/test_ubipop.py +++ b/tests/test_ubipop.py @@ -24,7 +24,6 @@ ) from ubipop import ( - ConfigMissing, RepoContent, RepoMissing, RepoSet, @@ -137,14 +136,12 @@ def fixture_executor(): @pytest.fixture(name="mock_ubipop_runner") -def fixture_mock_ubipop_runner(ubi_repo_set, test_ubiconfig, executor): +def fixture_mock_ubipop_runner(ubi_repo_set, executor): publisher_args = {"publish_options": {"clean": True}} yield UbiPopulateRunner( - legacy_client=MagicMock(), # legacy client - pulp_client=MagicMock(), # pubtools-pulplib client - output_repo_set=ubi_repo_set, - ubiconfig_item=test_ubiconfig, + pulp_client=MagicMock(), + repo_set=ubi_repo_set, dry_run=False, executor=executor, ubi_manifest_client=MagicMock(), @@ -168,51 +165,6 @@ def get_test_repo(**kwargs): ) -def test_get_output_repos(ubi_repo_set): - repos = ubi_repo_set.get_output_repos() - assert set([repo.id for repo in repos]) == set( - ["ubi-foo-rpms", "ubi-foo-source", "ubi-foo-debug"] - ) - - -def test_get_output_repos_no_debug(ubi_repo_set_no_debug): - repos = ubi_repo_set_no_debug.get_output_repos() - assert set([repo.id for repo in repos]) == set(["ubi-foo-rpms", "ubi-foo-source"]) - - -def test_raise_config_missing(caplog): - config_path = os.path.join(TEST_DATA_DIR, "ubi8") - ubipopulate = UbiPopulate( - "foo.pulp.com", ("foo", "foo"), False, ubiconfig_dir_or_url=config_path - ) - repo = YumRepository( - id="repo", - ubi_config_version="9.8", - ) - for config in ubipopulate.ubiconfig_list: - with pytest.raises(ConfigMissing): - ubipopulate._get_config(repo, config) - - assert ( - "Config file ubiconf_golang.yaml missing from 9.8 and default 9 branches" - in caplog.text - ) - - -def test_raise_error_for_missing_ubi_config_version(): - config_path = os.path.join(TEST_DATA_DIR, "ubi8") - ubipopulate = UbiPopulate( - "foo.pulp.com", ("foo", "foo"), False, ubiconfig_dir_or_url=config_path - ) - repo = YumRepository( - id="repo", - ubi_config_version="", - ) - for config in ubipopulate.ubiconfig_list: - with pytest.raises(ValueError): - ubipopulate._get_config(repo, config) - - def test_publish_out_repos(mock_ubipop_runner): dt = datetime(2019, 9, 12, 0, 0, 0) @@ -234,7 +186,7 @@ def test_publish_out_repos(mock_ubipop_runner): fake_pulp.insert_repository(repo) # Setup output repos, leave only binary repo as the actual one - mock_ubipop_runner.repos.out_repos = RepoSet( + mock_ubipop_runner.repo_set.out_repos = RepoSet( f_proxy(f_return(repo)), f_proxy(f_return()), f_proxy(f_return()) ) @@ -802,11 +754,11 @@ def test_get_pulp_actions(mock_ubipop_runner, mock_current_content): src_repo_id="foo-rpms", ) - mock_ubipop_runner.repos.packages = f_proxy(f_return([binary_rpm])) - mock_ubipop_runner.repos.debug_rpms = f_proxy(f_return([debug_rpm])) - mock_ubipop_runner.repos.source_rpms = f_proxy(f_return([source_rpm])) - mock_ubipop_runner.repos.modules = f_proxy(f_return([modulemd])) - mock_ubipop_runner.repos.module_defaults = f_proxy(f_return([modulemd_defaults])) + mock_ubipop_runner.repo_set.packages = f_proxy(f_return([binary_rpm])) + mock_ubipop_runner.repo_set.debug_rpms = f_proxy(f_return([debug_rpm])) + mock_ubipop_runner.repo_set.source_rpms = f_proxy(f_return([source_rpm])) + mock_ubipop_runner.repo_set.modules = f_proxy(f_return([modulemd])) + mock_ubipop_runner.repo_set.module_defaults = f_proxy(f_return([modulemd_defaults])) # pylint: disable=protected-access ( @@ -915,11 +867,11 @@ def test_get_pulp_actions_no_actions(mock_ubipop_runner, mock_current_content): src_repo_id="foo-rpms", ) - mock_ubipop_runner.repos.packages = f_proxy(f_return([binary_rpm])) - mock_ubipop_runner.repos.debug_rpms = f_proxy(f_return([debug_rpm])) - mock_ubipop_runner.repos.source_rpms = f_proxy(f_return([source_rpm])) - mock_ubipop_runner.repos.modules = f_proxy(f_return([modulemd])) - mock_ubipop_runner.repos.module_defaults = f_proxy(f_return([modulemd_defaults])) + mock_ubipop_runner.repo_set.packages = f_proxy(f_return([binary_rpm])) + mock_ubipop_runner.repo_set.debug_rpms = f_proxy(f_return([debug_rpm])) + mock_ubipop_runner.repo_set.source_rpms = f_proxy(f_return([source_rpm])) + mock_ubipop_runner.repo_set.modules = f_proxy(f_return([modulemd])) + mock_ubipop_runner.repo_set.module_defaults = f_proxy(f_return([modulemd_defaults])) # pylint: disable=protected-access ( @@ -1033,7 +985,7 @@ def test_associate_units(mock_ubipop_runner): AssociateActionModules([unit], dst_repo, [src_repo]), ] - mock_ubipop_runner.pulp.associate_modules.return_value = ["task_id"] + mock_ubipop_runner.pulp_client.associate_modules.return_value = ["task_id"] ret = mock_ubipop_runner._associate_unassociate_units( associations ) # pylint: disable=protected-access @@ -1072,8 +1024,12 @@ def test_associate_unassociate_md_defaults(mock_ubipop_runner): dst_repo, ) - mock_ubipop_runner.pulp.unassociate_module_defaults.return_value = ["task_id_0"] - mock_ubipop_runner.pulp.associate_module_defaults.return_value = ["task_id_1"] + mock_ubipop_runner.pulp_client.unassociate_module_defaults.return_value = [ + "task_id_0" + ] + mock_ubipop_runner.pulp_client.associate_module_defaults.return_value = [ + "task_id_1" + ] # pylint: disable=protected-access mock_ubipop_runner._associate_unassociate_md_defaults( @@ -1083,7 +1039,7 @@ def test_associate_unassociate_md_defaults(mock_ubipop_runner): # the calls has to be in order calls = [call(["task_id_0"]), call(["task_id_1"])] - mock_ubipop_runner.pulp.wait_for_tasks.assert_has_calls(calls) + mock_ubipop_runner.pulp_client.wait_for_tasks.assert_has_calls(calls) @pytest.mark.parametrize( @@ -1144,7 +1100,7 @@ def test_get_current_content(mock_ubipop_runner, pulp, skip_debug_repo): debug_repo = f_proxy(f_return(debug_repo)) # overwrite out_repos with the testing ones - mock_ubipop_runner.repos.out_repos = RepoSet( + mock_ubipop_runner.repo_set.out_repos = RepoSet( f_proxy(f_return(rpm_repo)), f_proxy(f_return(source_repo)), debug_repo ) @@ -1573,6 +1529,330 @@ def test_populate_ubi_repos_no_publish( ) +@patch("ubipop._cdn.Publisher") +@patch("pubtools.pulplib.YumRepository.get_source_repository") +@patch("pubtools.pulplib.YumRepository.get_debug_repository") +def test_populate_ubi_repos_dry_run( + get_debug_repository, + get_source_repository, + publisher, + requests_mock, + monkeypatch, + caplog, +): + """Test run of populate_ubi_repos with dry_run enabled.""" + monkeypatch.setenv("UBIPOP_SKIP_PUBLISH", "true") + + dt = datetime(2019, 9, 12, 0, 0, 0) + + d1 = Distributor( + id="yum_distributor", + type_id="yum_distributor", + repo_id="ubi_binary", + last_publish=dt, + relative_url="content/unit/2/client", + ) + + d2 = Distributor( + id="yum_distributor", + type_id="yum_distributor", + repo_id="ubi_source", + last_publish=dt, + relative_url="content/unit/3/client", + ) + + d3 = Distributor( + id="yum_distributor", + type_id="yum_distributor", + repo_id="ubi_debug", + last_publish=dt, + relative_url="content/unit/4/client", + ) + + output_binary_repo = YumRepository( + id="ubi_binary", + content_set="ubi-8-for-x86_64-appstream-rpms", + population_sources=["input_binary"], + ubi_population=True, + ubi_config_version="8", + eng_product_id=102, + distributors=[d1], + relative_url="content/unit/1/client", + ) + input_binary_repo = YumRepository(id="input_binary") + input_source_repo = YumRepository(id="input_source") + input_debug_repo = YumRepository(id="input_debug") + + output_source_repo = YumRepository( + id="ubi_source", + population_sources=["input_source"], + eng_product_id=102, + distributors=[d2], + relative_url="content/unit/2/client", + ) + output_debug_repo = YumRepository( + id="ubi_debug", + population_sources=["input_debug"], + eng_product_id=102, + distributors=[d3], + relative_url="content/unit/3/client", + ) + + ubi_populate = FakeUbiPopulate( + "foo.pulp.com", + ("foo", "foo"), + dry_run=True, + ubiconfig_dir_or_url=TEST_DATA_DIR, + ubi_manifest_url="https://ubi-manifest.com", + publisher=publisher, + ) + + fake_pulp = ubi_populate.pulp_client_controller + fake_pulp.insert_repository(input_binary_repo) + fake_pulp.insert_repository(input_source_repo) + fake_pulp.insert_repository(input_debug_repo) + + fake_pulp.insert_repository(output_binary_repo) + fake_pulp.insert_repository(output_source_repo) + fake_pulp.insert_repository(output_debug_repo) + + get_debug_repository.return_value = fake_pulp.client.get_repository("ubi_debug") + get_source_repository.return_value = fake_pulp.client.get_repository("ubi_source") + + old_rpm = RpmUnit( + name="golang", + version="1", + release="a", + arch="x86_64", + filename="golang-1.a.x86_64.rpm", + sourcerpm="golang-1.a.x86_64.src.rpm", + ) + + new_rpm = RpmUnit( + name="golang", + version="2", + release="a", + arch="x86_64", + filename="golang-2.a.x86_64.rpm", + sourcerpm="golang-2.a.x86_64.src.rpm", + ) + + old_modulemd = ModulemdUnit( + name="test_md", stream="s", version="100", context="c", arch="x86_64" + ) + new_modulemd = ModulemdUnit( + name="test_md", stream="s", version="200", context="c", arch="x86_64" + ) + + old_modulemd_defaults = ModulemdDefaultsUnit( + name="test_md_defaults", + stream="stream", + profiles={"minimal": ["name_1"]}, + repo_id="ubi_binary", + ) + new_modulemd_defaults = ModulemdDefaultsUnit( + name="test_md_defaults", + stream="stream", + profiles={"minimal": ["name_1", "name_2"]}, + repo_id="input_binary", + ) + + fake_pulp.insert_units( + output_binary_repo, [old_rpm, old_modulemd, old_modulemd_defaults] + ) + fake_pulp.insert_units( + input_binary_repo, [new_rpm, new_modulemd, new_modulemd_defaults] + ) + + url = "/pulp/api/v2/repositories/{dst_repo}/actions/associate/".format( + dst_repo="ubi_binary" + ) + + requests_mock.register_uri( + "POST", url, json={"spawned_tasks": [{"task_id": "foo_task_id"}]} + ) + + url = "/pulp/api/v2/repositories/{dst_repo}/actions/unassociate/".format( + dst_repo="ubi_binary" + ) + requests_mock.register_uri( + "POST", url, json={"spawned_tasks": [{"task_id": "foo_task_id"}]} + ) + + url = "/pulp/api/v2/tasks/{task_id}/".format(task_id="foo_task_id") + requests_mock.register_uri( + "GET", url, json={"state": "finished", "task_id": "foo_task_id"} + ) + # mock calls to ubi-manifest service + _create_ubi_manifest_mocks(requests_mock) + _create_fastpurge_mocks(requests_mock) + _create_cdn_mocks(requests_mock) + + # let's run actual population + ubi_populate.populate_ubi_repos() + + # should not have published + publisher.assert_not_called() + + # should have logged content and actions + for msg in ( + "Current rpms in repo:", + "Current srpms in repo:", + "Current module_defaults in repo:", + "Current modules in repo:", + "Would associate", + "Would unassociate", + ): + assert msg in caplog.text + + +@patch("pubtools.pulplib.YumRepository.get_source_repository") +def test_populate_ubi_repos_missing_repos( + get_source_repository, cache_purge_env, requests_mock, caplog +): + # pylint: disable=unused-argument + """Test run of populate_ubi_repos with a missing repo.""" + dt = datetime(2019, 9, 12, 0, 0, 0) + + d1 = Distributor( + id="yum_distributor", + type_id="yum_distributor", + repo_id="ubi_binary", + last_publish=dt, + relative_url="content/unit/2/client", + ) + + d2 = Distributor( + id="yum_distributor", + type_id="yum_distributor", + repo_id="ubi_source", + last_publish=dt, + relative_url="content/unit/3/client", + ) + + output_binary_repo = YumRepository( + id="ubi_binary", + content_set="ubi-8-for-x86_64-appstream-rpms", + population_sources=["input_binary"], + ubi_population=True, + ubi_config_version="8", + eng_product_id=102, + distributors=[d1], + relative_url="content/unit/1/client", + ) + input_binary_repo = YumRepository(id="input_binary") + input_source_repo = YumRepository(id="input_source") + input_debug_repo = YumRepository(id="input_debug") + + output_source_repo = YumRepository( + id="ubi_source", + population_sources=["input_source"], + eng_product_id=102, + distributors=[d2], + relative_url="content/unit/2/client", + ) + + # Omitting debug repo. + + ubi_populate = FakeUbiPopulate( + "foo.pulp.com", + ("foo", "foo"), + False, + ubiconfig_dir_or_url=TEST_DATA_DIR, + ubi_manifest_url="https://ubi-manifest.com", + ) + + fake_pulp = ubi_populate.pulp_client_controller + fake_pulp.insert_repository(input_binary_repo) + fake_pulp.insert_repository(input_source_repo) + fake_pulp.insert_repository(input_debug_repo) + + fake_pulp.insert_repository(output_binary_repo) + fake_pulp.insert_repository(output_source_repo) + # No debug repo. + + get_source_repository.return_value = fake_pulp.client.get_repository("ubi_source") + + old_rpm = RpmUnit( + name="golang", + version="1", + release="a", + arch="x86_64", + filename="golang-1.a.x86_64.rpm", + sourcerpm="golang-1.a.x86_64.src.rpm", + ) + + new_rpm = RpmUnit( + name="golang", + version="2", + release="a", + arch="x86_64", + filename="golang-2.a.x86_64.rpm", + sourcerpm="golang-2.a.x86_64.src.rpm", + ) + + old_modulemd = ModulemdUnit( + name="test_md", stream="s", version="100", context="c", arch="x86_64" + ) + new_modulemd = ModulemdUnit( + name="test_md", stream="s", version="200", context="c", arch="x86_64" + ) + + old_modulemd_defaults = ModulemdDefaultsUnit( + name="test_md_defaults", + stream="stream", + profiles={"minimal": ["name_1"]}, + repo_id="ubi_binary", + ) + new_modulemd_defaults = ModulemdDefaultsUnit( + name="test_md_defaults", + stream="stream", + profiles={"minimal": ["name_1", "name_2"]}, + repo_id="input_binary", + ) + + fake_pulp.insert_units( + output_binary_repo, [old_rpm, old_modulemd, old_modulemd_defaults] + ) + fake_pulp.insert_units( + input_binary_repo, [new_rpm, new_modulemd, new_modulemd_defaults] + ) + + url = "/pulp/api/v2/repositories/{dst_repo}/actions/associate/".format( + dst_repo="ubi_binary" + ) + + requests_mock.register_uri( + "POST", url, json={"spawned_tasks": [{"task_id": "foo_task_id"}]} + ) + + url = "/pulp/api/v2/repositories/{dst_repo}/actions/unassociate/".format( + dst_repo="ubi_binary" + ) + requests_mock.register_uri( + "POST", url, json={"spawned_tasks": [{"task_id": "foo_task_id"}]} + ) + + url = "/pulp/api/v2/tasks/{task_id}/".format(task_id="foo_task_id") + requests_mock.register_uri( + "GET", url, json={"state": "finished", "task_id": "foo_task_id"} + ) + # mock calls to ubi-manifest service + _create_ubi_manifest_mocks(requests_mock) + _create_fastpurge_mocks(requests_mock) + _create_cdn_mocks(requests_mock) + + # let's run actual population + ubi_populate.populate_ubi_repos() + history = fake_pulp.publish_history + + # should have published no repositories + assert len(history) == 0 + # should have logged an error and a warning + assert "Input Debug repo does not exist" in caplog.text + assert "Skipping current content triplet, some repos are missing" in caplog.text + + def _create_fastpurge_mocks(requests_mock): url = "https://some-host/ccu/v3/delete/url/production" seconds = 0.1 diff --git a/ubipop/__init__.py b/ubipop/__init__.py index 16edc52..69d9db7 100644 --- a/ubipop/__init__.py +++ b/ubipop/__init__.py @@ -1,32 +1,29 @@ -from datetime import date import logging -import re import os - -from collections import defaultdict, deque, namedtuple +import re +from collections import defaultdict, namedtuple from concurrent.futures import as_completed -from itertools import chain -from pubtools.pulplib import Client, Criteria, PublishOptions -from fastpurge import FastPurgeClient import attr import ubiconfig - from more_executors import Executors -from more_executors.futures import f_sequence, f_proxy, f_return +from more_executors.futures import f_proxy, f_return +from pubtools.pulplib import Client, Criteria + from ubipop._pulp_client import Pulp from ubipop._utils import ( - AssociateActionModules, AssociateActionModuleDefaults, + AssociateActionModules, AssociateActionRpms, - UnassociateActionModules, UnassociateActionModuleDefaults, + UnassociateActionModules, UnassociateActionRpms, flatten_md_defaults_name_profiles, ) + +from ._cdn import Publisher from ._matcher import Matcher from .ubi_manifest_client.client import Client as UbimClient -from ._cdn import Publisher _LOG = logging.getLogger("ubipop") @@ -39,10 +36,6 @@ class ConfigMissing(Exception): pass -class PopulationSourceMissing(Exception): - pass - - RepoSet = namedtuple("RepoSet", ["rpm", "source", "debug"]) @@ -119,7 +112,9 @@ def __init__( **kwargs ): # legacy client implemeted in this repo, it's expected to be replaced by pubtools.pulplib.Client - self.pulp = self._make_pulp_client(pulp_hostname, pulp_auth, verify, Pulp) + self.legacy_pulp_client = self._make_pulp_client( + pulp_hostname, pulp_auth, verify, Pulp + ) self._pulp_hostname = pulp_hostname self._pulp_auth = pulp_auth self._verify = verify @@ -135,9 +130,7 @@ def __init__( self._version = kwargs.get("version", None) self._content_set_regex = kwargs.get("content_set_regex", None) self._ubiconfig_map = None - self._ubi_manifest_url = kwargs.get("ubi_manifest_url") or None - arl_templates = os.getenv("UBIPOP_ARL_TEMPLATES", "") self._publisher_args = { "edgerc": os.getenv("UBIPOP_EDGERC_CFG", "/etc/.edgerc"), @@ -192,12 +185,6 @@ def _load_ubiconfig(self): return self._filter_ubi_conf_list(ubi_conf_list) - @property - def ubiconfig_map(self): - if self._ubiconfig_map is None: - self._ubiconfig_map = self._create_config_map() - return self._ubiconfig_map - def _filter_ubi_conf_list(self, config_list): """ Reduces the list of UBI configurations to only those matching @@ -254,57 +241,8 @@ def _filter_ubi_conf_list(self, config_list): return filtered_conf_list - def _create_config_map(self): - """Create a config map from self.ubiconfig_list, it has the form in: - { - "7.7": - { - "config_filename1": config1, - "config_filename2": config2, - ..., - }, - "8.1": - { - "config_filename1": config1, - ..., - }, - .... - } - """ - - config_map = {} - for config in self.ubiconfig_list: - config_map.setdefault(config.version, {}).setdefault( - config.file_name, config - ) - - return config_map - - def _get_config(self, repo, config): - # get the right config file by ubi_config_version attr of a repository - # if not found, try to fallback to the default version (major version) - if not repo.ubi_config_version: - raise ValueError("Repo: %s does not have ubi_config_version" % repo.id) - - _ubi_config_version = repo.ubi_config_version - if _ubi_config_version not in self.ubiconfig_map: - # if the config is missing, we need to use the default config branch - _ubi_config_version = _ubi_config_version.split(".")[0] - try: - right_config = self.ubiconfig_map[_ubi_config_version][config.file_name] - except KeyError: - _LOG.error( - "Config file %s missing from %s and default %s branches", - config.file_name, - repo.ubi_config_version, - repo.ubi_config_version.split(".")[0], - ) - raise ConfigMissing() - - return right_config - def populate_ubi_repos(self): - repo_pairs_list = [] + repo_sets_list = [] out_repos = set() # list of all affected repositories ubi_binary_repos = [] # binary repos used for generating manifest @@ -335,7 +273,7 @@ def populate_ubi_repos(self): _LOG.warning("Skipping current content triplet, some repos are missing") continue - repo_pairs_list.append((repo_sets, config)) + repo_sets_list.append(repo_sets) ubi_binary_repos.extend( [repo_set.out_repos.rpm.id for repo_set in repo_sets] ) @@ -347,12 +285,12 @@ def populate_ubi_repos(self): if self._skip_publish != "false": _LOG.warning("Repository publishing via ubipop will be skipped") self._run_ubi_population( - repo_pairs_list, out_repos, ubim_client, publisher=None + repo_sets_list, out_repos, ubim_client, publisher=None ) else: with Publisher(**self._publisher_args) as publisher: self._run_ubi_population( - repo_pairs_list, out_repos, ubim_client, publisher + repo_sets_list, out_repos, ubim_client, publisher ) publisher.wait_publish_and_purge_cache() @@ -361,26 +299,6 @@ def populate_ubi_repos(self): for repo in out_repos: f.write(repo.id.strip() + "\n") - def _run_ubi_population( - self, repo_pairs_list, out_repos, ubim_client=None, publisher=None - ): - for repo_sets, config in repo_pairs_list: - for repo_set in repo_sets: - right_config = self._get_config(repo_set.out_repos.rpm, config) - - UbiPopulateRunner( - self.pulp, - self.pulp_client, - repo_set, - right_config, - self.dry_run, - self._executor, - ubim_client, - publisher, - ).run_ubi_population() - - out_repos.update(repo_set.get_output_repos()) - def _get_ubi_repo_sets(self, ubi_binary_cs): """ Searches for ubi repository triplet (binary rpm, srpm, debug) for @@ -431,25 +349,36 @@ def _get_population_sources(self, out_repo): return repos + def _run_ubi_population( + self, repo_sets_list, out_repos, ubim_client=None, publisher=None + ): + for repo_sets in repo_sets_list: + for repo_set in repo_sets: + UbiPopulateRunner( + self.legacy_pulp_client, + repo_set, + self.dry_run, + self._executor, + ubim_client, + publisher, + ).run_ubi_population() + + out_repos.update(repo_set.get_output_repos()) + class UbiPopulateRunner(object): def __init__( self, - legacy_client, pulp_client, - output_repo_set, - ubiconfig_item, + repo_set, dry_run, executor, ubi_manifest_client=None, publisher=None, ): - self.pulp = legacy_client self.pulp_client = pulp_client self.ubim_client = ubi_manifest_client - - self.repos = output_repo_set - self.ubiconfig = ubiconfig_item + self.repo_set = repo_set self.dry_run = dry_run self._executor = executor self._publisher = publisher @@ -488,54 +417,56 @@ def _get_pulp_actions(self, current_content): """ modules_assoc, modules_unassoc = self._get_pulp_actions_mds( - self.repos.modules, current_content.modules + self.repo_set.modules, current_content.modules ) md_defaults_assoc, md_defaults_unassoc = self._get_pulp_actions_md_defaults( - self.repos.module_defaults, current_content.modulemd_defaults + self.repo_set.module_defaults, current_content.modulemd_defaults ) rpms_assoc, rpms_unassoc = self._get_pulp_actions_pkgs( - self.repos.packages, current_content.binary_rpms + self.repo_set.packages, current_content.binary_rpms ) srpms_assoc, srpms_unassoc = self._get_pulp_actions_pkgs( - self.repos.source_rpms, current_content.source_rpms + self.repo_set.source_rpms, current_content.source_rpms ) debug_assoc = None debug_unassoc = None if current_content.debug_rpms: debug_assoc, debug_unassoc = self._get_pulp_actions_pkgs( - self.repos.debug_rpms, current_content.debug_rpms + self.repo_set.debug_rpms, current_content.debug_rpms ) associations = ( AssociateActionModules( - modules_assoc, self.repos.out_repos.rpm, self.repos.in_repos.rpm + modules_assoc, self.repo_set.out_repos.rpm, self.repo_set.in_repos.rpm ), AssociateActionRpms( - rpms_assoc, self.repos.out_repos.rpm, self.repos.in_repos.rpm + rpms_assoc, self.repo_set.out_repos.rpm, self.repo_set.in_repos.rpm ), AssociateActionRpms( - srpms_assoc, self.repos.out_repos.source, self.repos.in_repos.source + srpms_assoc, + self.repo_set.out_repos.source, + self.repo_set.in_repos.source, ), AssociateActionRpms( - debug_assoc, self.repos.out_repos.debug, self.repos.in_repos.debug + debug_assoc, self.repo_set.out_repos.debug, self.repo_set.in_repos.debug ), ) unassociations = ( - UnassociateActionModules(modules_unassoc, self.repos.out_repos.rpm), - UnassociateActionRpms(rpms_unassoc, self.repos.out_repos.rpm), - UnassociateActionRpms(srpms_unassoc, self.repos.out_repos.source), - UnassociateActionRpms(debug_unassoc, self.repos.out_repos.debug), + UnassociateActionModules(modules_unassoc, self.repo_set.out_repos.rpm), + UnassociateActionRpms(rpms_unassoc, self.repo_set.out_repos.rpm), + UnassociateActionRpms(srpms_unassoc, self.repo_set.out_repos.source), + UnassociateActionRpms(debug_unassoc, self.repo_set.out_repos.debug), ) mdd_association = AssociateActionModuleDefaults( - md_defaults_assoc, self.repos.out_repos.rpm, self.repos.in_repos.rpm + md_defaults_assoc, self.repo_set.out_repos.rpm, self.repo_set.in_repos.rpm ) mdd_unassociation = UnassociateActionModuleDefaults( - md_defaults_unassoc, self.repos.out_repos.rpm + md_defaults_unassoc, self.repo_set.out_repos.rpm ) return associations, unassociations, mdd_association, mdd_unassociation @@ -568,7 +499,9 @@ def _search_expected_modulemd_defaults(self, modulemd_defaults): or_criteria = Matcher.create_or_criteria(fields, criteria_values) return f_proxy( self._executor.submit( - Matcher.search_modulemd_defaults, or_criteria, self.repos.in_repos.rpm + Matcher.search_modulemd_defaults, + or_criteria, + self.repo_set.in_repos.rpm, ) ) @@ -576,16 +509,18 @@ def run_ubi_population(self): current_content = self._get_current_content() # start async querying for modulemds and modular and non-modular packages - binary_manifest = self.ubim_client.get_manifest(self.repos.out_repos.rpm.id) - debug_manifest = self.ubim_client.get_manifest(self.repos.out_repos.debug.id) - source_manifest = self.ubim_client.get_manifest(self.repos.out_repos.source.id) - self.repos.modules = binary_manifest.modules - self.repos.module_defaults = self._search_expected_modulemd_defaults( + binary_manifest = self.ubim_client.get_manifest(self.repo_set.out_repos.rpm.id) + debug_manifest = self.ubim_client.get_manifest(self.repo_set.out_repos.debug.id) + source_manifest = self.ubim_client.get_manifest( + self.repo_set.out_repos.source.id + ) + self.repo_set.modules = binary_manifest.modules + self.repo_set.module_defaults = self._search_expected_modulemd_defaults( binary_manifest.modulemd_defaults ) - self.repos.packages = binary_manifest.packages - self.repos.debug_rpms = debug_manifest.packages - self.repos.source_rpms = source_manifest.packages + self.repo_set.packages = binary_manifest.packages + self.repo_set.debug_rpms = debug_manifest.packages + self.repo_set.source_rpms = source_manifest.packages ( associations, @@ -617,7 +552,10 @@ def _associate_unassociate_units(self, action_list): for action in action_list: if action.units: fts.extend( - [self._executor.submit(*a) for a in action.get_actions(self.pulp)] + [ + self._executor.submit(*a) + for a in action.get_actions(self.pulp_client) + ] ) return fts @@ -638,27 +576,27 @@ def _wait_pulp(self, futures): for ft in as_completed(futures): tasks = ft.result() if tasks: - self.pulp.wait_for_tasks(tasks) + self.pulp_client.wait_for_tasks(tasks) def log_curent_content(self, current_content): - _LOG.info("Current modules in repo: %s", self.repos.out_repos.rpm.id) + _LOG.info("Current modules in repo: %s", self.repo_set.out_repos.rpm.id) for module in current_content.modules: _LOG.info(module.nsvca) - _LOG.info("Current module_defaults in repo: %s", self.repos.out_repos.rpm.id) + _LOG.info("Current module_defaults in repo: %s", self.repo_set.out_repos.rpm.id) for md_d in current_content.modulemd_defaults: _LOG.info("module_defaults: %s, profiles: %s", md_d.name, md_d.profiles) - _LOG.info("Current rpms in repo: %s", self.repos.out_repos.rpm.id) + _LOG.info("Current rpms in repo: %s", self.repo_set.out_repos.rpm.id) for rpm in current_content.binary_rpms: _LOG.info(rpm.filename) - _LOG.info("Current srpms in repo: %s", self.repos.out_repos.source.id) + _LOG.info("Current srpms in repo: %s", self.repo_set.out_repos.source.id) for rpm in current_content.source_rpms: _LOG.info(rpm.filename) - if self.repos.out_repos.debug: - _LOG.info("Current rpms in repo: %s", self.repos.out_repos.debug.id) + if self.repo_set.out_repos.debug: + _LOG.info("Current rpms in repo: %s", self.repo_set.out_repos.debug.id) for rpm in current_content.debug_rpms: _LOG.info(rpm.filename) @@ -699,29 +637,31 @@ def _get_current_content(self): criteria = [Criteria.true()] current_modulemds = f_proxy( self._executor.submit( - Matcher.search_modulemds, criteria, [self.repos.out_repos.rpm] + Matcher.search_modulemds, criteria, [self.repo_set.out_repos.rpm] ) ) current_modulemd_defaults = f_proxy( self._executor.submit( - Matcher.search_modulemd_defaults, criteria, [self.repos.out_repos.rpm] + Matcher.search_modulemd_defaults, + criteria, + [self.repo_set.out_repos.rpm], ) ) current_rpms = f_proxy( self._executor.submit( - Matcher.search_rpms, criteria, [self.repos.out_repos.rpm] + Matcher.search_rpms, criteria, [self.repo_set.out_repos.rpm] ) ) current_srpms = f_proxy( self._executor.submit( - Matcher.search_srpms, criteria, [self.repos.out_repos.source] + Matcher.search_srpms, criteria, [self.repo_set.out_repos.source] ) ) - if self.repos.out_repos.debug.result(): + if self.repo_set.out_repos.debug.result(): current_debug_rpms = f_proxy( self._executor.submit( - Matcher.search_rpms, criteria, [self.repos.out_repos.debug] + Matcher.search_rpms, criteria, [self.repo_set.out_repos.debug] ) ) else: @@ -740,9 +680,9 @@ def _publish_out_repos(self): to_publish = [] for repo in ( - self.repos.out_repos.rpm, - self.repos.out_repos.debug, - self.repos.out_repos.source, + self.repo_set.out_repos.rpm, + self.repo_set.out_repos.debug, + self.repo_set.out_repos.source, ): if repo.result(): to_publish.append(repo)