diff --git a/CHANGELOG.md b/CHANGELOG.md index a045ca8a..7cde0c4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- Introduced `Repository.search_content` API for retrieving content units + from Pulp repositories. + ### Fixed - Fixed a bug that export an empty maintenance report would crash. diff --git a/docs/api/model.rst b/docs/api/model.rst index 09cfa87b..27782aea 100644 --- a/docs/api/model.rst +++ b/docs/api/model.rst @@ -49,6 +49,8 @@ Units .. autoclass:: pubtools.pulplib.ModulemdUnit() :members: +.. autoclass:: pubtools.pulplib.ModulemdDefaultUnit() + :members: Task ---- diff --git a/examples/search-repo-content b/examples/search-repo-content new file mode 100644 index 00000000..fc8ec30d --- /dev/null +++ b/examples/search-repo-content @@ -0,0 +1,62 @@ +#!/usr/bin/env python +import os +import logging +from argparse import ArgumentParser + +from pubtools.pulplib import Client + +log = logging.getLogger("search-repo-content") + + +def make_client(args): + auth = None + + if args.username: + password = args.password + if password is None: + password = os.environ.get("PULP_PASSWORD") + if not password: + log.warning("No password provided for %s", args.username) + auth = (args.username, args.password) + + return Client(args.url, auth=auth, verify=not args.insecure) + + +def main(): + log.setLevel(logging.INFO) + logging.basicConfig(format="%(message)s", level=logging.INFO) + + parser = ArgumentParser( + description="Retrieve unit objects of the given type from the given repository" + ) + parser.add_argument("--url", help="Pulp server URL", required=True) + parser.add_argument("--username", help="Pulp username") + parser.add_argument("--password", help="Pulp password (or set PULP_PASSWORD in env)") + parser.add_argument("--repo-id", action="store", required=True) + parser.add_argument("--content-type", action="store", required=True) + parser.add_argument("--debug", action="store_true") + parser.add_argument("--insecure", default=False, action="store_true") + + p = parser.parse_args() + + if p.debug: + logging.getLogger("pubtools.pulplib").setLevel(logging.DEBUG) + log.setLevel(logging.DEBUG) + + client = make_client(p) + repository = client.get_repository(p.repo_id) + units = repository.search_content(type_id=p.content_type) + + if units: + log.info("Found %s %s units: \n\t%s" % ( + len(units), p.content_type, + "\n\t".join(str(unit) for unit in units) + )) + else: + log.info("No %s units found." % p.content_type) + + return units + + +if __name__ == "__main__": + main() diff --git a/pubtools/pulplib/__init__.py b/pubtools/pulplib/__init__.py index c2e5daff..2a95db6c 100644 --- a/pubtools/pulplib/__init__.py +++ b/pubtools/pulplib/__init__.py @@ -13,6 +13,7 @@ FileUnit, RpmUnit, ModulemdUnit, + ModulemdDefaultsUnit, Distributor, PublishOptions, Task, diff --git a/pubtools/pulplib/_impl/client/client.py b/pubtools/pulplib/_impl/client/client.py index 29adb955..60ba7597 100644 --- a/pubtools/pulplib/_impl/client/client.py +++ b/pubtools/pulplib/_impl/client/client.py @@ -13,7 +13,7 @@ from ..page import Page from ..criteria import Criteria -from ..model import Repository, MaintenanceReport, Distributor +from ..model import Repository, MaintenanceReport, Distributor, Unit from .search import filters_for_criteria from .errors import PulpException from .poller import TaskPoller @@ -228,16 +228,29 @@ def _search(self, return_type, resource_type, criteria=None, search_options=None "limit": self._PAGE_SIZE, "filters": filters_for_criteria(criteria, return_type), } + + type_ids = search_options.pop("type_ids", None) if search_options else None + if type_ids: + pulp_crit["type_ids"] = type_ids + search = {"criteria": pulp_crit} search.update(search_options or {}) - response_f = self._do_search(resource_type, search) + url = os.path.join(self._url, "pulp/api/v2/%s/search/" % resource_type) + + if return_type is Unit and search["criteria"]["type_ids"]: + url = os.path.join(url, "units/") + + response_f = self._do_search(url, search) # When this request is resolved, we'll have the first page of data. # We'll need to convert that into a page and also keep going with # the search if there's more to be done. return f_proxy( - f_map(response_f, lambda data: self._handle_page(return_type, search, data)) + f_map( + response_f, + lambda data: self._handle_page(url, return_type, search, data), + ) ) def get_maintenance_report(self): @@ -417,12 +430,18 @@ def _log_spawned_tasks(cls, taskdata): pass return taskdata - def _handle_page(self, object_class, search, raw_data): + def _handle_page(self, url, object_class, search, raw_data): LOG.debug("Got pulp response for %s, %s elems", search, len(raw_data)) + # Extract metadata from Pulp units + if object_class is Unit and url.endswith("units/"): + raw_data = [elem["metadata"] for elem in raw_data] + page_data = [object_class.from_data(elem) for elem in raw_data] for obj in page_data: - obj._set_client(self) + # set_client is only applicable for repository and distributor objects + if hasattr(obj, "_set_client"): + obj._set_client(self) # Do we need a next page? next_page = None @@ -433,11 +452,11 @@ def _handle_page(self, object_class, search, raw_data): search = search.copy() search["criteria"] = search["criteria"].copy() search["criteria"]["skip"] = search["criteria"]["skip"] + limit - response_f = self._do_search("repositories", search) + response_f = self._do_search(url, search) next_page = f_proxy( f_map( response_f, - lambda data: self._handle_page(object_class, search, data), + lambda data: self._handle_page(url, object_class, search, data), ) ) @@ -458,9 +477,7 @@ def _new_session(self): def _do_request(self, **kwargs): return self._session.request(**kwargs) - def _do_search(self, resource_type, search): - url = os.path.join(self._url, "pulp/api/v2/{0}/search/".format(resource_type)) - + def _do_search(self, url, search): LOG.debug("Submitting %s search: %s", url, search) return self._request_executor.submit( self._do_request, method="POST", url=url, json=search diff --git a/pubtools/pulplib/_impl/client/search.py b/pubtools/pulplib/_impl/client/search.py index 640fb5a4..1e3b9604 100644 --- a/pubtools/pulplib/_impl/client/search.py +++ b/pubtools/pulplib/_impl/client/search.py @@ -1,3 +1,4 @@ +import logging import datetime from pubtools.pulplib._impl.criteria import ( AndCriteria, @@ -13,6 +14,9 @@ from pubtools.pulplib._impl import compat_attr as attr from pubtools.pulplib._impl.model.attr import PULP2_FIELD, PY_PULP2_CONVERTER +from pubtools.pulplib._impl.model.unit import SUPPORTED_UNIT_TYPES + +LOG = logging.getLogger("pubtools.pulplib") def all_subclasses(klass): @@ -113,3 +117,28 @@ def field_match(to_match): return {"$lt": to_mongo_json(to_match._value)} raise TypeError("Not a matcher: %s" % repr(to_match)) + + +def validate_type_ids(type_ids): + valid_type_ids = [] + invalid_type_ids = [] + + if isinstance(type_ids, str): + type_ids = [type_ids] + + if not isinstance(type_ids, (list, tuple)): + raise TypeError("Expected str, list, or tuple, got %s" % type(type_ids)) + + for type_id in type_ids: + if type_id in SUPPORTED_UNIT_TYPES: + valid_type_ids.append(type_id) + else: + invalid_type_ids.append(type_id) + + if invalid_type_ids: + LOG.error("Invalid content type ID(s): \n\t%s", ", ".join(invalid_type_ids)) + + if valid_type_ids: + return valid_type_ids + + raise ValueError("Must provide valid content type ID(s)") diff --git a/pubtools/pulplib/_impl/model/__init__.py b/pubtools/pulplib/_impl/model/__init__.py index c80929e1..8eeb7dc5 100644 --- a/pubtools/pulplib/_impl/model/__init__.py +++ b/pubtools/pulplib/_impl/model/__init__.py @@ -1,4 +1,8 @@ -from .common import PulpObject, DetachedException, InvalidDataException +from .common import ( + PulpObject, + DetachedException, + InvalidDataException, +) from .repository import ( Repository, YumRepository, @@ -6,7 +10,7 @@ ContainerImageRepository, PublishOptions, ) -from .unit import Unit, FileUnit, RpmUnit, ModulemdUnit +from .unit import Unit, FileUnit, RpmUnit, ModulemdUnit, ModulemdDefaultsUnit from .task import Task from .distributor import Distributor from .maintenance import MaintenanceReport, MaintenanceEntry diff --git a/pubtools/pulplib/_impl/model/repository/base.py b/pubtools/pulplib/_impl/model/repository/base.py index 8dd0d3b6..4109caff 100644 --- a/pubtools/pulplib/_impl/model/repository/base.py +++ b/pubtools/pulplib/_impl/model/repository/base.py @@ -4,10 +4,16 @@ from attr import validators from more_executors.futures import f_proxy -from ..common import PulpObject, Deletable, DetachedException +from ..common import ( + PulpObject, + Deletable, + DetachedException, +) from ..attr import pulp_attrib from ..distributor import Distributor from ..frozenlist import FrozenList +from ..unit import Unit +from ...client.search import validate_type_ids from ...schema import load_schema from ... import compat_attr as attr @@ -188,6 +194,88 @@ def distributor(self, distributor_id): """ return self._distributors_by_id.get(distributor_id) + @property + def iso_content(self): + """A list of iso units stored in this repository. + + Returns: + list[:class:`~pubtools.pulplib.FileUnit`] + + .. versionadded:: 2.4.0 + """ + return self.search_content("iso") + + @property + def rpm_content(self): + """A list of rpm units stored in this repository. + + Returns: + list[:class:`~pubtools.pulplib.RpmUnit`] + + .. versionadded:: 2.4.0 + """ + return self.search_content("rpm") + + @property + def srpm_content(self): + """A list of srpm units stored in this repository. + + Returns: + list[:class:`~pubtools.pulplib.Unit`] + + .. versionadded:: 2.4.0 + """ + return self.search_content("srpm") + + @property + def modulemd_content(self): + """A list of modulemd units stored in this repository. + + Returns: + list[:class:`~pubtools.pulplib.ModulemdUnit`] + + .. versionadded:: 2.4.0 + """ + return self.search_content("modulemd") + + @property + def modulemd_defaults_content(self): + """A list of modulemd_defaults units stored in this repository. + + Returns: + list[:class:`~pubtools.pulplib.ModulemdDefaultsUnit`] + + .. versionadded:: 2.4.0 + """ + return self.search_content("modulemd_defaults") + + def search_content(self, type_id, criteria=None): + """Search this repository for content matching the given criteria. + + Args: + type_id (str) + The content type to search + criteria (:class:`~pubtools.pulplib.Criteria`) + A criteria object used for this search. + + Returns: + list[:class:`~pubtools.pulplib.Unit`] + A list of zero or more :class:`~pubtools.pulplib.Unit` + subclasses found by the search operation. + + .. versionadded:: 2.4.0 + """ + if not self._client: + raise DetachedException() + + resource_type = "repositories/%s" % self.id + search_options = {"type_ids": validate_type_ids(type_id)} + return list( + self._client._search( + Unit, resource_type, criteria=criteria, search_options=search_options + ) + ) + def delete(self): """Delete this repository from Pulp. diff --git a/pubtools/pulplib/_impl/model/unit/__init__.py b/pubtools/pulplib/_impl/model/unit/__init__.py index 2ce4fa26..adbf66b4 100644 --- a/pubtools/pulplib/_impl/model/unit/__init__.py +++ b/pubtools/pulplib/_impl/model/unit/__init__.py @@ -2,3 +2,6 @@ from .file import FileUnit from .rpm import RpmUnit from .modulemd import ModulemdUnit +from .modulemd_defaults import ModulemdDefaultsUnit + +SUPPORTED_UNIT_TYPES = ("iso", "rpm", "srpm", "modulemd", "modulemd_defaults") diff --git a/pubtools/pulplib/_impl/model/unit/modulemd_defaults.py b/pubtools/pulplib/_impl/model/unit/modulemd_defaults.py new file mode 100644 index 00000000..912750f0 --- /dev/null +++ b/pubtools/pulplib/_impl/model/unit/modulemd_defaults.py @@ -0,0 +1,28 @@ +from .base import Unit, unit_type + +from ..attr import pulp_attrib +from ... import compat_attr as attr + + +@unit_type("modulemd_defaults") +@attr.s(kw_only=True, frozen=True) +class ModulemdDefaultsUnit(Unit): + """A :class:`~pubtools.pulplib.Unit` representing a modulemd_defaults document. + + .. versionadded:: 2.4.0 + """ + + name = pulp_attrib(type=str, pulp_field="name") + """The name of this modulemd defaults unit""" + + stream = pulp_attrib(type=str, pulp_field="stream") + """The stream of this modulemd defaults unit""" + + repo_id = pulp_attrib(type=str, pulp_field="repo_id") + """The repository ID bound to this modulemd defaults unit""" + + profiles = pulp_attrib(pulp_field="profiles") + """The profiles of this modulemd defaults unit. + + The type for this attribute is omitted to allow for either dict or None. + """ diff --git a/pubtools/pulplib/_impl/schema/unit.yaml b/pubtools/pulplib/_impl/schema/unit.yaml index ecc75b34..5e5bdce5 100644 --- a/pubtools/pulplib/_impl/schema/unit.yaml +++ b/pubtools/pulplib/_impl/schema/unit.yaml @@ -117,6 +117,30 @@ definitions: - context - arch + # modulemd_defaults units + modulemd_defaults: + type: object + + properties: + # Type of the unit + _content_type_id: + const: modulemd_defaults + + name: + type: string + repo_id: + type: string + stream: + type: string + profiles: + type: object + + required: + - _content_type_id + - name + - stream + - repo_id + # Schema for any unknown type of unit unknown: type: object @@ -133,6 +157,7 @@ definitions: - rpm - srpm - modulemd + - modulemd_defaults required: - _content_type_id @@ -141,4 +166,5 @@ anyOf: - $ref: "#/definitions/iso" - $ref: "#/definitions/rpm" - $ref: "#/definitions/modulemd" +- $ref: "#/definitions/modulemd_defaults" - $ref: "#/definitions/unknown" diff --git a/setup.py b/setup.py index d685d13b..5feb4896 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ def get_requirements(): setup( name="pubtools-pulplib", - version="2.3.2", + version="2.4.0", packages=find_packages(exclude=["tests"]), package_data={"pubtools.pulplib._impl.schema": ["*.yaml"]}, url="https://github.com/release-engineering/pubtools-pulplib", diff --git a/tests/client/test_client.py b/tests/client/test_client.py index c17444cc..9b5dc233 100644 --- a/tests/client/test_client.py +++ b/tests/client/test_client.py @@ -2,7 +2,6 @@ import datetime import json import pytest -import requests_mock from mock import patch diff --git a/tests/client/test_search.py b/tests/client/test_search.py index b3cfaeb6..fb980209 100644 --- a/tests/client/test_search.py +++ b/tests/client/test_search.py @@ -3,7 +3,11 @@ from pubtools.pulplib import Criteria, Matcher -from pubtools.pulplib._impl.client.search import filters_for_criteria, field_match +from pubtools.pulplib._impl.client.search import ( + filters_for_criteria, + field_match, + validate_type_ids, +) def test_null_criteria(): @@ -97,3 +101,25 @@ def test_dict_matcher_value(): assert filters_for_criteria(crit) == { "created": {"$lt": {"created_date": {"$date": "2019-09-04T00:00:00Z"}}} } + + +def test_valid_type_ids(caplog): + assert validate_type_ids(["srpm", "iso", "quark", "rpm"]) == ["srpm", "iso", "rpm"] + for m in ["Invalid content type ID(s):", "quark"]: + assert m in caplog.text + + +def test_invalid_type_ids(): + """validate_type_ids raises if called without valid criteria""" + with pytest.raises(ValueError) as e: + validate_type_ids("quark") + + assert "Must provide valid content type ID(s)" in str(e) + + +def test_invalid_type_ids_type(): + """validate_type_ids raises if called without valid criteria""" + with pytest.raises(TypeError) as e: + validate_type_ids({"srpm": "some-srpm"}) + + assert "Expected str, list, or tuple, got %s" % type({}) in str(e) diff --git a/tests/repository/test_search_content.py b/tests/repository/test_search_content.py new file mode 100644 index 00000000..0ce1a8d6 --- /dev/null +++ b/tests/repository/test_search_content.py @@ -0,0 +1,137 @@ +import pytest +from pubtools.pulplib import Repository, DetachedException + + +def test_detached(): + """content searches raise if called on a detached repository object""" + with pytest.raises(DetachedException): + assert not Repository(id="some-repo").iso_content + + +def test_iso_content(client, requests_mocker): + """iso_content returns correct unit types""" + repo = Repository(id="some-repo") + repo.__dict__["_client"] = client + requests_mocker.post( + "https://pulp.example.com/pulp/api/v2/repositories/some-repo/search/units/", + json=[ + { + "metadata": { + "_content_type_id": "iso", + "name": "hello.txt", + "size": 23, + "checksum": "a" * 64, + } + } + ], + ) + + isos = repo.iso_content + + assert len(isos) == 1 + assert isos[0].content_type_id == "iso" + + +def test_rpm_content(client, requests_mocker): + """rpm_content returns correct unit types""" + repo = Repository(id="some-repo") + repo.__dict__["_client"] = client + requests_mocker.post( + "https://pulp.example.com/pulp/api/v2/repositories/some-repo/search/units/", + json=[ + { + "metadata": { + "_content_type_id": "rpm", + "name": "bash", + "epoch": "0", + "filename": "bash-x86_64.rpm", + "version": "4.0", + "release": "1", + "arch": "x86_64", + } + } + ], + ) + + rpms = repo.rpm_content + + assert len(rpms) == 1 + assert rpms[0].content_type_id == "rpm" + + +def test_srpm_content(client, requests_mocker): + """srpm_content returns correct unit types""" + repo = Repository(id="some-repo") + repo.__dict__["_client"] = client + requests_mocker.post( + "https://pulp.example.com/pulp/api/v2/repositories/some-repo/search/units/", + json=[ + { + "metadata": { + "_content_type_id": "srpm", + "name": "bash", + "epoch": "0", + "filename": "bash-x86_64.srpm", + "version": "4.0", + "release": "1", + "arch": "x86_64", + } + } + ], + ) + + srpms = repo.srpm_content + + assert len(srpms) == 1 + assert srpms[0].content_type_id == "srpm" + + +def test_modulemd_content(client, requests_mocker): + """modulemd_content returns correct unit types""" + repo = Repository(id="some-repo") + repo.__dict__["_client"] = client + requests_mocker.post( + "https://pulp.example.com/pulp/api/v2/repositories/some-repo/search/units/", + json=[ + { + "metadata": { + "_content_type_id": "modulemd", + "name": "md", + "stream": "s1", + "version": 1234, + "context": "a1b2c3", + "arch": "s390x", + } + } + ], + ) + + modulemds = repo.modulemd_content + + assert len(modulemds) == 1 + assert modulemds[0].content_type_id == "modulemd" + + +def test_modulemd_defaults_content(client, requests_mocker): + """modulemd_defaults_content returns correct unit types""" + repo = Repository(id="some-repo") + repo.__dict__["_client"] = client + requests_mocker.post( + "https://pulp.example.com/pulp/api/v2/repositories/some-repo/search/units/", + json=[ + { + "metadata": { + "_content_type_id": "modulemd_defaults", + "name": "mdd", + "stream": "1.0", + "repo_id": "some-repo", + "profiles": {"p1": ["something"]}, + } + } + ], + ) + + modulemd_defaults = repo.modulemd_defaults_content + + assert len(modulemd_defaults) == 1 + assert modulemd_defaults[0].content_type_id == "modulemd_defaults"