Introduce search_content method for repositories [DELIVERY-7839]

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+- Introduced `Repository.search_content` API for retrieving content units
+  from Pulp repositories.
+
 ### Fixed
 - Fixed a bug that export an empty maintenance report would crash.
 

docs/api/model.rst

@@ -49,6 +49,8 @@ Units
 .. autoclass:: pubtools.pulplib.ModulemdUnit()
    :members:
 
+.. autoclass:: pubtools.pulplib.ModulemdDefaultUnit()
+   :members:
 
 Task
 ----

examples/search-repo-content

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+import os
+import logging
+from argparse import ArgumentParser
+
+from pubtools.pulplib import Client
+
+log = logging.getLogger("search-repo-content")
+
+
+def make_client(args):
+    auth = None
+
+    if args.username:
+        password = args.password
+        if password is None:
+            password = os.environ.get("PULP_PASSWORD")
+        if not password:
+            log.warning("No password provided for %s", args.username)
+        auth = (args.username, args.password)
+
+    return Client(args.url, auth=auth, verify=not args.insecure)
+
+
+def main():
+    log.setLevel(logging.INFO)
+    logging.basicConfig(format="%(message)s", level=logging.INFO)
+
+    parser = ArgumentParser(
+        description="Retrieve unit objects of the given type from the given repository"
+    )
+    parser.add_argument("--url", help="Pulp server URL", required=True)
+    parser.add_argument("--username", help="Pulp username")
+    parser.add_argument("--password", help="Pulp password (or set PULP_PASSWORD in env)")
+    parser.add_argument("--repo-id", action="store", required=True)
+    parser.add_argument("--content-type", action="store", required=True)
+    parser.add_argument("--debug", action="store_true")
+    parser.add_argument("--insecure", default=False, action="store_true")
+
+    p = parser.parse_args()
+
+    if p.debug:
+        logging.getLogger("pubtools.pulplib").setLevel(logging.DEBUG)
+        log.setLevel(logging.DEBUG)
+
+    client = make_client(p)
+    repository = client.get_repository(p.repo_id)
+    units = repository.search_content(type_id=p.content_type)
+
+    if units:
+        log.info("Found %s %s units: \n\t%s" % (
+            len(units), p.content_type,
+            "\n\t".join(str(unit) for unit in units)
+        ))
+    else:
+        log.info("No %s units found." % p.content_type)
+
+    return units
+
+
+if __name__ == "__main__":
+    main()

pubtools/pulplib/__init__.py

@@ -13,6 +13,7 @@
     FileUnit,
     RpmUnit,
     ModulemdUnit,
+    ModulemdDefaultsUnit,
     Distributor,
     PublishOptions,
     Task,

pubtools/pulplib/_impl/client/client.py

@@ -13,7 +13,7 @@
 
 from ..page import Page
 from ..criteria import Criteria
-from ..model import Repository, MaintenanceReport, Distributor
+from ..model import Repository, MaintenanceReport, Distributor, Unit
 from .search import filters_for_criteria
 from .errors import PulpException
 from .poller import TaskPoller
@@ -228,16 +228,29 @@ def _search(self, return_type, resource_type, criteria=None, search_options=None
             "limit": self._PAGE_SIZE,
             "filters": filters_for_criteria(criteria, return_type),
         }
+
+        type_ids = search_options.pop("type_ids", None) if search_options else None
+        if type_ids:
+            pulp_crit["type_ids"] = type_ids
+
         search = {"criteria": pulp_crit}
         search.update(search_options or {})
 
-        response_f = self._do_search(resource_type, search)
+        url = os.path.join(self._url, "pulp/api/v2/%s/search/" % resource_type)
+
+        if return_type is Unit and search["criteria"]["type_ids"]:
+            url = os.path.join(url, "units/")
+
+        response_f = self._do_search(url, search)
 
         # When this request is resolved, we'll have the first page of data.
         # We'll need to convert that into a page and also keep going with
         # the search if there's more to be done.
         return f_proxy(
-            f_map(response_f, lambda data: self._handle_page(return_type, search, data))
+            f_map(
+                response_f,
+                lambda data: self._handle_page(url, return_type, search, data),
+            )
         )
 
     def get_maintenance_report(self):
@@ -417,12 +430,18 @@ def _log_spawned_tasks(cls, taskdata):
             pass
         return taskdata
 
-    def _handle_page(self, object_class, search, raw_data):
+    def _handle_page(self, url, object_class, search, raw_data):
         LOG.debug("Got pulp response for %s, %s elems", search, len(raw_data))
 
+        # Extract metadata from Pulp units
+        if object_class is Unit and url.endswith("units/"):
+            raw_data = [elem["metadata"] for elem in raw_data]
+
         page_data = [object_class.from_data(elem) for elem in raw_data]
         for obj in page_data:
-            obj._set_client(self)
+            # set_client is only applicable for repository and distributor objects
+            if hasattr(obj, "_set_client"):
+                obj._set_client(self)
 
         # Do we need a next page?
         next_page = None
@@ -433,11 +452,11 @@ def _handle_page(self, object_class, search, raw_data):
             search = search.copy()
             search["criteria"] = search["criteria"].copy()
             search["criteria"]["skip"] = search["criteria"]["skip"] + limit
-            response_f = self._do_search("repositories", search)
+            response_f = self._do_search(url, search)
             next_page = f_proxy(
                 f_map(
                     response_f,
-                    lambda data: self._handle_page(object_class, search, data),
+                    lambda data: self._handle_page(url, object_class, search, data),
                 )
             )
 
@@ -458,9 +477,7 @@ def _new_session(self):
     def _do_request(self, **kwargs):
         return self._session.request(**kwargs)
 
-    def _do_search(self, resource_type, search):
-        url = os.path.join(self._url, "pulp/api/v2/{0}/search/".format(resource_type))
-
+    def _do_search(self, url, search):
         LOG.debug("Submitting %s search: %s", url, search)
         return self._request_executor.submit(
             self._do_request, method="POST", url=url, json=search

pubtools/pulplib/_impl/client/search.py

@@ -1,3 +1,4 @@
+import logging
 import datetime
 from pubtools.pulplib._impl.criteria import (
     AndCriteria,
@@ -13,6 +14,9 @@
 
 from pubtools.pulplib._impl import compat_attr as attr
 from pubtools.pulplib._impl.model.attr import PULP2_FIELD, PY_PULP2_CONVERTER
+from pubtools.pulplib._impl.model.unit import SUPPORTED_UNIT_TYPES
+
+LOG = logging.getLogger("pubtools.pulplib")
 
 
 def all_subclasses(klass):
@@ -113,3 +117,28 @@ def field_match(to_match):
         return {"$lt": to_mongo_json(to_match._value)}
 
     raise TypeError("Not a matcher: %s" % repr(to_match))
+
+
+def validate_type_ids(type_ids):
+    valid_type_ids = []
+    invalid_type_ids = []
+
+    if isinstance(type_ids, str):
+        type_ids = [type_ids]
+
+    if not isinstance(type_ids, (list, tuple)):
+        raise TypeError("Expected str, list, or tuple, got %s" % type(type_ids))
+
+    for type_id in type_ids:
+        if type_id in SUPPORTED_UNIT_TYPES:
+            valid_type_ids.append(type_id)
+        else:
+            invalid_type_ids.append(type_id)
+
+    if invalid_type_ids:
+        LOG.error("Invalid content type ID(s): \n\t%s", ", ".join(invalid_type_ids))
+
+    if valid_type_ids:
+        return valid_type_ids
+
+    raise ValueError("Must provide valid content type ID(s)")

pubtools/pulplib/_impl/model/__init__.py

@@ -1,12 +1,16 @@
-from .common import PulpObject, DetachedException, InvalidDataException
+from .common import (
+    PulpObject,
+    DetachedException,
+    InvalidDataException,
+)
 from .repository import (
     Repository,
     YumRepository,
     FileRepository,
     ContainerImageRepository,
     PublishOptions,
 )
-from .unit import Unit, FileUnit, RpmUnit, ModulemdUnit
+from .unit import Unit, FileUnit, RpmUnit, ModulemdUnit, ModulemdDefaultsUnit
 from .task import Task
 from .distributor import Distributor
 from .maintenance import MaintenanceReport, MaintenanceEntry

pubtools/pulplib/_impl/model/repository/base.py

@@ -4,10 +4,16 @@
 from attr import validators
 from more_executors.futures import f_proxy
 
-from ..common import PulpObject, Deletable, DetachedException
+from ..common import (
+    PulpObject,
+    Deletable,
+    DetachedException,
+)
 from ..attr import pulp_attrib
 from ..distributor import Distributor
 from ..frozenlist import FrozenList
+from ..unit import Unit
+from ...client.search import validate_type_ids
 from ...schema import load_schema
 from ... import compat_attr as attr
 
@@ -188,6 +194,88 @@ def distributor(self, distributor_id):
         """
         return self._distributors_by_id.get(distributor_id)
 
+    @property
+    def iso_content(self):
+        """A list of iso units stored in this repository.
+
+        Returns:
+            list[:class:`~pubtools.pulplib.FileUnit`]
+
+        .. versionadded:: 2.4.0
+        """
+        return self.search_content("iso")
+
+    @property
+    def rpm_content(self):
+        """A list of rpm units stored in this repository.
+
+        Returns:
+            list[:class:`~pubtools.pulplib.RpmUnit`]
+
+        .. versionadded:: 2.4.0
+        """
+        return self.search_content("rpm")
+
+    @property
+    def srpm_content(self):
+        """A list of srpm units stored in this repository.
+
+        Returns:
+            list[:class:`~pubtools.pulplib.Unit`]
+
+        .. versionadded:: 2.4.0
+        """
+        return self.search_content("srpm")
+
+    @property
+    def modulemd_content(self):
+        """A list of modulemd units stored in this repository.
+
+        Returns:
+            list[:class:`~pubtools.pulplib.ModulemdUnit`]
+
+        .. versionadded:: 2.4.0
+        """
+        return self.search_content("modulemd")
+
+    @property
+    def modulemd_defaults_content(self):
+        """A list of modulemd_defaults units stored in this repository.
+
+        Returns:
+            list[:class:`~pubtools.pulplib.ModulemdDefaultsUnit`]
+
+        .. versionadded:: 2.4.0
+        """
+        return self.search_content("modulemd_defaults")
+
+    def search_content(self, type_id, criteria=None):
+        """Search this repository for content matching the given criteria.
+
+        Args:
+            type_id (str)
+                The content type to search
+            criteria (:class:`~pubtools.pulplib.Criteria`)
+                A criteria object used for this search.
+
+        Returns:
+            list[:class:`~pubtools.pulplib.Unit`]
+                A list of zero or more :class:`~pubtools.pulplib.Unit`
+                subclasses found by the search operation.
+
+        .. versionadded:: 2.4.0
+        """
+        if not self._client:
+            raise DetachedException()
+
+        resource_type = "repositories/%s" % self.id
+        search_options = {"type_ids": validate_type_ids(type_id)}
+        return list(
+            self._client._search(
+                Unit, resource_type, criteria=criteria, search_options=search_options
+            )
+        )
+
     def delete(self):
         """Delete this repository from Pulp.
 

pubtools/pulplib/_impl/model/unit/__init__.py

@@ -2,3 +2,6 @@
 from .file import FileUnit
 from .rpm import RpmUnit
 from .modulemd import ModulemdUnit
+from .modulemd_defaults import ModulemdDefaultsUnit
+
+SUPPORTED_UNIT_TYPES = ("iso", "rpm", "srpm", "modulemd", "modulemd_defaults")

pubtools/pulplib/_impl/model/unit/modulemd_defaults.py

@@ -0,0 +1,28 @@
+from .base import Unit, unit_type
+
+from ..attr import pulp_attrib
+from ... import compat_attr as attr
+
+
+@unit_type("modulemd_defaults")
+@attr.s(kw_only=True, frozen=True)
+class ModulemdDefaultsUnit(Unit):
+    """A :class:`~pubtools.pulplib.Unit` representing a modulemd_defaults document.
+
+    .. versionadded:: 2.4.0
+    """
+
+    name = pulp_attrib(type=str, pulp_field="name")
+    """The name of this modulemd defaults unit"""
+
+    stream = pulp_attrib(type=str, pulp_field="stream")
+    """The stream of this modulemd defaults unit"""
+
+    repo_id = pulp_attrib(type=str, pulp_field="repo_id")
+    """The repository ID bound to this modulemd defaults unit"""
+
+    profiles = pulp_attrib(pulp_field="profiles")
+    """The profiles of this modulemd defaults unit.
+
+    The type for this attribute is omitted to allow for either dict or None.
+    """