Update upload_repo

pubtools/_pulp/tasks/push/items/base.py

@@ -95,6 +95,11 @@ class PulpPushItem(object):
     to a single unit (e.g. modulemd YAML files; comps.xml files).
     """
 
+    MULTI_CONTEXT = False
+    """
+    Can the push item class create a different upload context based on the data
+    it holds. e.g PulpRpmPushItem has checksum separated upload repos/contexts
+    """
     @classmethod
     def for_item(cls, pushsource_item, **kwargs):
         """Given a pushsource.PushItem, returns an instance of a PulpPushItem wrapper
@@ -140,8 +145,8 @@ def match_items_units(cls, items, units):
     def upload_context(cls, pulp_client):
         """Return a context object used during uploads.
 
-        The context object will be shared across all uploads for a specific content
-        type.
+        The context object should be shared across all uploads for a specific
+        content type. Subclasses classes may introduce multiple contexts for performance.
 
         Subclasses MAY override this to provide their own context
         (e.g. to cache a value rather than recalculating per upload).

pubtools/_pulp/tasks/push/items/erratum.py

@@ -18,7 +18,7 @@
 class ErratumUploadContext(UploadContext):
     """A custom context for Erratum uploads.
 
-    This context object avoids having to query the UPLOAD_REPO repeatedly.
+    This context object avoids having to query the PushItem's repo repeatedly.
     """
 
     upload_repo = attr.ib(default=None)
@@ -29,13 +29,17 @@ class ErratumUploadContext(UploadContext):
 class PulpErratumPushItem(PulpPushItem):
     """Handler for errata."""
 
-    # Erratums are always uploaded to this repo first.
-    # Pulp stores the erratum pkglist for (erratum_id, repo) pair. If an erratum is
-    # uploaded for an already existing ID, the pkglist is overwritten if the repo is
-    # same, else is appended. To keep the pkglist consistent, all the erratums are
-    # uploaded to the same repo instead of a random repo from the list of destinations.
-    # For more details, refer RHELDST-12782.
-    UPLOAD_REPO = "all-rpm-content"
+    MULTI_CONTEXT = True
+    # To improve performance, we split the errata into different repos by year
+    # Assuming the advisory name is formatted like RHXA-YYYY[extravalues]
+    @property
+    def upload_repo(self):
+        try:
+            erratum_year = self.pushsource_item.name[5:9]
+            return "all-erratum-content-%s".format(erratum_year)
+        except KeyError:
+            # Poorly formatted errata name, use default
+            return "all-erratum-content-0000"
 
     @property
     def unit_type(self):
@@ -89,11 +93,10 @@ def with_unit(self, unit):
 
         return out
 
-    @classmethod
-    def upload_context(cls, pulp_client):
+    def upload_context(self, pulp_client):
         return ErratumUploadContext(
             client=pulp_client,
-            upload_repo=pulp_client.get_repository(cls.UPLOAD_REPO),
+            upload_repo=pulp_client.get_repository(self.upload_repo),
         )
 
     @classmethod

pubtools/_pulp/tasks/push/items/rpm.py

@@ -9,9 +9,10 @@
 
 @attr.s(frozen=True, slots=True)
 class RpmUploadContext(UploadContext):
-    """A custom context for RPM uploads.
-
-    This context object avoids having to query the all-rpm-content repo repeatedly.
+    """
+    A custom context for RPM uploads.
+    This context object lets us avoid querying the all-rpm-content-XX repos
+    repeatedly.
     """
 
     upload_repo = attr.ib(default=None)
@@ -22,8 +23,11 @@ class RpmUploadContext(UploadContext):
 class PulpRpmPushItem(PulpPushItem):
     """Handler for RPMs."""
 
-    # RPMs are always uploaded to this repo first.
-    UPLOAD_REPO = "all-rpm-content"
+    MULTI_CONTEXT = True
+
+    @property
+    def upload_repo(self):
+        return "all-rpm-content-%s" % self.pushsource_item.sha256sum[0:2]
 
     @property
     def unit_type(self):
@@ -89,11 +93,10 @@ def match_items_units(cls, items, units):
         for item in items:
             yield item.with_unit(units_by_sum.get(item.pushsource_item.sha256sum))
 
-    @classmethod
-    def upload_context(cls, pulp_client):
+    def upload_context(self, pulp_client):
         return RpmUploadContext(
             client=pulp_client,
-            upload_repo=pulp_client.get_repository(cls.UPLOAD_REPO),
+            upload_repo=pulp_client.get_repository(self.upload_repo),
         )
 
     @property

pubtools/_pulp/tasks/push/phase/upload.py

@@ -58,15 +58,20 @@ def run(self):
                 prepush_skipped += 1
                 self.put_output(item)
             else:
-                # This item is not in Pulp, or otherwise needs a reupload.
                 item_type = type(item)
-                if item_type not in upload_context:
-                    upload_context[item_type] = item_type.upload_context(
-                        self.pulp_client
-                    )
-
-                ctx = upload_context[item_type]
-
+                if item_type.MULTI_CONTEXT:
+                    if item_type not in upload_context:
+                        upload_context[item_type] = {}
+                        upload_context[item_type][item.upload_repo] = \
+                            item.upload_context(self.pulp_client)
+                    elif item.upload_repo not in upload_context[item_type]:
+                        upload_context[item_type][item.upload_repo] = \
+                            item.upload_context( self.pulp_client)
+                    ctx = upload_context[item_type][item.upload_repo]
+                else:
+                    if item_type not in upload_context:
+                        upload_context[item_type] = item.upload_context(self.pulp_client)
+                    ctx = upload_context[item_type]
                 uploading += 1
                 self.put_future_output(item.ensure_uploaded(ctx))
 

tests/push/test_upload_sharing.py

@@ -39,8 +39,9 @@ def test_uploads_shared(data_path):
     """Upload phase allows for uploads of identical content to be reused."""
 
     pulp_ctrl = FakeController()
+    pulp_ctrl.insert_repository(YumRepository(id="all-rpm-content-54"))
+    pulp_ctrl.insert_repository(YumRepository(id="all-rpm-content-e8"))
 
-    pulp_ctrl.insert_repository(YumRepository(id="all-rpm-content"))
     pulp_ctrl.insert_repository(YumRepository(id="repo1"))
     pulp_ctrl.insert_repository(YumRepository(id="repo2"))
     pulp_ctrl.insert_repository(YumRepository(id="repo3"))