Do garbage collect in batches [RHELDST-20725]

Scheduled garbage collection jobs were failing because of out of memory
exceptions. This change makes garbage collect run in batches to limit
the amount of resources used at once.

pubtools/_pulp/tasks/garbage_collect.py

@@ -1,3 +1,4 @@
+import os
 import logging
 from datetime import datetime, timedelta
 
@@ -10,6 +11,8 @@
 LOG = logging.getLogger("pubtools.pulp")
 step = PulpTask.step
 
+UNASSOCIATE_BATCH_LIMIT = int(
+    os.getenv("PULP_GC_UNASSOCIATE_BATCH_LIMIT", "10000"))
 
 class GarbageCollect(PulpClientService, PulpTask):
     """Perform garbage collection on Pulp data.
@@ -96,16 +99,28 @@ def clean_all_rpm_content(self):
             LOG.info("No repos found for cleaning.")
             return
         arc_repo = clean_repos[0]
-        deleted_arc = list(arc_repo.remove_content(criteria=criteria))
+
         deleted_content = []
-        for task in deleted_arc:
-            if task.repo_id == "all-rpm-content":
+
+        while True:
+            deletion_tasks = arc_repo.remove_content(
+                criteria=criteria, limit=UNASSOCIATE_BATCH_LIMIT
+            ).result()
+            arc_tasks = [t for t in deletion_tasks if
+                         t.repo_id == "all-rpm-content"]
+            for task in arc_tasks:
                 for unit in task.units:
                     LOG.info("Old all-rpm-content deleted: %s", unit.name)
                     deleted_content.append(unit)
-        if not deleted_content:
-            LOG.info("No all-rpm-content found older than %s", arc_threshold)
 
+            if not arc_tasks or \
+                    any([t for t in arc_tasks
+                         if len(t.units) < UNASSOCIATE_BATCH_LIMIT]):
+                break
+
+        if not deleted_content:
+            LOG.info("No all-rpm-content found older than %s",
+                     arc_threshold)
 
 def entry_point():
     with GarbageCollect() as instance:

requirements.txt

@@ -301,9 +301,9 @@ jsonschema-specifications==2023.7.1 \
     --hash=sha256:05adf340b659828a004220a9613be00fa3f223f2b82002e273dee62fd50524b1 \
     --hash=sha256:c91a50404e88a1f6ba40636778e2ee08f6e24c5613fe4c53ac24578a5a7f72bb
     # via jsonschema
-kobo==0.32.0 \
-    --hash=sha256:df68cec9f5412c4c9e8aeab15b13105b9f3611ea9054b0b2ca9dc5565ef91a86 \
-    --hash=sha256:fc74f56d823633203330aa19e4a8be4478c194968b19c0d6272082f17cd9ae62
+kobo==0.33.0 \
+    --hash=sha256:34711f4ad0f38600bf7aafe6039b24d94458212de8b2b67c4b0c3144d47ef617 \
+    --hash=sha256:50f2eef41d1e528eba0bcdc04e6d88073353e6815abb86bf4fe3b1a7be449039
     # via pushsource
 koji==1.33.1 \
     --hash=sha256:16493a182e7194f7a5be3426a48dafb5982a450ad4cf79d14f36eb5c8a2c863d \
@@ -338,9 +338,9 @@ pubtools==1.3.0 \
     # via
     #   -r requirements.in
     #   pubtools-pulplib
-pubtools-pulplib==2.37.2 \
-    --hash=sha256:6b9b5cddc367e1bf37c8588d60a4ada0e5b079c71f18174dd70cd48f09b8aa1d \
-    --hash=sha256:c09c52910ff4d413a6da5da23922eb692e5fdac5d1694b718bf739a03959c85b
+pubtools-pulplib==2.38.0 \
+    --hash=sha256:5963cad7ec5c072e3c5bed69115bdd2092d32d1a3202889b57882bf149e9d451 \
+    --hash=sha256:83574ec3695a2963b42951fb757c6b0ee84ff6d5ed6fa726dc0de887dc44923d
     # via -r requirements.in
 pushcollector==1.3.0 \
     --hash=sha256:18281fcd4247ca8b668c25348bbc2f9cc032912cecc7037824a74ad8adcbb1c3 \
@@ -585,6 +585,5 @@ zipp==3.17.0 \
     # via importlib-resources
 
 # WARNING: The following packages were not pinned, but pip requires them to be
-# pinned when the requirements file includes hashes and the requirement is not
-# satisfied by a package already installed. Consider using the --allow-unsafe flag.
+# pinned when the requirements file includes hashes. Consider using the --allow-unsafe flag.
 # setuptools

test-requirements.txt

@@ -30,25 +30,25 @@ bandit==1.7.5 ; python_version >= "3" \
     --hash=sha256:75665181dc1e0096369112541a056c59d1c5f66f9bb74a8d686c3c362b83f549 \
     --hash=sha256:bdfc739baa03b880c2d15d0431b31c658ffc348e907fe197e54e0389dd59e11e
     # via -r test-requirements.in
-black==23.10.0 ; python_version >= "3" \
-    --hash=sha256:0e232f24a337fed7a82c1185ae46c56c4a6167fb0fe37411b43e876892c76699 \
-    --hash=sha256:30b78ac9b54cf87bcb9910ee3d499d2bc893afd52495066c49d9ee6b21eee06e \
-    --hash=sha256:31946ec6f9c54ed7ba431c38bc81d758970dd734b96b8e8c2b17a367d7908171 \
-    --hash=sha256:31b9f87b277a68d0e99d2905edae08807c007973eaa609da5f0c62def6b7c0bd \
-    --hash=sha256:47c4510f70ec2e8f9135ba490811c071419c115e46f143e4dce2ac45afdcf4c9 \
-    --hash=sha256:481167c60cd3e6b1cb8ef2aac0f76165843a374346aeeaa9d86765fe0dd0318b \
-    --hash=sha256:6901631b937acbee93c75537e74f69463adaf34379a04eef32425b88aca88a23 \
-    --hash=sha256:76baba9281e5e5b230c9b7f83a96daf67a95e919c2dfc240d9e6295eab7b9204 \
-    --hash=sha256:7fb5fc36bb65160df21498d5a3dd330af8b6401be3f25af60c6ebfe23753f747 \
-    --hash=sha256:960c21555be135c4b37b7018d63d6248bdae8514e5c55b71e994ad37407f45b8 \
-    --hash=sha256:a3c2ddb35f71976a4cfeca558848c2f2f89abc86b06e8dd89b5a65c1e6c0f22a \
-    --hash=sha256:c870bee76ad5f7a5ea7bd01dc646028d05568d33b0b09b7ecfc8ec0da3f3f39c \
-    --hash=sha256:d3d9129ce05b0829730323bdcb00f928a448a124af5acf90aa94d9aba6969604 \
-    --hash=sha256:db451a3363b1e765c172c3fd86213a4ce63fb8524c938ebd82919bf2a6e28c6a \
-    --hash=sha256:e223b731a0e025f8ef427dd79d8cd69c167da807f5710add30cdf131f13dd62e \
-    --hash=sha256:f20ff03f3fdd2fd4460b4f631663813e57dc277e37fb216463f3b907aa5a9bdd \
-    --hash=sha256:f74892b4b836e5162aa0452393112a574dac85e13902c57dfbaaf388e4eda37c \
-    --hash=sha256:f8dc7d50d94063cdfd13c82368afd8588bac4ce360e4224ac399e769d6704e98
+black==23.10.1 ; python_version >= "3" \
+    --hash=sha256:037e9b4664cafda5f025a1728c50a9e9aedb99a759c89f760bd83730e76ba884 \
+    --hash=sha256:1b917a2aa020ca600483a7b340c165970b26e9029067f019e3755b56e8dd5916 \
+    --hash=sha256:1f8ce316753428ff68749c65a5f7844631aa18c8679dfd3ca9dc1a289979c258 \
+    --hash=sha256:33d40f5b06be80c1bbce17b173cda17994fbad096ce60eb22054da021bf933d1 \
+    --hash=sha256:3f157a8945a7b2d424da3335f7ace89c14a3b0625e6593d21139c2d8214d55ce \
+    --hash=sha256:5ed45ac9a613fb52dad3b61c8dea2ec9510bf3108d4db88422bacc7d1ba1243d \
+    --hash=sha256:6d23d7822140e3fef190734216cefb262521789367fbdc0b3f22af6744058982 \
+    --hash=sha256:7670242e90dc129c539e9ca17665e39a146a761e681805c54fbd86015c7c84f7 \
+    --hash=sha256:7b4d10b0f016616a0d93d24a448100adf1699712fb7a4efd0e2c32bbb219b173 \
+    --hash=sha256:7cb5936e686e782fddb1c73f8aa6f459e1ad38a6a7b0e54b403f1f05a1507ee9 \
+    --hash=sha256:7d56124b7a61d092cb52cce34182a5280e160e6aff3137172a68c2c2c4b76bcb \
+    --hash=sha256:840015166dbdfbc47992871325799fd2dc0dcf9395e401ada6d88fe11498abad \
+    --hash=sha256:9c74de4c77b849e6359c6f01987e94873c707098322b91490d24296f66d067dc \
+    --hash=sha256:b15b75fc53a2fbcac8a87d3e20f69874d161beef13954747e053bca7a1ce53a0 \
+    --hash=sha256:cfcce6f0a384d0da692119f2d72d79ed07c7159879d0bb1bb32d2e443382bf3a \
+    --hash=sha256:d431e6739f727bb2e0495df64a6c7a5310758e87505f5f8cde9ff6c0f2d7e4fe \
+    --hash=sha256:e293e4c2f4a992b980032bbd62df07c1bcff82d6964d6c9496f2cd726e246ace \
+    --hash=sha256:ec3f8e6234c4e46ff9e16d9ae96f4ef69fa328bb4ad08198c8cee45bb1f08c69
     # via -r test-requirements.in
 certifi==2023.7.22 \
     --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 \
@@ -460,9 +460,9 @@ jsonschema-specifications==2023.7.1 \
     --hash=sha256:05adf340b659828a004220a9613be00fa3f223f2b82002e273dee62fd50524b1 \
     --hash=sha256:c91a50404e88a1f6ba40636778e2ee08f6e24c5613fe4c53ac24578a5a7f72bb
     # via jsonschema
-kobo==0.32.0 \
-    --hash=sha256:df68cec9f5412c4c9e8aeab15b13105b9f3611ea9054b0b2ca9dc5565ef91a86 \
-    --hash=sha256:fc74f56d823633203330aa19e4a8be4478c194968b19c0d6272082f17cd9ae62
+kobo==0.33.0 \
+    --hash=sha256:34711f4ad0f38600bf7aafe6039b24d94458212de8b2b67c4b0c3144d47ef617 \
+    --hash=sha256:50f2eef41d1e528eba0bcdc04e6d88073353e6815abb86bf4fe3b1a7be449039
     # via pushsource
 koji==1.33.1 \
     --hash=sha256:16493a182e7194f7a5be3426a48dafb5982a450ad4cf79d14f36eb5c8a2c863d \
@@ -611,9 +611,9 @@ pubtools==1.3.0 \
     # via
     #   -r requirements.in
     #   pubtools-pulplib
-pubtools-pulplib==2.37.2 \
-    --hash=sha256:6b9b5cddc367e1bf37c8588d60a4ada0e5b079c71f18174dd70cd48f09b8aa1d \
-    --hash=sha256:c09c52910ff4d413a6da5da23922eb692e5fdac5d1694b718bf739a03959c85b
+pubtools-pulplib==2.38.0 \
+    --hash=sha256:5963cad7ec5c072e3c5bed69115bdd2092d32d1a3202889b57882bf149e9d451 \
+    --hash=sha256:83574ec3695a2963b42951fb757c6b0ee84ff6d5ed6fa726dc0de887dc44923d
     # via -r requirements.in
 pushcollector==1.3.0 \
     --hash=sha256:18281fcd4247ca8b668c25348bbc2f9cc032912cecc7037824a74ad8adcbb1c3 \
@@ -962,9 +962,9 @@ urllib3==2.0.7 \
     # via
     #   edgegrid-python
     #   requests
-virtualenv==20.24.5 \
-    --hash=sha256:b80039f280f4919c77b30f1c23294ae357c4c8701042086e3fc005963e4e537b \
-    --hash=sha256:e8361967f6da6fbdf1426483bfe9fca8287c242ac0bc30429905721cefbff752
+virtualenv==20.24.6 \
+    --hash=sha256:02ece4f56fbf939dbbc33c0715159951d6bf14aaf5457b092e4548e1382455af \
+    --hash=sha256:520d056652454c5098a00c0f073611ccbea4c79089331f60bf9d7ba247bb7381
     # via pre-commit
 zipp==3.17.0 \
     --hash=sha256:0e923e726174922dce09c53c59ad483ff7bbb8e572e00c7f7c46b88556409f31 \
@@ -974,6 +974,5 @@ zipp==3.17.0 \
     #   importlib-resources
 
 # WARNING: The following packages were not pinned, but pip requires them to be
-# pinned when the requirements file includes hashes and the requirement is not
-# satisfied by a package already installed. Consider using the --allow-unsafe flag.
+# pinned when the requirements file includes hashes. Consider using the --allow-unsafe flag.
 # setuptools