code review feedback

metadata-ingestion/src/datahub/ingestion/source/gc/execution_request_cleanup.py

@@ -141,7 +141,9 @@ def _scroll_execution_requests(
                 break
             if self.report.ergc_read_errors >= self.config.max_read_errors:
                 self.report.failure(
-                    f"ergc({self.instance_id}): too many read errors, aborting."
+                    title="Too many read errors, aborting",
+                    message="Too many read errors, aborting",
+                    context=str(self.instance_id),
                 )
                 break
             try:
@@ -159,8 +161,8 @@ def _scroll_execution_requests(
                 params["scrollId"] = document["scrollId"]
             except Exception as e:
                 self.report.failure(
-                    title="failed to fetch next batch of execution requests",
-                    message="failed to fetch next batch of execution requests",
+                    title="Failed to fetch next batch of execution requests",
+                    message="Failed to fetch next batch of execution requests",
                     context=str(self.instance_id),
                     exc=e,
                 )
@@ -235,8 +237,8 @@ def _delete_entry(self, entry: CleanupRecord) -> None:
         except Exception as e:
             self.report.ergc_delete_errors += 1
             self.report.failure(
-                title="failed to delete ExecutionRequest",
-                message="failed to delete ExecutionRequest",
+                title="Failed to delete ExecutionRequest",
+                message="Failed to delete ExecutionRequest",
                 context=str(self.instance_id),
                 exc=e,
             )

metadata-ingestion/src/datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py

@@ -105,6 +105,8 @@ class SoftDeletedEntitiesReport(SourceReport):
     sample_hard_deleted_aspects_by_type: TopKDict[str, LossyList[str]] = field(
         default_factory=TopKDict
     )
+    runtime_limit_reached: bool = False
+    deletion_limit_reached: bool = False
 
 
 class SoftDeletedEntitiesCleanup:
@@ -207,8 +209,8 @@ def _process_futures(self, futures: Dict[Future, str]) -> Dict[Future, str]:
             if future.exception():
                 self.report.failure(
                     title="Failed to delete entity",
-                    message=futures[future],
-                    context=str(futures[future]),
+                    message="Failed to delete entity",
+                    context=futures[future],
                     exc=future.exception(),
                 )
             self.report.num_soft_deleted_entity_processed += 1
@@ -280,9 +282,8 @@ def _times_up(self) -> bool:
             self.config.runtime_limit_seconds
             and time.time() - self.start_time > self.config.runtime_limit_seconds
         ):
-            logger.info(
-                f"Runtime limit of {self.config.runtime_limit_seconds} seconds reached"
-            )
+            with self._report_lock:
+                self.report.runtime_limit_reached = True
             return True
         return False
 
@@ -291,9 +292,8 @@ def _deletion_limit_reached(self) -> bool:
             self.config.limit_entities_delete
             and self.report.num_hard_deleted > self.config.limit_entities_delete
         ):
-            logger.info(
-                f"Limit of {self.config.limit_entities_delete} entities reached"
-            )
+            with self._report_lock:
+                self.report.deletion_limit_reached = True
             return True
         return False