Deploy

kyryl-opens-ml · Dec 1, 2024 · ce22862 · ce22862
1 parent f8fd8be
commit ce22862
Showing 1 changed file with 5 additions and 2 deletions.
diff --git a/ai-search-demo/ai_search_demo/qdrant_inexing.py b/ai-search-demo/ai_search_demo/qdrant_inexing.py
@@ -178,13 +178,16 @@ def pdfs_to_hf_dataset(path_to_folder):
                 "page_text": text
             })
             global_index += 1
+            # Print memory usage after processing each image
+            current, peak = tracemalloc.get_traced_memory()
+            print(f"IMAGE: Current memory usage is {current / 10**6}MB; Peak was {peak / 10**6}MB")
 
         # Print memory usage after processing each PDF
         current, peak = tracemalloc.get_traced_memory()
-        print(f"Current memory usage is {current / 10**6}MB; Peak was {peak / 10**6}MB")
+        print(f"PDF: Current memory usage is {current / 10**6}MB; Peak was {peak / 10**6}MB")
 
     current, peak = tracemalloc.get_traced_memory()
-    print(f"Current memory usage is {current / 10**6}MB; Peak was {peak / 10**6}MB")
+    print(f"TOTAL: Current memory usage is {current / 10**6}MB; Peak was {peak / 10**6}MB")
     tracemalloc.stop()  # Stop tracing memory allocations
 
     print("Done processing")