From ce22862e72d6741f53d2342bd7cf5e75d596883d Mon Sep 17 00:00:00 2001 From: truskovskiyk Date: Sun, 1 Dec 2024 15:16:54 -0500 Subject: [PATCH] Deploy --- ai-search-demo/ai_search_demo/qdrant_inexing.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ai-search-demo/ai_search_demo/qdrant_inexing.py b/ai-search-demo/ai_search_demo/qdrant_inexing.py index 29025dc..f431a8f 100644 --- a/ai-search-demo/ai_search_demo/qdrant_inexing.py +++ b/ai-search-demo/ai_search_demo/qdrant_inexing.py @@ -178,13 +178,16 @@ def pdfs_to_hf_dataset(path_to_folder): "page_text": text }) global_index += 1 + # Print memory usage after processing each image + current, peak = tracemalloc.get_traced_memory() + print(f"IMAGE: Current memory usage is {current / 10**6}MB; Peak was {peak / 10**6}MB") # Print memory usage after processing each PDF current, peak = tracemalloc.get_traced_memory() - print(f"Current memory usage is {current / 10**6}MB; Peak was {peak / 10**6}MB") + print(f"PDF: Current memory usage is {current / 10**6}MB; Peak was {peak / 10**6}MB") current, peak = tracemalloc.get_traced_memory() - print(f"Current memory usage is {current / 10**6}MB; Peak was {peak / 10**6}MB") + print(f"TOTAL: Current memory usage is {current / 10**6}MB; Peak was {peak / 10**6}MB") tracemalloc.stop() # Stop tracing memory allocations print("Done processing")