delete duplicate messages in database

slatinsky · Aug 27, 2023 · 34d0bbb · 34d0bbb
1 parent a0fea70
commit 34d0bbb
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 3 deletions.
diff --git a/backend/preprocess/JsonFileStreamer.py b/backend/preprocess/JsonFileStreamer.py
@@ -96,7 +96,7 @@ def get_exported_at(self) -> str:
 					latestTimestamp = value
 
 		if latestTimestamp is not None:
-			print(f'    warning: exportedAt filend not found, using estimated value {latestTimestamp}. Use DCE v2.40.1 or newer')
+			print(f'    warning: exportedAt field not found, using estimated value {latestTimestamp}. Use DCE v2.40.1 or newer')
 			return latestTimestamp
 
 		print('    ERROR: exportedAt not found, using current time')

diff --git a/backend/preprocess/JsonProcessor.py b/backend/preprocess/JsonProcessor.py
@@ -527,8 +527,7 @@ def process(self):
 				existing_messages = self.collection_messages.find({"_id": {"$in": message_ids}})
 
 				print('        removing existing messages')
-				for existing_message in existing_messages:
-					message_ids.remove(existing_message["_id"])
+				self.collection_messages.delete_many({"_id": {"$in": message_ids}})
 
 				messages = self.merge_messages(list(messages), list(existing_messages))