Merge pull request #613 from Bennu-Li/update_benchmark

update benchmark test script for Milvus 2.0.0-rc5

benchmark_test/scripts/README.md

@@ -1,6 +1,7 @@
 # README
 
 ## Preparation
+This project is the benchmark test based Milvus 2.0-rc5.
 
 Before running this project script, you need to start the service of milvus 2.0.
 

benchmark_test/scripts/config.py

@@ -1,8 +1,8 @@
 # from milvus import *
 import os
 
-MILVUS_HOST = "192.168.1.85"
-MILVUS_PORT = 19537
+MILVUS_HOST = "127.0.0.1"
+MILVUS_PORT = 19530
 
 ##################### Collection Parameters ########################################################
 
@@ -12,7 +12,7 @@
 ##################### Indexing Parameters ##########################################################
 
 # index IVF parameters
-NLIST = 2000
+NLIST = 4096
 PQ_M = 12
 
 # index NSG parameters
@@ -44,8 +44,8 @@
 # Does the data need to be normalized before insertion
 IF_NORMALIZE = False
 # If dealing with bvecs or fvecs files. Import chunk size must be <= 256mb
-TOTAL_VECTOR_COUNT = 20000
-IMPORT_CHUNK_SIZE = 20000
+TOTAL_VECTOR_COUNT = 1000000
+IMPORT_CHUNK_SIZE = 100000
 
 ##################### Performance Test Parameters ##################################################
 

benchmark_test/scripts/main.py

@@ -6,23 +6,24 @@
 from load import insert_data, create_index
 
 
-
 def main():
     try:
         opts, args = getopt.getopt(
             sys.argv[1:],
             "hc",
-            ["help", "collection=", "dim=", "index_type=", "percentile=", "create", "insert", "create_index", "performance", "index_info", "describe",
-             "show", "has", "rows", "describe_index", "drop", "drop_index", "version", "percentile_test","release",
-             "search_param=", "recall", "partition_name=", "create_partition", "load", "load_progress", "index_progress"]
+            ["help", "collection=", "dim=", "index_type=", "percentile=", "create", "insert", "create_index",
+             "performance", "index_info", "describe", "list", "has", "rows", "describe_index", "drop", "drop_index",
+             "version", "percentile_test", "release", "search_param=", "recall", "partition_name=", "create_partition",
+             "load", "load_progress", "index_progress", "calculate"]
         )
     except getopt.GetoptError:
         print("Usage: python milvus_toolkindex_type.py -q <nq> -k <topk> -c <collection> -s")
         sys.exit(2)
 
     for opt_name, opt_value in opts:
         if opt_name in ("-h", "--help"):
-            print("For parameter descriptions, please refer to https://github.com/milvus-io/bootcamp/tree/master/benchmark_test/scripts")
+            print(
+                "For parameter descriptions, please refer to https://github.com/milvus-io/bootcamp/tree/master/benchmark_test/scripts")
             sys.exit(2)
 
         elif opt_name == "--collection":
@@ -34,7 +35,7 @@ def main():
 
         elif opt_name == "--search_param":
             search_param = int(opt_value)
-            
+
         elif opt_name == "--percentile":
             percentile = int(opt_value)
 
@@ -70,13 +71,13 @@ def main():
             client = MilvusHelper()
             performance(client, collection_name, search_param)
             sys.exit(2)
-            
+
         elif opt_name == "--percentile_test":
             client = MilvusHelper()
             percentile_test(client, collection_name, search_param, percentile)
             sys.exit(2)
-                    
-  
+
+
         # save search result 
         elif opt_name == "--recall":
             client = MilvusHelper()
@@ -98,13 +99,13 @@ def main():
             print(client.get_index_params(collection_name))
             sys.exit(2)
 
-        #Show if collection exists
+        # Show if collection exists
         elif opt_name == "--has":
             client = MilvusHelper()
             print(client.has_collection(collection_name))
             sys.exit(2)
 
-        #Get collection row count
+        # Get collection row count
         elif opt_name == "--rows":
             client = MilvusHelper()
             print(client.count(collection_name))
@@ -122,33 +123,35 @@ def main():
             client = MilvusHelper()
             client.delete_index(collection_name)
             sys.exit(2)
-            
+
         elif opt_name == "--load":
             client = MilvusHelper()
             client.load_data(collection_name)
             sys.exit(2)
-            
-        elif opt_name == "--show":
+
+        elif opt_name == "--list":
             client = MilvusHelper()
-            print(client.show_collection())
+            print(client.list_collection())
             sys.exit(2)
-            
+
         elif opt_name == "--index_progress":
             client = MilvusHelper()
             print(client.get_index_progress(collection_name))
             sys.exit(2)
-        
+
         elif opt_name == "--load_progress":
             client = MilvusHelper()
             print(client.get_loading_progress(collection_name))
             sys.exit(2)
-            
+
         elif opt_name == "--release":
             client = MilvusHelper()
-            print(client.release_mem(collection_name))
+            print(client.release_data(collection_name))
             sys.exit(2)
-
-
+
+        # elif opt_name == "--calculate":
+        #     client = MilvusHelper()
+        #     print(client.calculate_distance(vectors_left, vectors_right))
 
 
 if __name__ == '__main__':

benchmark_test/scripts/milvus_helpers.py

@@ -1,10 +1,7 @@
 import sys
-from pymilvus_orm import connections, Index
-from pymilvus_orm.types import DataType
-from pymilvus_orm.schema import FieldSchema, CollectionSchema
-from pymilvus_orm.collection import Collection
+from pymilvus import connections, FieldSchema, CollectionSchema, DataType, Collection, utility
+
 from config import MILVUS_HOST, MILVUS_PORT, VECTOR_DIMENSION, METRIC_TYPE, NLIST
-from pymilvus_orm import utility
 from logs import LOGGER
 
 
@@ -25,7 +22,7 @@ def set_collection(self, collection_name):
             else:
                 raise Exception("There has no collection named:{}".format(collection_name))
         except Exception as e:
-            LOGGER.error("Failed to set collection: {}".format(e))
+            LOGGER.error("ERROR: {}".format(e))
             sys.exit(1)
 
     # Return if Milvus has the collection
@@ -94,8 +91,9 @@ def get_index_params(self, collection_name):
     # Delete Milvus collection
     def delete_collection(self, collection_name):
         try:
-            self.set_collection(collection_name)
-            self.collection.drop()
+            # self.set_collection(collection_name)
+            # self.collection.drop()
+            utility.drop_collection(collection_name)
             LOGGER.debug("Successfully drop collection!")
             return "ok"
         except Exception as e:
@@ -126,6 +124,7 @@ def count(self, collection_name):
             LOGGER.error("Failed to count vectors in Milvus: {}".format(e))
             sys.exit(1)
 
+    # create a partition for Milvus
     def create_partition(self, collection_name, partition_name):
         self.set_collection(collection_name)
         if self.collection.has_partition(partition_name):
@@ -134,23 +133,33 @@ def create_partition(self, collection_name, partition_name):
             partition = self.collection.create_partition(partition_name)
             return partition
 
+    # drop index
     def delete_index(self, collection_name):
         self.set_collection(collection_name)
         self.collection.drop_index()
-
+
+    # load data from disk to memory
     def load_data(self, collection_name):
         self.set_collection(collection_name)
         self.collection.load()
-
-    def show_collection(self):
+
+    # List all collections.
+    def list_collection(self):
         return utility.list_collections()
-
+
+    # Query the progress of loading.
     def get_loading_progress(self, collection_name):
         return utility.loading_progress(collection_name)
-
+
+    # Query the progress of index building.
     def get_index_progress(self, collection_name):
         return utility.index_building_progress(collection_name)
-
-    def release_mem(self, collection_name):
+
+    # release collection data from memory
+    def release_data(self, collection_name):
         self.set_collection(collection_name)
         return self.collection.release()
+
+    # Calculate distance between two vector arrays.
+    def calculate_distance(self, vectors_left, vectors_right):
+        return utility.calc_distance(vectors_left, vectors_right)

benchmark_test/scripts/requirements.txt

@@ -1 +1 @@
-pymilvus-orm==2.0.0rc1
+pymilvus==2.0.0rc5