replace diskann distance function by faiss distance function (#634)

Signed-off-by: cqy123456 <[email protected]>
milvus-io · Jan 13, 2023 · 9a9aac5 · 9a9aac5
1 parent 747cd46
commit 9a9aac5
Show file tree

Hide file tree

Showing 16 changed files with 150 additions and 1,218 deletions.
diff --git a/knowhere/index/vector_index/IndexDiskANN.cpp b/knowhere/index/vector_index/IndexDiskANN.cpp
@@ -536,6 +536,4 @@ IndexDiskANN<T>::GetCachedNodeNum(const float cache_dram_budget, const uint64_t
 
 // Explicit template instantiation
 template class IndexDiskANN<float>;
-template class IndexDiskANN<uint8_t>;
-template class IndexDiskANN<int8_t>;
 }  // namespace knowhere
diff --git a/knowhere/index/vector_index/IndexDiskANN.h b/knowhere/index/vector_index/IndexDiskANN.h
@@ -24,8 +24,7 @@ namespace knowhere {
 
 template <typename T>
 class IndexDiskANN : public VecIndex {
-    static_assert(std::is_same_v<T, float> || std::is_same_v<T, int8_t> || std::is_same_v<T, uint8_t>,
-                  "DiskANN only support float, int8 and uint8");
+    static_assert(std::is_same_v<T, float>, "DiskANN only support float");
 
  public:
     explicit IndexDiskANN(std::string index_prefix, MetricType metric_type, std::shared_ptr<FileManager> file_manager);

diff --git a/python/knowhere/__init__.py b/python/knowhere/__init__.py
@@ -49,13 +49,8 @@ def CreateIndexDiskANN(index_name, index_prefix, metric_type, simd_type="auto"):
 
     if index_name == "diskann_f":
         return buildDiskANNf(index_prefix, metric_type)
-    if index_name == "diskann_i8":
-        return buildDiskANNi8(index_prefix, metric_type)
-    if index_name == "diskann_ui8":
-        return buildDiskANNui8(index_prefix, metric_type)
     raise ValueError(
-        """ index name only support 
-            'diskann_f' 'diskann_i8' 'diskann_ui8'."""
+        """ index name only support 'diskann_f'. """
     )
 
 
@@ -67,15 +62,15 @@ def CreateAsyncIndex(index_name, index_prefix="", metric_type="", simd_type="aut
 
     if index_name not in ["bin_flat", "bin_ivf_flat", "flat", "ivf_flat", "ivf_pq", "ivf_sq8",
                           "hnsw", "annoy", "gpu_flat", "gpu_ivf_flat",
-                          "gpu_ivf_pq", "gpu_ivf_sq8", "diskann_f", "diskann_i8", "diskann_ui8"]:
+                          "gpu_ivf_pq", "gpu_ivf_sq8", "diskann_f"]:
         raise ValueError(
             """ index name only support
             'bin_flat', 'bin_ivf_flat', 'flat', 'ivf_flat', 'ivf_pq', 'ivf_sq8',
             'hnsw', 'annoy', 'gpu_flat', 'gpu_ivf_flat',
-            'gpu_ivf_pq', 'gpu_ivf_sq8', 'diskann_f', 'diskann_i8', 'diskann_ui8'."""
+            'gpu_ivf_pq', 'gpu_ivf_sq8', 'diskann_f'."""
         )
 
-    if index_name in ["diskann_f", "diskann_i8", "diskann_ui8"]:
+    if index_name == "diskann_f":
         if index_prefix == "":
             raise ValueError("Must pass index_prefix to DiskANN")
         if metric_type == "":

diff --git a/python/knowhere/knowhere.i b/python/knowhere/knowhere.i
@@ -77,8 +77,6 @@ import_array();
 %template(DatasetPtr) std::shared_ptr<Dataset>;
 #ifdef KNOWHERE_WITH_DISKANN
 %shared_ptr(knowhere::IndexDiskANN<float>)
-%shared_ptr(knowhere::IndexDiskANN<int8_t>)
-%shared_ptr(knowhere::IndexDiskANN<uint8_t>)
 #endif
 %shared_ptr(knowhere::AsyncIndex)
 %include <common/Dataset.h>
@@ -130,8 +128,6 @@ import_array();
 // Support for DiskANN
 #ifdef KNOWHERE_WITH_DISKANN
 %template(IndexDiskANNf) knowhere::IndexDiskANN<float>;
-%template(IndexDiskANNi8) knowhere::IndexDiskANN<int8_t>;
-%template(IndexDiskANNui8) knowhere::IndexDiskANN<uint8_t>;
 
 %inline %{
 
@@ -145,20 +141,6 @@ buildDiskANNf(std::string index_prefix, std::string metric_type) {
             std::shared_ptr<knowhere::FileManager>(new knowhere::LocalFileManager));
 }
 
-std::shared_ptr<knowhere::IndexDiskANN<int8_t>>
-buildDiskANNi8(std::string index_prefix, std::string metric_type) {
-    TOUPPER(metric_type);
-    return std::make_shared<knowhere::IndexDiskANN<int8_t>>(index_prefix, metric_type,
-            std::shared_ptr<knowhere::FileManager>(new knowhere::LocalFileManager));
-}
-
-std::shared_ptr<knowhere::IndexDiskANN<uint8_t>>
-buildDiskANNui8(std::string index_prefix, std::string metric_type) {
-    TOUPPER(metric_type);
-    return std::make_shared<knowhere::IndexDiskANN<uint8_t>>(index_prefix, metric_type,
-            std::shared_ptr<knowhere::FileManager>(new knowhere::LocalFileManager));
-}
-
 %}
 #endif
 

diff --git a/thirdparty/DiskANN/CMakeLists.txt b/thirdparty/DiskANN/CMakeLists.txt
@@ -217,8 +217,8 @@ else()
 	set(ENV{TCMALLOC_LARGE_ALLOC_REPORT_THRESHOLD} 500000000000)
     #	set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g -DDEBUG -O0 -fsanitize=address -fsanitize=leak -fsanitize=undefined")
     set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g -DDEBUG -Wall -Wextra")
-	set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -Ofast -DELPP_DISABLE_DEBUG_LOGS -DNDEBUG -march=skylake -ftree-vectorize")
-	add_compile_options(-march=skylake -Wall -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free -fopenmp -fopenmp-simd -funroll-loops -Wfatal-errors -DUSE_AVX2)
+	set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -Ofast -DELPP_DISABLE_DEBUG_LOGS -DNDEBUG -ftree-vectorize")
+	add_compile_options(-Wall -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free -fopenmp -fopenmp-simd -funroll-loops -Wfatal-errors)
 endif()
 
 add_definitions( -DAUTO_INITIALIZE_EASYLOGGINGPP )

diff --git a/thirdparty/DiskANN/include/cosine_similarity.h b/thirdparty/DiskANN/include/cosine_similarity.h