Rename common/Utils to vector_index/helpers/Slice and (#225)

move INDEX_FILE_SLICE_SIZE_IN_MEGA to meta::SLICE_SIZE

Signed-off-by: yudong.cai <[email protected]>

knowhere/CMakeLists.txt

@@ -20,7 +20,6 @@ set(KNOWHERE_THIRDPARTY_SRC ${KNOWHERE_SOURCE_DIR}/thirdparty)
 set(external_srcs
         common/Exception.cpp
         common/Timer.cpp
-        common/Utils.cpp
         common/Log.cpp
         ${KNOWHERE_THIRDPARTY_SRC}/easyloggingpp/easylogging++.cc
         )
@@ -60,6 +59,7 @@ set(vector_index_srcs
         index/vector_index/adapter/VectorAdapter.cpp
         index/vector_index/helpers/FaissIO.cpp
         index/vector_index/helpers/IndexParameter.cpp
+        index/vector_index/helpers/Slice.cpp
         index/vector_index/ConfAdapter.cpp
         index/vector_index/ConfAdapterMgr.cpp
         index/vector_index/FaissBaseBinaryIndex.cpp

knowhere/archive/KnowhereConfig.cpp

@@ -13,8 +13,8 @@
 
 #include "archive/KnowhereConfig.h"
 #include "common/Log.h"
-#include "common/Utils.h"
 #include "index/vector_index/Statistics.h"
+#include "index/vector_index/helpers/Slice.h"
 #include "faiss/Clustering.h"
 #include "faiss/FaissHook.h"
 #include "faiss/utils/distances.h"

knowhere/index/VecIndex.h

@@ -18,10 +18,10 @@
 #include "knowhere/common/Dataset.h"
 #include "knowhere/common/Exception.h"
 #include "knowhere/common/Typedef.h"
-#include "knowhere/common/Utils.h"
 #include "knowhere/index/Index.h"
 #include "knowhere/index/IndexType.h"
 #include "knowhere/index/vector_index/Statistics.h"
+#include "knowhere/index/vector_index/helpers/Slice.h"
 #include "knowhere/utils/BitsetView.h"
 
 namespace knowhere {

knowhere/index/vector_index/helpers/IndexParameter.h

@@ -21,6 +21,7 @@ namespace knowhere {
 using MetaType = std::string_view;
 
 namespace meta {
+constexpr MetaType SLICE_SIZE = "SLICE_SIZE";
 constexpr MetaType METRIC_TYPE = "metric_type";
 constexpr MetaType DIM = "dim";
 constexpr MetaType TENSOR = "tensor";
@@ -86,14 +87,19 @@ constexpr MetricType SUPERSTRUCTURE = "SUPERSTRUCTURE";
 }  // namespace metric
 
 ///////////////////////////////////////////////////////////////////////////////
+inline bool
+CheckKeyInConfig(const Config& cfg, const std::string_view& key) {
+    return cfg.contains(std::string(key));
+}
+
 template <typename T>
-T
+inline T
 GetValueFromConfig(const Config& cfg, const std::string_view& key) {
     return cfg.at(std::string(key)).get<T>();
 }
 
 template <typename T>
-void
+inline void
 SetValueToConfig(Config& cfg, const std::string_view& key, const T value) {
     cfg[std::string(key)] = value;
 }
@@ -110,6 +116,8 @@ inline void func_name(Config& cfg, T1 value) {          \
 
 ///////////////////////////////////////////////////////////////////////////////
 // APIs to access meta
+DEFINE_CONFIG_GETTER(GetMetaSliceSize, meta::SLICE_SIZE, int64_t)
+DEFINE_CONFIG_SETTER(SetMetaSliceSize, meta::SLICE_SIZE, int64_t, int64_t)
 
 DEFINE_CONFIG_GETTER(GetMetaMetricType, meta::METRIC_TYPE, std::string)
 DEFINE_CONFIG_SETTER(SetMetaMetricType, meta::METRIC_TYPE, MetricType, std::string)

knowhere/common/Utils.cpp → knowhere/index/vector_index/helpers/Slice.cpp

@@ -13,14 +13,14 @@
 #include <memory>
 #include <vector>
 
-#include "common/Utils.h"
+#include "index/vector_index/helpers/IndexParameter.h"
+#include "index/vector_index/helpers/Slice.h"
 
 namespace knowhere {
 
-const int64_t DEFAULT_INDEX_FILE_SLICE_SIZE = 4;
-const char* INDEX_FILE_SLICE_SIZE_IN_MEGABYTE = "SLICE_SIZE";
-static const char* INDEX_FILE_SLICE_META = "SLICE_META";
+const int64_t DEFAULT_INDEX_FILE_SLICE_SIZE = 4; // megabytes
 
+static const char* INDEX_FILE_SLICE_META = "SLICE_META";
 static const char* META = "meta";
 static const char* NAME = "name";
 static const char* SLICE_NUM = "slice_num";
@@ -78,7 +78,7 @@ Assemble(BinarySet& binarySet) {
 
 void
 Disassemble(BinarySet& binarySet, const Config& config) {
-    if (!config.contains(INDEX_FILE_SLICE_SIZE_IN_MEGABYTE)) {
+    if (!CheckKeyInConfig(config, meta::SLICE_SIZE)) {
         return;
     }
 
@@ -92,7 +92,7 @@ Disassemble(BinarySet& binarySet, const Config& config) {
         }
     }
 
-    const int64_t slice_size_in_byte = config[INDEX_FILE_SLICE_SIZE_IN_MEGABYTE].get<int64_t>() << 20;
+    const int64_t slice_size_in_byte = GetMetaSliceSize(config) << 20;
     std::vector<std::string> slice_key_list;
     for (auto& kv : binarySet.binary_map_) {
         if (kv.second->size > slice_size_in_byte) {

knowhere/common/Utils.h → knowhere/index/vector_index/helpers/Slice.h

@@ -19,7 +19,6 @@
 
 namespace knowhere {
 
-extern const char* INDEX_FILE_SLICE_SIZE_IN_MEGABYTE;
 extern int64_t index_file_slice_size;
 
 void

unittest/Helper.h

@@ -50,16 +50,17 @@ class ParamGenerator {
     Gen(const knowhere::IndexType& type) {
         if (type == knowhere::IndexEnum::INDEX_FAISS_IVFFLAT) {
             return knowhere::Config{
+                {knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size},
                 {knowhere::meta::METRIC_TYPE, knowhere::metric::L2},
                 {knowhere::meta::DIM, DIM},
                 {knowhere::meta::TOPK, K},
                 {knowhere::meta::DEVICE_ID, DEVICE_ID},
                 {knowhere::indexparam::NLIST, 16},
                 {knowhere::indexparam::NPROBE, 8},
-                {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
             };
         } else if (type == knowhere::IndexEnum::INDEX_FAISS_IVFPQ) {
             return knowhere::Config{
+                {knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size},
                 {knowhere::meta::METRIC_TYPE, knowhere::metric::L2},
                 {knowhere::meta::DIM, DIM},
                 {knowhere::meta::TOPK, K},
@@ -68,22 +69,22 @@ class ParamGenerator {
                 {knowhere::indexparam::NPROBE, 8},
                 {knowhere::indexparam::M, 4},
                 {knowhere::indexparam::NBITS, 8},
-                {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
             };
         } else if (type == knowhere::IndexEnum::INDEX_FAISS_IVFSQ8 ||
                    type == knowhere::IndexEnum::INDEX_FAISS_IVFSQ8H) {
             return knowhere::Config{
+                {knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size},
                 {knowhere::meta::METRIC_TYPE, knowhere::metric::L2},
                 {knowhere::meta::DIM, DIM},
                 {knowhere::meta::TOPK, K},
                 {knowhere::meta::DEVICE_ID, DEVICE_ID},
                 {knowhere::indexparam::NLIST, 16},
                 {knowhere::indexparam::NPROBE, 8},
                 {knowhere::indexparam::NBITS, 8},
-                {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
             };
         } else if (type == knowhere::IndexEnum::INDEX_FAISS_IVFHNSW) {
             return knowhere::Config{
+                {knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size},
                 {knowhere::meta::METRIC_TYPE, knowhere::metric::L2},
                 {knowhere::meta::DIM, DIM},
                 {knowhere::meta::TOPK, K},
@@ -93,7 +94,6 @@ class ParamGenerator {
                 {knowhere::indexparam::HNSW_M, 16},
                 {knowhere::indexparam::EFCONSTRUCTION, 200},
                 {knowhere::indexparam::EF, 200},
-                {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
             };
         } else if (type == knowhere::IndexEnum::INDEX_HNSW) {
             return knowhere::Config {
@@ -106,34 +106,34 @@ class ParamGenerator {
             };
         } else if (type == knowhere::IndexEnum::INDEX_RHNSWFlat) {
             return knowhere::Config{
+                {knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size},
                 {knowhere::meta::METRIC_TYPE, knowhere::metric::L2},
                 {knowhere::meta::DIM, DIM},
                 {knowhere::meta::TOPK, K},
                 {knowhere::indexparam::HNSW_M, 16},
                 {knowhere::indexparam::EFCONSTRUCTION, 200},
                 {knowhere::indexparam::EF, 200},
-                {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
             };
         } else if (type == knowhere::IndexEnum::INDEX_RHNSWPQ) {
             return knowhere::Config{
+                {knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size},
                 {knowhere::meta::METRIC_TYPE, knowhere::metric::L2},
                 {knowhere::meta::DIM, DIM},
                 {knowhere::meta::TOPK, K},
                 {knowhere::indexparam::HNSW_M, 16},
                 {knowhere::indexparam::EFCONSTRUCTION, 200},
                 {knowhere::indexparam::EF, 200},
                 {knowhere::indexparam::PQ_M, 8},
-                {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
             };
         } else if (type == knowhere::IndexEnum::INDEX_RHNSWSQ) {
             return knowhere::Config{
+                {knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size},
                 {knowhere::meta::METRIC_TYPE, knowhere::metric::L2},
                 {knowhere::meta::DIM, DIM},
                 {knowhere::meta::TOPK, K},
                 {knowhere::indexparam::HNSW_M, 16},
                 {knowhere::indexparam::EFCONSTRUCTION, 200},
                 {knowhere::indexparam::EF, 200},
-                {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
             };
         } else {
             std::cout << "Invalid index type " << type << std::endl;

unittest/test_annoy.cpp

@@ -28,12 +28,12 @@ class AnnoyTest : public DataGen, public TestWithParam<std::string> {
         Generate(128, 10000, 10);
         index_ = std::make_shared<knowhere::IndexAnnoy>();
         conf = knowhere::Config{
+            {knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size},
             {knowhere::meta::METRIC_TYPE, knowhere::metric::L2},
             {knowhere::meta::DIM, dim},
             {knowhere::meta::TOPK, 10},
             {knowhere::indexparam::N_TREES, 4},
             {knowhere::indexparam::SEARCH_K, 100},
-            {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
         };
     }
 

unittest/test_binaryidmap.cpp

@@ -31,11 +31,11 @@ class BinaryIDMAPTest : public DataGen,
         Init_with_default(true);
 
         conf_ = knowhere::Config{
+            {knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size},
             {knowhere::meta::METRIC_TYPE, knowhere::metric::HAMMING},
             {knowhere::meta::DIM, dim},
             {knowhere::meta::TOPK, k},
             {knowhere::meta::RADIUS, radius},
-            {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
         };
         index_mode_ = GetParam();
         index_type_ = knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP;

unittest/test_idmap.cpp

@@ -171,10 +171,10 @@ TEST_P(IDMAPTest, idmap_serialize) {
 
 TEST_P(IDMAPTest, idmap_slice) {
     knowhere::Config conf{
+        {knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size},
         {knowhere::meta::METRIC_TYPE, knowhere::metric::L2},
         {knowhere::meta::DIM, dim},
         {knowhere::meta::TOPK, k},
-        {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
     };
 
     index_->BuildAll(base_dataset, conf);

unittest/test_ngtonng.cpp

@@ -31,6 +31,7 @@ class NGTONNGTest : public DataGen, public TestWithParam<std::string> {
         Generate(128, 10000, 10);
         index_ = std::make_shared<knowhere::IndexNGTONNG>();
         conf = knowhere::Config{
+            {knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size},
             {knowhere::meta::METRIC_TYPE, knowhere::metric::L2},
             {knowhere::meta::DIM, dim},
             {knowhere::meta::TOPK, 10},
@@ -39,7 +40,6 @@ class NGTONNGTest : public DataGen, public TestWithParam<std::string> {
             {knowhere::indexparam::MAX_SEARCH_EDGES, 50},
             {knowhere::indexparam::OUTGOING_EDGE_SIZE, 5},
             {knowhere::indexparam::INCOMING_EDGE_SIZE, 40},
-            {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
         };
     }
 

unittest/test_ngtpanng.cpp

@@ -31,6 +31,7 @@ class NGTPANNGTest : public DataGen, public TestWithParam<std::string> {
         Generate(128, 10000, 10);
         index_ = std::make_shared<knowhere::IndexNGTPANNG>();
         conf = knowhere::Config{
+            {knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size},
             {knowhere::meta::METRIC_TYPE, knowhere::metric::L2},
             {knowhere::meta::DIM, dim},
             {knowhere::meta::TOPK, 10},
@@ -39,7 +40,6 @@ class NGTPANNGTest : public DataGen, public TestWithParam<std::string> {
             {knowhere::indexParam::MAX_SEARCH_EDGES, 50},
             {knowhere::indexParam::FORCEDLY_PRUNED_EDGE_SIZE, 60},
             {knowhere::indexParam::SELECTIVELY_PRUNED_EDGE_SIZE, 30},
-            {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
         };
     }
 

unittest/test_nsg.cpp

@@ -56,9 +56,9 @@ class NSGInterfaceTest : public DataGen, public ::testing::Test {
         };
 
         search_conf = knowhere::Config{
+            {knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size},
             {knowhere::meta::TOPK, k},
             {knowhere::IndexParams::search_length, 30},
-            {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
         };
     }
 

unittest/test_sptag.cpp

@@ -33,17 +33,17 @@ class SPTAGTest : public DataGen, public TestWithParam<std::string> {
         index_ = std::make_shared<knowhere::CPUSPTAGRNG>(IndexType);
         if (IndexType == "KDT") {
             conf = knowhere::Config{
+                {knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size},
                 {knowhere::meta::METRIC_TYPE, knowhere::metric::L2},
                 {knowhere::meta::DIM, dim},
                 {knowhere::meta::TOPK, 10},
-                {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
             };
         } else {
             conf = knowhere::Config{
+                {knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size},
                 {knowhere::meta::METRIC_TYPE, knowhere::metric::L2},
                 {knowhere::meta::DIM, dim},
                 {knowhere::meta::TOPK, 10},
-                {knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
             };
         }