Use OpenDAL to access object store

Signed-off-by: Enwei Jiao <[email protected]>
jiaoew1991 · Oct 24, 2023 · 4e8cc0f · 4e8cc0f
1 parent e0222b2
commit 4e8cc0f
Show file tree

Hide file tree

Showing 27 changed files with 1,075 additions and 464 deletions.
diff --git a/.github/workflows/code-checker.yaml b/.github/workflows/code-checker.yaml
@@ -35,6 +35,14 @@ jobs:
     strategy:
       fail-fast: false
     steps:
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          root-reserve-mb: 10240
+          swap-size-mb: 1024
+          remove-dotnet: 'true'
+          remove-android: 'true'
+          remove-haskell: 'true'
       - name: Checkout
         uses: actions/checkout@v2
       - name: 'Generate CCache Hash'
@@ -74,6 +82,14 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 180
     steps:
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          root-reserve-mb: 10240
+          swap-size-mb: 1024
+          remove-dotnet: 'true'
+          remove-android: 'true'
+          remove-haskell: 'true'
       - name: Checkout
         uses: actions/checkout@v2
       - name: 'Generate CCache Hash'

diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
@@ -47,6 +47,14 @@ jobs:
     env:
       UBUNTU: ${{ matrix.ubuntu }}
     steps:
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          root-reserve-mb: 10240
+          swap-size-mb: 1024
+          remove-dotnet: 'true'
+          remove-android: 'true'
+          remove-haskell: 'true'
       - name: Checkout
         uses: actions/checkout@v2
         with:
@@ -92,7 +100,7 @@ jobs:
         run: |
           ./build/builder.sh /bin/bash -c "make USE_ASAN=${{env.useasan}} build-cpp-with-coverage"
       - run: |
-          zip -r code.zip . -x "./.docker/*"
+          zip -r code.zip . -x "./.docker/*" -x "./cmake_build/thirdparty/*"
       - name: Archive code
         uses: actions/upload-artifact@v3
         with:
@@ -127,8 +135,6 @@ jobs:
         shell: bash
         run: |
           docker-compose up -d azurite
-#      - name: 'Setup upterm session'
-#        uses: lhotari/action-upterm@v1
       - name: UT
         run: |
           chmod +x build/builder.sh
@@ -173,8 +179,6 @@ jobs:
         shell: bash
         run: |
           docker-compose up -d pulsar etcd minio azurite
-#      - name: 'Setup upterm session'
-#        uses: lhotari/action-upterm@v1
       - name: UT
         run: |
           chmod +x build/builder.sh

diff --git a/configs/milvus.yaml b/configs/milvus.yaml
@@ -91,7 +91,7 @@ minio:
   # Supported level:  off, fatal, error, warn, info, debug, trace
   logLevel: fatal
   # Cloud data center region
-  region: ""
+  region: ''
   # Cloud whether use virtual host bucket mode
   useVirtualHost: false
   # timeout for request time in milliseconds
@@ -498,7 +498,7 @@ common:
     BeamWidthRatio: 4
   gracefulTime: 5000 # milliseconds. it represents the interval (in ms) by which the request arrival time needs to be subtracted in the case of Bounded Consistency.
   gracefulStopTimeout: 1800 # seconds. it will force quit the server if the graceful stop process is not completed during this time.
-  storageType: minio # please adjust in embedded Milvus: local
+  storageType: opendal # please adjust in embedded Milvus: local
   # Default value: auto
   # Valid values: [auto, avx512, avx2, avx, sse4_2]
   # This configuration is only used by querynode and indexnode, it selects CPU instruction set for Searching and Index-building.
@@ -518,9 +518,9 @@ common:
     enabled: false
     # support pre-created topics
     # the name of pre-created topics
-    names: ["topic1", "topic2"]
+    names: ['topic1', 'topic2']
     # need to set a separated topic to stand for currently consumed timestamp for each channel
-    timeticker: "timetick-channel"
+    timeticker: 'timetick-channel'
 
   ImportMaxFileSize: 17179869184 # 16 * 1024 * 1024 * 1024
   # max file size to import for bulkInsert

diff --git a/internal/core/conanfile.py b/internal/core/conanfile.py
@@ -6,7 +6,7 @@ class MilvusConan(ConanFile):
     requires = (
         "rocksdb/6.29.5",
         "boost/1.82.0",
-        "onetbb/2021.7.0",
+        "onetbb/2021.9.0",
         "nlohmann_json/3.11.2",
         "zstd/1.5.4",
         "lz4/1.9.4",

diff --git a/internal/core/src/common/CMakeLists.txt b/internal/core/src/common/CMakeLists.txt
@@ -33,6 +33,7 @@ target_link_libraries(milvus_common
         yaml-cpp
         boost_bitset_ext
         simdjson
+        opendal
         ${CONAN_LIBS}
         )
 

diff --git a/internal/core/src/indexbuilder/CMakeLists.txt b/internal/core/src/indexbuilder/CMakeLists.txt
@@ -20,17 +20,7 @@ set(INDEXBUILDER_FILES
 milvus_add_pkg_config("milvus_indexbuilder")
 add_library(milvus_indexbuilder SHARED ${INDEXBUILDER_FILES})
 
-find_library(TBB NAMES tbb)
-set(PLATFORM_LIBS dl)
-if (MSYS)
-set(PLATFORM_LIBS -Wl,--allow-multiple-definition)
-endif ()
-
 # link order matters
-target_link_libraries(milvus_indexbuilder
-        milvus_index
-        ${TBB}
-        ${PLATFORM_LIBS}
-        )
+target_link_libraries(milvus_indexbuilder milvus_index)
 
 install(TARGETS milvus_indexbuilder DESTINATION "${CMAKE_INSTALL_LIBDIR}")
diff --git a/internal/core/src/indexbuilder/index_c.cpp b/internal/core/src/indexbuilder/index_c.cpp
@@ -310,6 +310,8 @@ NewBuildIndexInfo(CBuildIndexInfo* c_build_index_info,
             std::string(c_storage_config.cloud_provider);
         storage_config.iam_endpoint =
             std::string(c_storage_config.iam_endpoint);
+        storage_config.cloud_provider =
+            std::string(c_storage_config.cloud_provider);
         storage_config.useSSL = c_storage_config.useSSL;
         storage_config.useIAM = c_storage_config.useIAM;
         storage_config.region = c_storage_config.region;