[pull] main from zilliztech:main

ci/BuildCPUBuilderImage.groovy

@@ -92,9 +92,9 @@ spec:
 
     parameters{
         string(
-            description: 'os(ubuntu20.04,centos7,ubuntu18.04)',
+            description: 'os(ubuntu22.04,ubuntu20.04,centos7,ubuntu18.04)',
             name: 'os',
-            defaultValue: 'ubuntu20.04'
+            defaultValue: 'ubuntu22.04'
         )
     }
      stages {

ci/BuildGPUBuilderImage.groovy

@@ -92,9 +92,9 @@ spec:
 
     parameters{
         string(
-            description: 'os(ubuntu20.04,centos7,ubuntu18.04)',
+            description: 'os(ubuntu22.04,ubuntu20.04,centos7,ubuntu18.04)',
             name: 'os',
-            defaultValue: 'ubuntu20.04'
+            defaultValue: 'ubuntu22.04'
         )
     }
      stages {

ci/docker/builder/cpu/ubuntu22.04/Dockerfile

@@ -0,0 +1,50 @@
+FROM ubuntu:22.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV CMAKE_VERSION="v3.28"
+ENV CMAKE_TAR="cmake-3.28.4-linux-x86_64.tar.gz"
+ENV CCACHE_VERSION="v4.9.1"
+ENV CCACHE_TAR="ccache-4.9.1-linux-x86_64.tar.xz"
+
+RUN apt update \
+    && apt install -y ca-certificates apt-transport-https software-properties-common lsb-release \
+    && gpg --list-keys \
+    && gpg --no-default-keyring --keyring /usr/share/keyrings/deadsnakes.gpg \
+           --keyserver keyserver.ubuntu.com --recv-keys F23C5A6CF475977595C89F51BA6932366A755776 \
+    && echo "deb [signed-by=/usr/share/keyrings/deadsnakes.gpg] https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu \
+       $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/python.list \
+    && apt install -y --no-install-recommends wget curl git g++ gcc make gfortran swig \
+    && apt install -y python3.11 python3.11-dev python3.11-distutils \
+    && apt install -y python3-setuptools \
+    && cd /usr/bin \
+    && unlink python3 && ln -s python3.11 python3 \
+    && curl -sS https://bootstrap.pypa.io/get-pip.py | python3 \
+    && pip3 install wheel \
+    && apt remove --purge -y \
+    && rm -rf /var/lib/apt/lists/*
+
+# install cmake and ccache
+RUN cd /tmp \
+    && wget --tries=3 --retry-connrefused "https://cmake.org/files/${CMAKE_VERSION}/${CMAKE_TAR}" \
+    && tar --strip-components=1 -xz -C /usr/local -f ${CMAKE_TAR} \
+    && rm -f ${CMAKE_TAR} \
+    && wget --tries=3 --retry-connrefused "https://github.com/ccache/ccache/releases/download/${CCACHE_VERSION}/${CCACHE_TAR}" \
+    && tar -xf ${CCACHE_TAR} \
+    && cp ccache-4.9.1-linux-x86_64/ccache /usr/local/bin \
+    && rm -f ${CCACHE_TAR}
+
+# install knowhere dependancies
+RUN apt update \
+    && apt install -y libopenblas-dev libcurl4-openssl-dev libaio-dev libevent-dev lcov \
+    && pip3 install conan==1.61.0 \
+    && conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local \
+    && export PATH=$PATH:$HOME/.local/bin
+
+# clone knowhere repo and build to update .conan
+RUN git clone https://github.com/zilliztech/knowhere.git \
+    && cd knowhere \
+    && mkdir build && cd build \
+    && conan install .. --build=missing -o with_ut=True -o with_diskann=True -s compiler.libcxx=libstdc++11 -s build_type=Release \
+    && conan build .. \
+    && cd ../.. \
+    && rm -rf knowhere

ci/docker/builder/gpu/ubuntu22.04/Dockerfile

@@ -0,0 +1,50 @@
+FROM nvidia/cuda:11.8.0-devel-ubuntu22.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV CMAKE_VERSION="v3.28"
+ENV CMAKE_TAR="cmake-3.28.4-linux-x86_64.tar.gz"
+ENV CCACHE_VERSION="v4.9.1"
+ENV CCACHE_TAR="ccache-4.9.1-linux-x86_64.tar.xz"
+
+RUN apt update \
+    && apt install -y ca-certificates apt-transport-https software-properties-common lsb-release \
+    && gpg --list-keys \
+    && gpg --no-default-keyring --keyring /usr/share/keyrings/deadsnakes.gpg \
+           --keyserver keyserver.ubuntu.com --recv-keys F23C5A6CF475977595C89F51BA6932366A755776 \
+    && echo "deb [signed-by=/usr/share/keyrings/deadsnakes.gpg] https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu \
+       $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/python.list \
+    && apt install -y --no-install-recommends wget curl git g++ gcc make gfortran swig \
+    && apt install -y python3.11 python3.11-dev python3.11-distutils \
+    && apt install -y python3-setuptools \
+    && cd /usr/bin \
+    && unlink python3 && ln -s python3.11 python3 \
+    && curl -sS https://bootstrap.pypa.io/get-pip.py | python3 \
+    && pip3 install wheel \
+    && apt remove --purge -y \
+    && rm -rf /var/lib/apt/lists/*
+
+# install cmake and ccache
+RUN cd /tmp \
+    && wget --tries=3 --retry-connrefused "https://cmake.org/files/${CMAKE_VERSION}/${CMAKE_TAR}" \
+    && tar --strip-components=1 -xz -C /usr/local -f ${CMAKE_TAR} \
+    && rm -f ${CMAKE_TAR} \
+    && wget --tries=3 --retry-connrefused "https://github.com/ccache/ccache/releases/download/${CCACHE_VERSION}/${CCACHE_TAR}" \
+    && tar -xf ${CCACHE_TAR} \
+    && cp ccache-4.9.1-linux-x86_64/ccache /usr/local/bin \
+    && rm -f ${CCACHE_TAR}
+
+# install knowhere dependancies
+RUN apt update \
+    && apt install -y libopenblas-dev libcurl4-openssl-dev libaio-dev libevent-dev lcov \
+    && pip3 install conan==1.61.0 \
+    && conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local \
+    && export PATH=$PATH:$HOME/.local/bin
+
+# clone knowhere repo and build to update .conan
+RUN git clone https://github.com/zilliztech/knowhere.git \
+    && cd knowhere \
+    && mkdir build && cd build \
+    && conan install .. --build=missing -o with_ut=True -o with_diskann=True -o with_raft=True -s compiler.libcxx=libstdc++11 -s build_type=Release \
+    && conan build .. \
+    && cd ../.. \
+    && rm -rf knowhere

cmake/libs/libcardinal.cmake

@@ -1,5 +1,5 @@
 # Use short SHA1 as version
-set(CARDINAL_VERSION 0a78003 )
+set(CARDINAL_VERSION c902b74 )
 set(CARDINAL_REPO_URL "https://github.com/zilliztech/cardinal.git")
 
 set(CARDINAL_REPO_DIR "${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/cardinal")

include/knowhere/comp/knowhere_config.h

@@ -40,6 +40,17 @@ class KnowhereConfig {
     static std::string
     SetSimdType(const SimdType simd_type);
 
+    /**
+     *The purpose of this interface is: part of the sealed indexes default to using bf16 as the base data to achieve
+     *higher capacity; to ensure consistency in computation between growing and sealed, it is necessary to maintain the
+     *same precision in growing calculations as in sealed.
+     */
+    static void
+    EnablePatchForComputeFP32AsBF16();
+
+    static void
+    DisablePatchForComputeFP32AsBF16();
+
     /**
      * Set openblas threshold
      *   if nq < use_blas_threshold, calculated by omp

include/knowhere/operands.h

@@ -24,6 +24,13 @@ union fp32_bits {
     float as_value;
 };
 
+__attribute__((always_inline)) inline float
+bf16_float(float f) {
+    auto u32 = fp32_bits{.as_value = f}.as_bits;
+    // Round off
+    return fp32_bits{.as_bits = (u32 + 0x8000) & 0xFFFF0000}.as_value;
+}
+
 inline float
 fp32_from_bits(const uint32_t& w) {
     return fp32_bits{.as_bits = w}.as_value;

src/common/comp/knowhere_config.cc

@@ -90,6 +90,18 @@ KnowhereConfig::SetSimdType(const SimdType simd_type) {
     return simd_str;
 }
 
+void
+KnowhereConfig::EnablePatchForComputeFP32AsBF16() {
+    LOG_KNOWHERE_INFO_ << "Enable patch for compute fp32 as bf16";
+    faiss::enable_patch_for_fp32_bf16();
+}
+
+void
+KnowhereConfig::DisablePatchForComputeFP32AsBF16() {
+    LOG_KNOWHERE_INFO_ << "Disable patch for compute fp32 as bf16";
+    faiss::disable_patch_for_fp32_bf16();
+}
+
 void
 KnowhereConfig::SetBlasThreshold(const int64_t use_blas_threshold) {
     LOG_KNOWHERE_INFO_ << "Set faiss::distance_compute_blas_threshold to " << use_blas_threshold;

src/simd/distances_avx.cc

@@ -18,6 +18,7 @@
 #include <cassert>
 
 #include "faiss/impl/platform_macros.h"
+#include "knowhere/operands.h"
 
 namespace faiss {
 
@@ -54,6 +55,20 @@ fvec_inner_product_avx(const float* x, const float* y, size_t d) {
 }
 FAISS_PRAGMA_IMPRECISE_FUNCTION_END
 
+// trust the compiler to unroll this properly
+FAISS_PRAGMA_IMPRECISE_FUNCTION_BEGIN
+float
+fvec_inner_product_avx_bf16_patch(const float* x, const float* y, size_t d) {
+    size_t i;
+    float res = 0;
+    FAISS_PRAGMA_IMPRECISE_LOOP
+    for (i = 0; i < d; i++) {
+        res += x[i] * bf16_float(y[i]);
+    }
+    return res;
+}
+FAISS_PRAGMA_IMPRECISE_FUNCTION_END
+
 // trust the compiler to unroll this properly
 FAISS_PRAGMA_IMPRECISE_FUNCTION_BEGIN
 float
@@ -69,6 +84,20 @@ fvec_L2sqr_avx(const float* x, const float* y, size_t d) {
 }
 FAISS_PRAGMA_IMPRECISE_FUNCTION_END
 
+FAISS_PRAGMA_IMPRECISE_FUNCTION_BEGIN
+float
+fvec_L2sqr_avx_bf16_patch(const float* x, const float* y, size_t d) {
+    size_t i;
+    float res = 0;
+    FAISS_PRAGMA_IMPRECISE_LOOP
+    for (i = 0; i < d; i++) {
+        const float tmp = x[i] - bf16_float(y[i]);
+        res += tmp * tmp;
+    }
+    return res;
+}
+FAISS_PRAGMA_IMPRECISE_FUNCTION_END
+
 float
 fvec_L1_avx(const float* x, const float* y, size_t d) {
     __m256 msum1 = _mm256_setzero_ps();
@@ -187,6 +216,32 @@ fvec_inner_product_batch_4_avx(const float* __restrict x, const float* __restric
 }
 FAISS_PRAGMA_IMPRECISE_FUNCTION_END
 
+// trust the compiler to unroll this properly
+FAISS_PRAGMA_IMPRECISE_FUNCTION_BEGIN
+void
+fvec_inner_product_batch_4_avx_bf16_patch(const float* __restrict x, const float* __restrict y0,
+                                          const float* __restrict y1, const float* __restrict y2,
+                                          const float* __restrict y3, const size_t d, float& dis0, float& dis1,
+                                          float& dis2, float& dis3) {
+    float d0 = 0;
+    float d1 = 0;
+    float d2 = 0;
+    float d3 = 0;
+    FAISS_PRAGMA_IMPRECISE_LOOP
+    for (size_t i = 0; i < d; ++i) {
+        d0 += x[i] * bf16_float(y0[i]);
+        d1 += x[i] * bf16_float(y1[i]);
+        d2 += x[i] * bf16_float(y2[i]);
+        d3 += x[i] * bf16_float(y3[i]);
+    }
+
+    dis0 = d0;
+    dis1 = d1;
+    dis2 = d2;
+    dis3 = d3;
+}
+FAISS_PRAGMA_IMPRECISE_FUNCTION_END
+
 // trust the compiler to unroll this properly
 FAISS_PRAGMA_IMPRECISE_FUNCTION_BEGIN
 void
@@ -215,6 +270,34 @@ fvec_L2sqr_batch_4_avx(const float* x, const float* y0, const float* y1, const f
 }
 FAISS_PRAGMA_IMPRECISE_FUNCTION_END
 
+// trust the compiler to unroll this properly
+FAISS_PRAGMA_IMPRECISE_FUNCTION_BEGIN
+void
+fvec_L2sqr_batch_4_avx_bf16_patch(const float* x, const float* y0, const float* y1, const float* y2, const float* y3,
+                                  const size_t d, float& dis0, float& dis1, float& dis2, float& dis3) {
+    float d0 = 0;
+    float d1 = 0;
+    float d2 = 0;
+    float d3 = 0;
+    FAISS_PRAGMA_IMPRECISE_LOOP
+    for (size_t i = 0; i < d; ++i) {
+        const float q0 = x[i] - bf16_float(y0[i]);
+        const float q1 = x[i] - bf16_float(y1[i]);
+        const float q2 = x[i] - bf16_float(y2[i]);
+        const float q3 = x[i] - bf16_float(y3[i]);
+        d0 += q0 * q0;
+        d1 += q1 * q1;
+        d2 += q2 * q2;
+        d3 += q3 * q3;
+    }
+
+    dis0 = d0;
+    dis1 = d1;
+    dis2 = d2;
+    dis3 = d3;
+}
+FAISS_PRAGMA_IMPRECISE_FUNCTION_END
+
 // trust the compiler to unroll this properly
 int32_t
 ivec_inner_product_avx(const int8_t* x, const int8_t* y, size_t d) {

src/simd/distances_avx.h

@@ -21,10 +21,15 @@ namespace faiss {
 float
 fvec_L2sqr_avx(const float* x, const float* y, size_t d);
 
+float
+fvec_L2sqr_avx_bf16_patch(const float* x, const float* y, size_t d);
+
 /// inner product
 float
 fvec_inner_product_avx(const float* x, const float* y, size_t d);
 
+float
+fvec_inner_product_avx_bf16_patch(const float* x, const float* y, size_t d);
 /// L1 distance
 float
 fvec_L1_avx(const float* x, const float* y, size_t d);
@@ -40,10 +45,19 @@ void
 fvec_inner_product_batch_4_avx(const float* x, const float* y0, const float* y1, const float* y2, const float* y3,
                                const size_t d, float& dis0, float& dis1, float& dis2, float& dis3);
 
+void
+fvec_inner_product_batch_4_avx_bf16_patch(const float* x, const float* y0, const float* y1, const float* y2,
+                                          const float* y3, const size_t d, float& dis0, float& dis1, float& dis2,
+                                          float& dis3);
+
 void
 fvec_L2sqr_batch_4_avx(const float* x, const float* y0, const float* y1, const float* y2, const float* y3,
                        const size_t d, float& dis0, float& dis1, float& dis2, float& dis3);
 
+void
+fvec_L2sqr_batch_4_avx_bf16_patch(const float* x, const float* y0, const float* y1, const float* y2, const float* y3,
+                                  const size_t d, float& dis0, float& dis1, float& dis2, float& dis3);
+
 int32_t
 ivec_inner_product_avx(const int8_t* x, const int8_t* y, size_t d);