fix the missing std header files

.bazelrc

@@ -50,12 +50,11 @@ build --action_env PYTHON_BIN_PATH="/usr/bin/python3"
 build --workspace_status_command "sh ./package/git_commit/git_info.sh"
 build --cxxopt="-std=c++17" --cxxopt="-Wno-register"
 build --config=old_abi
-build --config=havenask
 build:old_abi --copt="-D_GLIBCXX_USE_CXX11_ABI=0" --define use_abi=0
 build:clang --action_env=CC=/usr/bin/clang
 build:clang --action_env=CXX=/usr/bin/clang++
 build:clang --define compiler_type=clang++
-build:clang --repo_env=BAZEL_CXXOPTS="-fcoroutines-ts"
+build:clang --action_env=BAZEL_CXXOPTS="-fcoroutines-ts"
 build:clang --copt -Werror,-Wunused-result
 build:clang --copt -Wall
 build:clang --copt -Wextra
@@ -88,6 +87,19 @@ build:clang --copt -Werror
 build:clang --copt -fno-limit-debug-info
 build:clang --linkopt -fuse-ld=lld
 build:clang --linkopt -lstdc++
+build:asan --copt -fsanitize=address
+build:asan --copt -DADDRESS_SANITIZER
+build:asan --copt -DFL_ASAN_ENABLED
+build:asan --copt -fno-omit-frame-pointer
+build:asan --copt -fPIC # for "fix relocation truncated to fit: R_X86_64_32 against `.debug_info'" collect2 error
+build:asan --copt -fdebug-types-section # for "fix relocation truncated to fit: R_X86_64_32 against `.debug_info'" collect2 error
+build:asan --linkopt -fsanitize=address
 build:havenask --define repo_mode=aios_open_source
 build:havenask --copt -DAIOS_OPEN_SOURCE
+build:havenask --build_tag_filters=-not_for_opensource
+build:havenask --test_tag_filters=-not_for_opensource
+build:havenask --define disable_catalog_mongodb=true
 test --test_tag_filters=-slow_case,-codegen_case,-one_box --test_env CHECK_CASE_TIMEOUT=false
+build:havenask --distdir=/external/__distdir__
+build:havenask --define cpu_avx512=false
+build --config=havenask

BUILD

@@ -1,7 +1,6 @@
 load(
     '@bazel_tools//tools/jdk:default_java_toolchain.bzl',
-    'default_java_toolchain', 'VANILLA_TOOLCHAIN_CONFIGURATION',
-    'JDK8_JVM_OPTS', 'DEFAULT_JAVACOPTS'
+    'default_java_toolchain', 'VANILLA_TOOLCHAIN_CONFIGURATION'
 )
 load('@bazel_skylib//rules:common_settings.bzl', 'bool_flag')
 default_java_toolchain(
@@ -17,6 +16,7 @@ default_java_toolchain(
 config_setting(name='arm_cpu', values={'cpu': 'aarch64'})
 config_setting(name='clang_build', values={'define': 'compiler_type=clang++'})
 config_setting(name='enable_rdma', values={'copt': '-DUSE_RDMA=1'})
+config_setting(name='enable_solar', values={'copt': '-DACCL_USE_SOLAR=1'})
 config_setting(
     name='new_abi_mode',
     define_values={'use_abi': '1'},
@@ -60,6 +60,21 @@ config_setting(
     define_values={'repo_mode': 'aios_open_source'},
     visibility=['//visibility:public']
 )
+config_setting(
+    name='cpu_with_avx512',
+    define_values={'cpu_avx512': 'true'},
+    visibility=['//visibility:public']
+)
+config_setting(
+    name='cpu_without_avx512',
+    define_values={'cpu_avx512': 'false'},
+    visibility=['//visibility:public']
+)
+config_setting(
+    name='hack_get_set_env',
+    define_values={'hack_get_set_env': 'true'},
+    visibility=['//visibility:public']
+)
 bool_flag(name='using_cuda', build_setting_default=False)
 bool_flag(name='using_ppu', build_setting_default=False)
 config_setting(
@@ -86,6 +101,12 @@ config_setting(
     },
     visibility=['//visibility:public']
 )
+
+config_setting(
+    name='disable_catalog_mongodb',
+    values={'define': 'disable_catalog_mongodb=true'},
+    visibility=['//visibility:public']
+)
 load(
     '@io_bazel_rules_docker//container:container.bzl', 'container_layer',
     'container_image'

README.md

@@ -1,5 +1,7 @@
 ## Note
-> old version：[main-stable-0.4](https://github.com/alibaba/havenask/tree/main-stable-0.4)
+> old version:
+> [1.0.0-beta](https://github.com/alibaba/havenask/tree/1.0.0-beta),
+> [main-stable-0.4](https://github.com/alibaba/havenask/tree/main-stable-0.4)<br>
 > The new and old main branches have a significant difference, and if the sync code branch produces errors, please make them consistent with the current main branch.
 
 ## Introduction
@@ -32,67 +34,48 @@ Havenask delivers the following benefits:
 
 * Havenask Docker images (image platform: amd64)
   * <strong>ha3_runtime</strong>: runtime image allows you to quickly start a search service without compiling.
-  * <strong>ha3_dev </strong>: development image includes all libraries necessary for compiling havenask. （[Wiki: Compile Code](https://github.com/alibaba/havenask/wiki/Get-Started-en-1.0.0-beta#compile-code)）
+  * <strong>ha3_dev </strong>: development image includes all libraries necessary for compiling havenask. （[QuickStart](docs/havenask_docs/快速开始.md)）
 * Requirements
   * Runtime: cpu > 2 cores, memory > 4G, disk > 20G
   * Dev: cpu > 2 cores, memory > 10G, disk > 50G
   * Install and Start Docker
+  * Check that you can ssh to the localhost without a passphrase, ref[passphraseless ssh](docs/havenask_docs/Hape单机模式.md)
 
-
-
-
-### Run the container
-
-* clone the repository
+## Start Havenask Service
+* Create the container
+CONTAINER_NAME specifies the name of the container.
 ```
 cd ~
 git clone [email protected]:alibaba/havenask.git
 cd ~/havenask/docker/havenask
+docker pull registry.cn-hangzhou.aliyuncs.com/havenask/ha3_runtime:latest
+./create_container.sh <CONTAINER_NAME> registry.cn-hangzhou.aliyuncs.com/havenask/ha3_runtime:latest
 ```
 
-* create the container. CONTAINER_NAME specifies the name of the container.
+* Log on to the container
 ```
-docker pull havenask/ha3_runtime:1.0.0-beta2
-./create_container.sh <CONTAINER_NAME> havenask/ha3_runtime:1.0.0-beta2
-```
-* If the connection to Docker Hub is unstable, you can try to download the images from Alibaba Cloud Container Registry.
-```
-docker pull registry.cn-hangzhou.aliyuncs.com/havenask/ha3_runtime:1.0.0-beta2
-./create_container.sh <CONTAINER_NAME> registry.cn-hangzhou.aliyuncs.com/havenask/ha3_runtime:1.0.0-beta2
+./<CONTAINER_NAME>/sshme
 ```
 
-
-
-* Log on to the container.
-
+* Start Havneask service
 ```
-./<CONTAINER_NAME>/sshme
+/ha3_install/hape start havenask
 ```
-
-
-
-### Build Demo Index
+* Create table
 ```
-cd ~/havenask/example/
-python build_demo_data.py /ha3_install
+/ha3_install/hape create table -t in0 -s /ha3_install/hape_conf/example/cases/normal/in0_schema.json -p 1
 ```
 
-### Query Demo Index
-* start havenask
+* Insert data
 ```
-python start_demo_searcher.py /ha3_install
+/ha3_install/sql_query.py --query "insert into in0 (createtime,hits,id,title,subject)values(1,2,4,'test', 'test')"
 ```
-* havenask container listens on the default port of 45800. Here are some examples. ([More examples](https://github.com/alibaba/havenask/tree/main/example))
-
-
+* Query
 ```
-python curl_http.py 45800 "query=select count(*) from in0&&kvpair=databaseName:in0;timeout:2000"
-
-python curl_http.py 45800 "query=select id,hits from in0 where MATCHINDEX('title', '搜索词典')&&kvpair=databaseName:in0;timeout:2000"
-
-python curl_http.py 45800 "query=select title, subject from in0_summary_ where id=1 or id=2&&kvpair=databaseName:in0;timeout:2000"
+/ha3_install/sql_query.py --query "select * from in0"
 ```
 
+
 ## Contact Us
 Havenask DingTalk Group：
 
@@ -127,56 +110,46 @@ Havenask 的核心能力与优势，有以下几点：
 ## 开始使用
 * 镜像介绍（镜像平台: amd64）
    * ha3_runtime：可直接运行的镜像，无需编译代码，包含了问天引擎的可执行文件。
-   * ha3_dev：用于开发测试的镜像，里面包含了开发测试时需要的各种依赖库和头文件，如何编译问天引擎请参考[编译代码](https://github.com/alibaba/havenask/wiki/%E5%BF%AB%E9%80%9F%E5%BC%80%E5%A7%8B-1.0.0-beta)。
+   * ha3_dev：用于开发测试的镜像，里面包含了开发测试时需要的各种依赖库和头文件，如何编译问天引擎请参考[快速开始](docs/havenask_docs/快速开始.md)。
 * 环境要求
    * 运行镜像：确保机器内存大于4G，cpu大于2核，磁盘大小大于20G。
    * 开发镜像：确保机器内存大于10G，cpu大于2核，磁盘大小大于50G。
    * 使用前确保设备已经安装和启动Docker服务。
+   * 单机模式需要确保本机能免密ssh登录自己，详情参考[免密ssh](docs/havenask_docs/Hape单机模式.md)
+
+## 启动服务
 
-### 启动容器
-克隆仓库
+* 创建容器
+其中CONTAINER_NAME为指定的容器名
 ```
 cd ~
 git clone [email protected]:alibaba/havenask.git
 cd ~/havenask/docker/havenask
+docker pull registry.cn-hangzhou.aliyuncs.com/havenask/ha3_runtime:latest
+./create_container.sh <CONTAINER_NAME> registry.cn-hangzhou.aliyuncs.com/havenask/ha3_runtime:latest
 ```
-创建容器，其中CONTAINER_NAME为指定的容器名
-```
-docker pull havenask/ha3_runtime:1.0.0-beta2
-./create_container.sh <CONTAINER_NAME> havenask/ha3_runtime:1.0.0-beta2
-```
-如果由于Docker Hub访问不稳定无法下载以上镜像，可以尝试阿里云镜像源
-```
-docker pull registry.cn-hangzhou.aliyuncs.com/havenask/ha3_runtime:1.0.0-beta2
-./create_container.sh <CONTAINER_NAME> registry.cn-hangzhou.aliyuncs.com/havenask/ha3_runtime:1.0.0-beta2
-```
-登陆容器
+
+* 登陆容器
 ```
 ./<CONTAINER_NAME>/sshme
 ```
 
-### 测试索引构建
-
-构建全量索引
+* 启动havneask服务
 ```
-cd ~/havenask/example/
-python build_demo_data.py /ha3_install
+/ha3_install/hape start havenask
 ```
-
-### 测试引擎查询
-启动havenask引擎
+* 创建表
 ```
-python start_demo_searcher.py /ha3_install
+/ha3_install/hape create table -t in0 -s /ha3_install/hape_conf/example/cases/normal/in0_schema.json -p 1
 ```
 
-引擎的默认查询端口为45800，使用脚本进行查询测试。下面是一些测试query。更多测试case见[example](https://github.com/alibaba/havenask/tree/main/example)
-
+* 写入数据
 ```
-python curl_http.py 45800 "query=select count(*) from in0&&kvpair=databaseName:in0;timeout:2000"
-
-python curl_http.py 45800 "query=select id,hits from in0 where MATCHINDEX('title', '搜索词典')&&kvpair=databaseName:in0;timeout:2000"
-
-python curl_http.py 45800 "query=select title, subject from in0_summary_ where id=1 or id=2&&kvpair=databaseName:in0;timeout:2000"
+/ha3_install/sql_query.py --query "insert into in0 (createtime,hits,id,title,subject)values(1,2,4,'测试', '测试')"
+```
+* 查询数据
+```
+/ha3_install/sql_query.py --query "select * from in0"
 ```
 
 ## 联系我们

WORKSPACE

@@ -1,18 +1,3 @@
-
 workspace(name='com_taobao_aios')
 load('//:workspace.bzl', 'aios_workspace')
 aios_workspace()
-load('@com_github_mjbots_bazel_deps//tools/workspace:default.bzl', 'add_default_repositories')
-add_default_repositories()
-load('@rules_foreign_cc//:workspace_definitions.bzl', 'rules_foreign_cc_dependencies')
-rules_foreign_cc_dependencies()
-load('@org_tensorflow//tensorflow:version_check.bzl', 'check_bazel_version_at_least')
-check_bazel_version_at_least('0.24.1')
-load('@org_tensorflow//tensorflow:workspace.bzl', 'tf_workspace')
-tf_workspace(path_prefix='', tf_repo_name='org_tensorflow')
-load('@tvm//:workspace.bzl', 'tvm_workspace')
-tvm_workspace()
-load('@rules_pkg//:deps.bzl', 'rules_pkg_dependencies')
-rules_pkg_dependencies()
-load('@bazel_skylib//:workspace.bzl', 'bazel_skylib_workspace')
-bazel_skylib_workspace()

aios/aios/common/beeper/havenask_mock/common/common_type.h

@@ -20,7 +20,7 @@
 #include <sys/time.h>
 #include "autil/legacy/jsonizable.h"
 
-namespace beeper { 
+namespace beeper {
 
 enum EventLevel
 {
@@ -44,7 +44,7 @@ class EventTags
 public:
     typedef std::map<std::string, std::string> TagMap;
     typedef TagMap::const_iterator TagMapIter;
-    
+
 public:
     EventTags(const TagMap& _kvMap) {}
     EventTags() {}
@@ -56,9 +56,12 @@ class EventTags
     void MergeTags(EventTags* tags) const
     {
     }
-
-private:
-    TagMap mKVMap;
+    const std::string& FindTag(const std::string &k) const {
+        static std::string emptyStr;
+        return emptyStr;
+    }
+    void DelTag(const std::string& k) { }
+    size_t Size() const { return 0; }
 };
 typedef std::shared_ptr<EventTags> EventTagsPtr;
 

aios/alog/include/alog/Logger.h

@@ -122,6 +122,8 @@ class Logger {
     void
     logVaList(uint32_t level, const char *filename, int line, const char *function, const char *format, va_list ap);
 
+    void logVaList(uint32_t level, const char *format, va_list ap);
+
     /**
      *@brief output a message like printf() function in c language, and output the __FILE__, __FUNCTION__, __LINE__
      *informations

aios/alog/src/cpp/Appender.cpp

@@ -681,7 +681,7 @@ void* FileAppender :: compressHook(void *p)
     }
     if(cacheLimit > 0)
     {
-        if(posix_memalign(&tmp, 512, CHUNK) != 0)
+        if(posix_memalign(&tmp, stbuf.st_blksize, CHUNK) != 0)
         {
             fprintf(stderr, "posix_memalignes error: %s\n", strerror(errno));
             ::close(source);

aios/alog/src/cpp/Logger.cpp

@@ -217,7 +217,21 @@ void Logger::removeAllAppenders()
         m_appenderSet.clear();
     }
 }
+void Logger::logVaList(uint32_t level, const char *format, va_list ap) {
+    if (__builtin_expect((!isLevelEnabled(level)),1))
+        return;
 
+    char *buffer = getTLSBufferStorage<Logger>();
+    if (!buffer) {
+        return;
+    }
+    vsnprintf(buffer, MAX_MESSAGE_LENGTH, format, ap);
+    std::string msg(buffer);
+    LoggingEvent event(m_loggerName, msg, level);
+
+    _log(event);
+
+}
 void Logger::logVaList(uint32_t level, const char * file, int line, const char * func, const char* fmt, va_list ap)
 {
     if (__builtin_expect((!isLevelEnabled(level)),1))