diff --git a/.circleci/config.yml b/.circleci/config.yml index 534544d1b..6b7a4396e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,8 +21,8 @@ workflows: jobs: pull-request-check: - docker: - - image: vsaglib/vsag:ci-x86 + machine: + image: ubuntu-2204:2025.09.1 resource_class: large steps: - checkout @@ -30,26 +30,56 @@ jobs: keys: - fork-cache-{{ checksum "CMakeLists.txt" }}-{{ checksum ".circleci/fresh_ci_cache.commit" }} - run: - command: export CMAKE_GENERATOR="Ninja" && make test_parallel + name: prepare_env_and_create_swap_file + command: | + sudo apt update + sudo apt install -y gfortran python3-dev libomp-15-dev gcc make cmake g++ lcov libaio-dev libcurl4-openssl-dev ninja-build + sudo apt-get install -y libopenblas-dev + sudo dd if=/dev/zero of=.swapfile bs=2M count=2048 + sudo chmod 600 .swapfile + sudo mkswap .swapfile + sudo swapon .swapfile + - run: + command: export CMAKE_GENERATOR="Ninja" && export VSAG_ENABLE_INTEL_MKL=OFF && export COMPILE_JOBS=4 && make test_parallel no_output_timeout: 50m + - run: + name: remove_swap_file + command: | + sudo swapoff .swapfile + sudo rm .swapfile - save_cache: key: fork-cache-{{ checksum "CMakeLists.txt" }}-{{ checksum ".circleci/fresh_ci_cache.commit" }} paths: - ./build main-branch-check: - docker: - - image: vsaglib/vsag:ci-x86 + machine: + image: ubuntu-2204:2025.09.1 resource_class: large steps: - checkout - restore_cache: keys: - - main-ccache-{{ checksum "CMakeLists.txt" }}-{{ checksum ".circleci/fresh_ci_cache.commit" }} + - main-cache-{{ checksum "CMakeLists.txt" }}-{{ checksum ".circleci/fresh_ci_cache.commit" }} - run: - command: export CMAKE_GENERATOR="Ninja" && make test_parallel + name: prepare_env_and_create_swap_file + command: | + sudo apt update + sudo apt install -y gfortran python3-dev libomp-15-dev gcc make cmake g++ lcov libaio-dev libcurl4-openssl-dev ninja-build + sudo apt-get install -y libopenblas-dev + sudo dd if=/dev/zero of=.swapfile bs=2M count=2048 + sudo chmod 600 .swapfile + sudo mkswap .swapfile + sudo swapon .swapfile + - run: + command: export CMAKE_GENERATOR="Ninja" && export VSAG_ENABLE_INTEL_MKL=OFF && export COMPILE_JOBS=4 && make test_parallel no_output_timeout: 50m + - run: + name: remove_swap_file + command: | + sudo swapoff .swapfile + sudo rm .swapfile - save_cache: - key: main-ccache-{{ checksum "CMakeLists.txt" }}-{{ checksum ".circleci/fresh_ci_cache.commit" }} + key: main-cache-{{ checksum "CMakeLists.txt" }}-{{ checksum ".circleci/fresh_ci_cache.commit" }} paths: - ./build diff --git a/.github/workflows/asan_build_and_test.yml b/.github/workflows/asan_build_and_test.yml index 4ebdaf48d..462e7bd91 100644 --- a/.github/workflows/asan_build_and_test.yml +++ b/.github/workflows/asan_build_and_test.yml @@ -19,7 +19,10 @@ jobs: cancel-in-progress: ${{ github.event_name == 'pull_request' }} steps: - name: Free Disk Space - run: rm -rf /useless/hostedtoolcache + run: | + rm -rf /useless/* + rm -rf /opt/hostedtoolcache + rm -rf /opt/intel/oneapi/mkl - uses: actions/checkout@v4 - name: Load Cache uses: hendrikmuhs/ccache-action@v1.2 @@ -28,9 +31,15 @@ jobs: save: ${{ github.event_name != 'pull_request' }} key: build-${{ hashFiles('./CMakeLists.txt') }}-${{ hashFiles('./.circleci/fresh_ci_cache.commit') }} - name: Make Asan - run: export CMAKE_GENERATOR="Ninja"; make asan + run: export CMAKE_GENERATOR="Ninja"; make asan VSAG_ENABLE_INTEL_MKL=OFF - name: Clean - run: find ./build -type f -name "*.o" -exec rm -f {} + + run: | + find ./build -type f -name "*.o" -exec rm -f {} + + find ./build -type f -name "*.cpp" -exec rm -f {} + + find ./build -type f -name "*.a" -exec rm -f {} + + rm -rf ./build/hdf5 + rm -rf ./build/boost + rm -rf ./build/tools - name: Save Test uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/tsan_build_and_test.yml b/.github/workflows/tsan_build_and_test.yml index 201843960..e03893429 100644 --- a/.github/workflows/tsan_build_and_test.yml +++ b/.github/workflows/tsan_build_and_test.yml @@ -15,7 +15,13 @@ jobs: cancel-in-progress: ${{ github.event_name == 'pull_request' }} container: image: vsaglib/vsag:ci-x86 + volumes: + - /opt:/useless steps: + - name: Free Disk Space + run: | + rm -rf /useless/* + rm -rf /opt/hostedtoolcache - uses: actions/checkout@v4 with: fetch-depth: '0' diff --git a/Makefile b/Makefile index 9521010a9..c1c7a1890 100644 --- a/Makefile +++ b/Makefile @@ -4,16 +4,22 @@ CMAKE_INSTALL_PREFIX ?= "/usr/local/" COMPILE_JOBS ?= 6 DEBUG_BUILD_DIR ?= "./build/" RELEASE_BUILD_DIR ?= "./build-release/" +VSAG_ENABLE_TESTS ?= ON +VSAG_ENABLE_PYBINDS ?= ON +VSAG_ENABLE_TOOLS ?= ON +VSAG_ENABLE_EXAMPLES ?= ON +VSAG_ENABLE_INTEL_MKL ?= ON + +VSAG_CMAKE_ARGS := -DCMAKE_EXPORT_COMPILE_COMMANDS=1 -DCMAKE_COLOR_DIAGNOSTICS=ON +VSAG_CMAKE_ARGS := ${VSAG_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX} -DNUM_BUILDING_JOBS=${COMPILE_JOBS} +VSAG_CMAKE_ARGS := ${VSAG_CMAKE_ARGS} -DENABLE_TESTS=${VSAG_ENABLE_TESTS} -DENABLE_PYBINDS=${VSAG_ENABLE_PYBINDS} +VSAG_CMAKE_ARGS := ${VSAG_CMAKE_ARGS} -DENABLE_TOOLS=${VSAG_ENABLE_TOOLS} -DENABLE_EXAMPLES=${VSAG_ENABLE_EXAMPLES} +VSAG_CMAKE_ARGS := ${VSAG_CMAKE_ARGS} -DENABLE_INTEL_MKL=${VSAG_ENABLE_INTEL_MKL} -OTHER_DEFINE="" ifdef EXTRA_DEFINED - OTHER_DEFINE=${EXTRA_DEFINED} + VSAG_CMAKE_ARGS := ${VSAG_CMAKE_ARGS} ${EXTRA_DEFINED} endif - -VSAG_CMAKE_ARGS := -DCMAKE_EXPORT_COMPILE_COMMANDS=1 -VSAG_CMAKE_ARGS := ${VSAG_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX} -DNUM_BUILDING_JOBS=${COMPILE_JOBS} -VSAG_CMAKE_ARGS := ${VSAG_CMAKE_ARGS} -DENABLE_TESTS=ON -DENABLE_PYBINDS=ON -DENABLE_TOOLS=ON -VSAG_CMAKE_ARGS := ${VSAG_CMAKE_ARGS} ${OTHER_DEFINE} -G ${CMAKE_GENERATOR} -S. +VSAG_CMAKE_ARGS := ${VSAG_CMAKE_ARGS} -G ${CMAKE_GENERATOR} -S. UT_FILTER = "" ifdef CASE diff --git a/src/algorithm/hgraph.cpp b/src/algorithm/hgraph.cpp index ffb66e6a0..7ceead451 100644 --- a/src/algorithm/hgraph.cpp +++ b/src/algorithm/hgraph.cpp @@ -1003,9 +1003,12 @@ HGraph::GetExtraInfoByIds(const int64_t* ids, int64_t count, char* extra_infos) void HGraph::add_one_point(const void* data, int level, InnerIdType inner_id) { - this->basic_flatten_codes_->InsertVector(data, inner_id); - if (use_reorder_) { - this->high_precise_codes_->InsertVector(data, inner_id); + { + std::shared_lock add_lock(add_mutex_); + this->basic_flatten_codes_->InsertVector(data, inner_id); + if (use_reorder_) { + this->high_precise_codes_->InsertVector(data, inner_id); + } } std::unique_lock add_lock(add_mutex_); if (level >= static_cast(this->route_graphs_.size()) || bottom_graph_->TotalCount() == 0) { diff --git a/src/data_cell/flatten_interface.cpp b/src/data_cell/flatten_interface.cpp index 2a42e770c..78b8ec774 100644 --- a/src/data_cell/flatten_interface.cpp +++ b/src/data_cell/flatten_interface.cpp @@ -23,20 +23,32 @@ #include "sparse_vector_datacell.h" namespace vsag { + template static FlattenInterfacePtr -make_instance(const FlattenInterfaceParamPtr& param, const IndexCommonParam& common_param) { +make_instance_flatten(const FlattenInterfaceParamPtr& param, const IndexCommonParam& common_param) { auto& io_param = param->io_parameter; auto& quantizer_param = param->quantizer_parameter; - if (param->name == SPARSE_VECTOR_DATA_CELL) { - return std::make_shared>( - quantizer_param, io_param, common_param); - } if (param->name == FLATTEN_DATA_CELL) { return std::make_shared>( quantizer_param, io_param, common_param); } - return nullptr; + throw VsagException(ErrorType::INVALID_ARGUMENT, + fmt::format("Unknown flatten interface name: {}", param->name)); +} + +template +static FlattenInterfacePtr +make_instance_sparse(const FlattenInterfaceParamPtr& param, const IndexCommonParam& common_param) { + auto& io_param = param->io_parameter; + auto& quantizer_param = param->quantizer_parameter; + + if (param->name == SPARSE_VECTOR_DATA_CELL) { + return std::make_shared>( + quantizer_param, io_param, common_param); + } + throw VsagException(ErrorType::INVALID_ARGUMENT, + fmt::format("Unknown flatten interface name: {}", param->name)); } template @@ -44,37 +56,37 @@ static FlattenInterfacePtr make_instance(const FlattenInterfaceParamPtr& param, const IndexCommonParam& common_param) { std::string quantization_string = param->quantizer_parameter->GetTypeName(); if (quantization_string == QUANTIZATION_TYPE_VALUE_SQ8) { - return make_instance, IOTemp>(param, common_param); + return make_instance_flatten, IOTemp>(param, common_param); } if (quantization_string == QUANTIZATION_TYPE_VALUE_FP32) { - return make_instance, IOTemp>(param, common_param); + return make_instance_flatten, IOTemp>(param, common_param); } if (quantization_string == QUANTIZATION_TYPE_VALUE_SQ4) { - return make_instance, IOTemp>(param, common_param); + return make_instance_flatten, IOTemp>(param, common_param); } if (quantization_string == QUANTIZATION_TYPE_VALUE_SQ4_UNIFORM) { - return make_instance, IOTemp>(param, common_param); + return make_instance_flatten, IOTemp>(param, common_param); } if (quantization_string == QUANTIZATION_TYPE_VALUE_SQ8_UNIFORM) { - return make_instance, IOTemp>(param, common_param); + return make_instance_flatten, IOTemp>(param, common_param); } if (quantization_string == QUANTIZATION_TYPE_VALUE_BF16) { - return make_instance, IOTemp>(param, common_param); + return make_instance_flatten, IOTemp>(param, common_param); } if (quantization_string == QUANTIZATION_TYPE_VALUE_FP16) { - return make_instance, IOTemp>(param, common_param); + return make_instance_flatten, IOTemp>(param, common_param); } if (quantization_string == QUANTIZATION_TYPE_VALUE_PQ) { - return make_instance, IOTemp>(param, common_param); + return make_instance_flatten, IOTemp>(param, common_param); } if (quantization_string == QUANTIZATION_TYPE_VALUE_PQFS) { - return make_instance, IOTemp>(param, common_param); + return make_instance_flatten, IOTemp>(param, common_param); } if (quantization_string == QUANTIZATION_TYPE_VALUE_RABITQ) { - return make_instance, IOTemp>(param, common_param); + return make_instance_flatten, IOTemp>(param, common_param); } if (quantization_string == QUANTIZATION_TYPE_VALUE_SPARSE) { - return make_instance, IOTemp>(param, common_param); + return make_instance_sparse, IOTemp>(param, common_param); } return nullptr; }