[WIP] Mix data cell

CMakeLists.txt

@@ -189,6 +189,7 @@ endif ()
 
 include_directories (${CMAKE_CURRENT_BINARY_DIR}/spdlog/install/include)
 include_directories (include)
+include_directories (src)
 
 set (CMAKE_CXX_STANDARD 17)
 

src/CMakeLists.txt

@@ -1,17 +1,28 @@
 
 add_subdirectory (simd)
+set (SIMD_QUANT_SRCS quantization/sq4_simd.cpp quantization/sq8_simd.cpp quantization/fp32_simd.cpp quantization/sq4_uniform_simd.cpp)
+set_source_files_properties (${SIMD_QUANT_SRCS} PROPERTIES COMPILE_FLAGS "-msse -msse2 -msse3 -mssse3 -msse4 -msse4a -msse4.1 -msse4.2")
+set_source_files_properties (${SIMD_QUANT_SRCS} PROPERTIES COMPILE_FLAGS "-mavx")
+set_source_files_properties (${SIMD_QUANT_SRCS} PROPERTIES COMPILE_FLAGS "-mavx2 -mfma")
+set_source_files_properties (${SIMD_QUANT_SRCS} PROPERTIES COMPILE_FLAGS "-mavx512f -mavx512pf -mavx512er -mavx512cd -mavx512vl -mavx512bw -mavx512dq -mavx512ifma -mavx512vbmi")
+
 
 file (GLOB CPP_SRCS "*.cpp")
 file (GLOB CPP_FACTORY_SRCS "factory/*.cpp")
 file (GLOB CPP_CONJUGATE_GRAPH_SRCS "impl/*.cpp")
 file (GLOB CPP_INDEX_SRCS "index/*.cpp")
 file (GLOB CPP_HNSWLIB_SRCS "algorithm/hnswlib/*.cpp")
+file (GLOB CPP_QUANTIZATION_SRCS "quantization/*.cpp")
+file (GLOB CPP_SEARCHER_SRCS "searcher/*.cpp")
 list (FILTER CPP_SRCS EXCLUDE REGEX "_test.cpp")
 list (FILTER CPP_FACTORY_SRCS EXCLUDE REGEX "_test.cpp")
 list (FILTER CPP_CONJUGATE_GRAPH_SRCS EXCLUDE REGEX "_test.cpp")
 list (FILTER CPP_INDEX_SRCS EXCLUDE REGEX "_test.cpp")
+list (FILTER CPP_QUANTIZATION_SRCS EXCLUDE REGEX "_test.cpp")
+list (FILTER CPP_SEARCHER_SRCS EXCLUDE REGEX "_test.cpp")
 
-set (VSAG_SRCS ${CPP_SRCS} ${CPP_FACTORY_SRCS} ${CPP_INDEX_SRCS} ${CPP_CONJUGATE_GRAPH_SRCS} ${CPP_HNSWLIB_SRCS})
+set (VSAG_SRCS ${CPP_SRCS} ${CPP_FACTORY_SRCS} ${CPP_INDEX_SRCS}
+        ${CPP_CONJUGATE_GRAPH_SRCS} ${CPP_HNSWLIB_SRCS} ${CPP_QUANTIZATION_SRCS} ${CPP_SEARCHER_SRCS})
 add_library (vsag SHARED ${VSAG_SRCS})
 add_library (vsag_static STATIC ${VSAG_SRCS})
 

src/algorithm/hnswlib/hnswalg.h

@@ -39,7 +39,6 @@
 #include "visited_list_pool.h"
 
 namespace hnswlib {
-using tableint = unsigned int;
 using linklistsizeint = unsigned int;
 using reverselinklist = std::
     unordered_set<tableint, std::hash<tableint>, std::equal_to<>, vsag::AllocatorWrapper<tableint>>;
@@ -250,6 +249,11 @@ class HierarchicalNSW : public AlgorithmInterface<float> {
         }
     }
 
+    size_t
+    getMaxDegree() override {
+        return maxM0_;
+    };
+
     float
     getDistanceByLabel(labeltype label, const void* data_point) override {
         std::unique_lock<std::mutex> lock_table(label_lookup_lock);
@@ -274,14 +278,6 @@ class HierarchicalNSW : public AlgorithmInterface<float> {
         return is_valid;
     }
 
-    struct CompareByFirst {
-        constexpr bool
-        operator()(std::pair<float, tableint> const& a,
-                   std::pair<float, tableint> const& b) const noexcept {
-            return a.first < b.first;
-        }
-    };
-
     inline std::mutex&
     getLabelOpMutex(labeltype label) const {
         // calculate hash

src/algorithm/hnswlib/hnswalg_static.h

@@ -237,6 +237,11 @@ class StaticHierarchicalNSW : public AlgorithmInterface<float> {
         return (int)r;
     }
 
+    size_t
+    getMaxDegree() override {
+        return maxM0_;
+    };
+
     size_t
     getMaxElements() override {
         return max_elements_;

src/algorithm/hnswlib/hnswlib.h

@@ -140,8 +140,17 @@ AVX512Capable() {
 #include <vector>
 
 namespace hnswlib {
+using tableint = unsigned int;
 typedef size_t labeltype;
 
+struct CompareByFirst {
+    constexpr bool
+    operator()(std::pair<float, tableint> const& a,
+               std::pair<float, tableint> const& b) const noexcept {
+        return a.first < b.first;
+    }
+};
+
 // This can be extended to store state for filtering (e.g. from a std::set)
 class BaseFilterFunctor {
 public:
@@ -225,6 +234,9 @@ class AlgorithmInterface {
     virtual size_t
     getMaxElements() = 0;
 
+    virtual size_t
+    getMaxDegree() = 0;
+
     virtual float
     getDistanceByLabel(labeltype label, const void* data_point) = 0;
 
@@ -257,6 +269,17 @@ class AlgorithmInterface {
     virtual bool
     isValidLabel(labeltype label) = 0;
 
+    //    virtual
+    //    std::priority_queue<std::pair<float, tableint>,
+    //                        std::vector<std::pair<float, tableint>>,
+    //                        CompareByFirst>
+    //        searchBaseLayerST(tableint ep_id,
+    //                          const void* data_point,
+    //                          size_t ef,
+    //                          BaseFilterFunctor* isIdAllowed = nullptr) const {
+    //        throw std::runtime_error("unsupported func");
+    //    };
+
     virtual ~AlgorithmInterface() {
     }
 };

src/index/flatten_index.cpp

@@ -0,0 +1,17 @@
+
+// Copyright 2024-present the vsag project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "flatten_index.h"
+namespace vsag {}