support basic optimizer and searcher

include/vsag/constants.h

@@ -50,6 +50,10 @@ extern const char* const PARAMETER_METRIC_TYPE;
 extern const char* const PARAMETER_USE_CONJUGATE_GRAPH;
 extern const char* const PARAMETER_USE_CONJUGATE_GRAPH_SEARCH;
 
+extern const char* const PREFETCH_NEIGHBOR_VISIT_NUM;
+extern const char* const PREFETCH_NEIGHBOR_CODE_NUM;
+extern const char* const PREFETCH_CACHE_LINE;
+
 extern const char* const DISKANN_PARAMETER_L;
 extern const char* const DISKANN_PARAMETER_R;
 extern const char* const DISKANN_PARAMETER_P_VAL;

src/algorithm/hnswlib/hnswalg.cpp

@@ -1548,4 +1548,10 @@ HierarchicalNSW::checkIntegrity() {
     std::cout << "integrity ok, checked " << connections_checked << " connections\n";
 }
 
+template MaxHeap
+HierarchicalNSW::searchBaseLayerST<false, false>(InnerIdType ep_id,
+                                                 const void* data_point,
+                                                 size_t ef,
+                                                 vsag::BaseFilterFunctor* isIdAllowed) const;
+
 }  // namespace hnswlib

src/algorithm/hnswlib/hnswalg.h

@@ -149,6 +149,17 @@ class HierarchicalNSW : public AlgorithmInterface<float> {
     bool
     isValidLabel(LabelType label) override;
 
+    size_t
+    getMaxDegree() {
+        return maxM0_;
+    };
+
+    linklistsizeint*
+    get_linklist0(InnerIdType internal_id) const {
+        // only for test now
+        return (linklistsizeint*)(data_level0_memory_->GetElementPtr(internal_id, offsetLevel0_));
+    }
+
     inline LabelType
     getExternalLabel(InnerIdType internal_id) const {
         std::shared_lock lock(points_locks_[internal_id]);

src/allocator_wrapper.h

@@ -71,4 +71,17 @@ class AllocatorWrapper {
 
     Allocator* allocator_{};
 };
+
+template <typename T, typename U>
+bool
+operator==(const AllocatorWrapper<T>&, const AllocatorWrapper<U>&) noexcept {
+    return true;
+}
+
+template <typename T, typename U>
+bool
+operator!=(const AllocatorWrapper<T>& a, const AllocatorWrapper<U>& b) noexcept {
+    return !(a == b);
+}
+
 }  // namespace vsag

src/constants.cpp

@@ -52,6 +52,10 @@ const char* const PARAMETER_METRIC_TYPE = "metric_type";
 const char* const PARAMETER_USE_CONJUGATE_GRAPH = "use_conjugate_graph";
 const char* const PARAMETER_USE_CONJUGATE_GRAPH_SEARCH = "use_conjugate_graph_search";
 
+const char* const PREFETCH_NEIGHBOR_VISIT_NUM = "prefetch_neighbor_visit_num";
+const char* const PREFETCH_NEIGHBOR_CODE_NUM = "prefetch_neighbor_codes_num";
+const char* const PREFETCH_CACHE_LINE = "prefetch_cache_line";
+
 const char* const DISKANN_PARAMETER_L = "ef_construction";
 const char* const DISKANN_PARAMETER_R = "max_degree";
 const char* const DISKANN_PARAMETER_P_VAL = "pq_sample_rate";

src/data_cell/adapter_graph_datacell.h

@@ -0,0 +1,55 @@
+
+// Copyright 2024-present the vsag project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#include "algorithm/hnswlib/hnswalg.h"
+#include "algorithm/hnswlib/space_l2.h"
+
+namespace vsag {
+class AdaptGraphDataCell {
+public:
+    AdaptGraphDataCell(std::shared_ptr<hnswlib::HierarchicalNSW> alg_hnsw) : alg_hnsw_(alg_hnsw){};
+
+    void
+    GetNeighbors(InnerIdType id, Vector<InnerIdType>& neighbor_ids) {
+        int* data = (int*)alg_hnsw_->get_linklist0(id);
+        uint32_t size = alg_hnsw_->getListCount((hnswlib::linklistsizeint*)data);
+        neighbor_ids.resize(size);
+        for (uint32_t i = 0; i < size; i++) {
+            neighbor_ids[i] = *(data + i + 1);
+        }
+    }
+
+    uint32_t
+    GetNeighborSize(InnerIdType id) {
+        int* data = (int*)alg_hnsw_->get_linklist0(id);
+        return alg_hnsw_->getListCount((hnswlib::linklistsizeint*)data);
+    }
+
+    void
+    Prefetch(InnerIdType id, InnerIdType neighbor_i) {
+        int* data = (int*)alg_hnsw_->get_linklist0(id);
+        _mm_prefetch(data + neighbor_i + 1, _MM_HINT_T0);
+    }
+
+    uint32_t
+    MaximumDegree() {
+        return alg_hnsw_->getMaxDegree();
+    }
+
+private:
+    std::shared_ptr<hnswlib::HierarchicalNSW> alg_hnsw_;
+};
+}  // namespace vsag

src/data_cell/adapter_graph_datacell_test.cpp

@@ -0,0 +1,68 @@
+
+// Copyright 2024-present the vsag project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "adapter_graph_datacell.h"
+
+#include "catch2/catch_template_test_macros.hpp"
+#include "fixtures.h"
+#include "fmt/format-inl.h"
+#include "graph_interface_test.h"
+#include "io/io_headers.h"
+#include "safe_allocator.h"
+
+using namespace vsag;
+
+TEST_CASE("basic usage for graph data cell (adapter of hnsw)", "[ut][GraphDataCell]") {
+    uint32_t M = 32;
+    uint32_t data_size = 1000;
+    uint32_t ef_construction = 100;
+    uint64_t DEFAULT_MAX_ELEMENT = 1;
+    uint64_t dim = 960;
+    auto vectors = fixtures::generate_vectors(data_size, dim);
+    std::vector<int64_t> ids(data_size);
+    std::iota(ids.begin(), ids.end(), 0);
+
+    auto allocator = SafeAllocator::FactoryDefaultAllocator();
+    auto space = std::make_shared<hnswlib::L2Space>(dim);
+    auto io = std::make_shared<MemoryIO>(allocator.get());
+    auto alg_hnsw =
+        std::make_shared<hnswlib::HierarchicalNSW>(space.get(),
+                                                   DEFAULT_MAX_ELEMENT,
+                                                   allocator.get(),
+                                                   M / 2,
+                                                   ef_construction,
+                                                   Options::Instance().block_size_limit());
+    alg_hnsw->init_memory_space();
+    for (int64_t i = 0; i < data_size; ++i) {
+        auto successful_insert =
+            alg_hnsw->addPoint((const void*)(vectors.data() + i * dim), ids[i]);
+        REQUIRE(successful_insert == true);
+    }
+
+    auto graph_data_cell = std::make_shared<AdaptGraphDataCell>(alg_hnsw);
+
+    for (uint32_t i = 0; i < data_size; i++) {
+        auto neighbor_size = graph_data_cell->GetNeighborSize(i);
+        Vector<InnerIdType> neighbor_ids(neighbor_size, allocator.get());
+        graph_data_cell->GetNeighbors(i, neighbor_ids);
+
+        int* data = (int*)alg_hnsw->get_linklist0(i);
+        REQUIRE(neighbor_size == alg_hnsw->getListCount((hnswlib::linklistsizeint*)data));
+
+        for (uint32_t j = 0; j < neighbor_size; j++) {
+            REQUIRE(neighbor_ids[j] == *(data + j + 1));
+        }
+    }
+}

src/data_cell/flatten_datacell.h

@@ -70,6 +70,11 @@ class FlattenDataCell : public FlattenInterface {
         io_->Prefetch(id * code_size_);
     };
 
+    bool
+    Decode(const uint8_t* codes, DataType* data) override {
+        return this->quantizer_->DecodeOne(codes, data);
+    }
+
     [[nodiscard]] std::string
     GetQuantizerName() override;
 
@@ -226,7 +231,18 @@ FlattenDataCell<QuantTmpl, IOTmpl>::query(float* result_dists,
                                           const std::shared_ptr<Computer<QuantTmpl>>& computer,
                                           const InnerIdType* idx,
                                           InnerIdType id_count) {
+    for (uint32_t i = 0; i < this->prefetch_neighbor_codes_num_ and i < id_count; i++) {
+        this->io_->Prefetch(static_cast<uint64_t>(idx[i]) * static_cast<uint64_t>(code_size_),
+                            this->prefetch_cache_line_);
+    }
+
     for (int64_t i = 0; i < id_count; ++i) {
+        if (i + this->prefetch_neighbor_codes_num_ < id_count) {
+            this->io_->Prefetch(static_cast<uint64_t>(idx[i + this->prefetch_neighbor_codes_num_]) *
+                                    static_cast<uint64_t>(code_size_),
+                                this->prefetch_cache_line_);
+        }
+
         bool release = false;
         const auto* codes = this->GetCodesById(idx[i], release);
         computer->ComputeDist(codes, result_dists + i);

src/data_cell/flatten_interface.h

@@ -18,11 +18,13 @@
 #include <nlohmann/json.hpp>
 #include <string>
 
+#include "impl/runtime_parameter.h"
 #include "index/index_common_param.h"
 #include "quantization/computer.h"
 #include "stream_reader.h"
 #include "stream_writer.h"
 #include "typing.h"
+#include "vsag/constants.h"
 
 namespace vsag {
 class FlattenInterface;
@@ -83,6 +85,11 @@ class FlattenInterface {
         return false;
     }
 
+    virtual bool
+    Decode(const uint8_t* codes, DataType* vector) {
+        return false;
+    }
+
     [[nodiscard]] virtual InnerIdType
     TotalCount() const {
         return this->total_count_;
@@ -102,10 +109,23 @@ class FlattenInterface {
         StreamReader::ReadObj(reader, this->code_size_);
     }
 
+    virtual void
+    SetRuntimeParameters(const UnorderedMap<std::string, ParamValue>& new_params) {
+        if (new_params.find(PREFETCH_NEIGHBOR_CODE_NUM) != new_params.end()) {
+            prefetch_neighbor_codes_num_ = std::get<int>(new_params.at(PREFETCH_NEIGHBOR_CODE_NUM));
+        }
+
+        if (new_params.find(PREFETCH_CACHE_LINE) != new_params.end()) {
+            prefetch_cache_line_ = std::get<int>(new_params.at(PREFETCH_CACHE_LINE));
+        }
+    }
+
 public:
     InnerIdType total_count_{0};
     InnerIdType max_capacity_{1000000};
     uint32_t code_size_{0};
+    uint32_t prefetch_neighbor_codes_num_{1};
+    uint32_t prefetch_cache_line_{1};
 };
 
 }  // namespace vsag

src/impl/basic_optimizer.cpp

@@ -0,0 +1,22 @@
+
+// Copyright 2024-present the vsag project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "basic_optimizer.h"
+
+namespace vsag {
+template class Optimizer<
+    BasicSearcher<AdaptGraphDataCell,
+                  FlattenDataCell<FP32Quantizer<vsag::MetricType::METRIC_TYPE_L2SQR>, MemoryIO>>>;
+}

src/impl/basic_optimizer.h

@@ -0,0 +1,92 @@
+
+// Copyright 2024-present the vsag project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#include "../utils.h"
+#include "basic_searcher.h"
+#include "logger.h"
+#include "runtime_parameter.h"
+
+namespace vsag {
+
+template <typename OptimizableOBJ>
+class Optimizer {
+public:
+    Optimizer(const IndexCommonParam& common_param, int trials = 100)
+        : parameters_(common_param.allocator_.get()),
+          best_params_(common_param.allocator_.get()),
+          n_trials_(trials),
+          best_loss_(std::numeric_limits<double>::max()) {
+        allocator_ = common_param.allocator_.get();
+        std::random_device rd;
+        gen_.seed(rd());
+    }
+
+    void
+    RegisterParameter(const std::shared_ptr<RuntimeParameter>& runtime_parameter) {
+        parameters_.push_back(runtime_parameter);
+    }
+
+    void
+    Optimize(std::shared_ptr<OptimizableOBJ> obj) {
+        double original_loss = obj->MockRun();
+
+        for (int i = 0; i < n_trials_; ++i) {
+            // generate a group of runtime params
+            UnorderedMap<std::string, ParamValue> current_params(allocator_);
+            for (auto& param : parameters_) {
+                current_params[param->name_] = param->sample(gen_);
+            }
+            obj->SetRuntimeParameters(current_params);
+
+            // evaluate
+            double loss = obj->MockRun();
+
+            // update
+            if (loss < best_loss_) {
+                best_loss_ = loss;
+                best_params_ = current_params;
+                vsag::logger::debug(fmt::format("Trial {}: new best loss = {}, improving = {}",
+                                                i + 1,
+                                                best_loss_,
+                                                (original_loss - best_loss_) / original_loss));
+            }
+        }
+
+        obj->SetRuntimeParameters(best_params_);
+    }
+
+    UnorderedMap<std::string, ParamValue>
+    GetBestParameters() const {
+        return best_params_;
+    }
+
+    double
+    GetBestLoss() const {
+        return best_loss_;
+    }
+
+private:
+    Allocator* allocator_{nullptr};
+
+    Vector<std::shared_ptr<RuntimeParameter>> parameters_;
+    int n_trials_{0};
+    std::mt19937 gen_;
+
+    UnorderedMap<std::string, ParamValue> best_params_;
+    double best_loss_{0};
+};
+
+}  // namespace vsag