Skip to content

Commit dfbb45f

Browse files
LHT129LHT129
committed
refactor param to relace JsonType internal
Signed-off-by: LHT129 <[email protected]>
1 parent 496b712 commit dfbb45f

File tree

82 files changed

+1654
-620
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

82 files changed

+1654
-620
lines changed

‎src/CMakeLists.txt

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,30 @@
11

22
add_subdirectory (simd)
3+
add_subdirectory (io)
4+
add_subdirectory (quantization)
35

46
file (GLOB CPP_SRCS "*.cpp")
57
file (GLOB CPP_FACTORY_SRCS "factory/*.cpp")
68
file (GLOB CPP_CONJUGATE_GRAPH_SRCS "impl/*.cpp")
79
file (GLOB CPP_INDEX_SRCS "index/*.cpp")
810
file (GLOB CPP_HNSWLIB_SRCS "algorithm/hnswlib/*.cpp")
9-
file (GLOB CPP_QUANTIZATION_SRCS "quantization/*.cpp")
1011
file (GLOB CPP_DATA_CELL_SRCS "data_cell/*.cpp")
1112
file (GLOB CPP_ALGORITHM_SRCS "algorithm/*.cpp")
1213
list (FILTER CPP_SRCS EXCLUDE REGEX "_test.cpp")
1314
list (FILTER CPP_FACTORY_SRCS EXCLUDE REGEX "_test.cpp")
1415
list (FILTER CPP_CONJUGATE_GRAPH_SRCS EXCLUDE REGEX "_test.cpp")
1516
list (FILTER CPP_INDEX_SRCS EXCLUDE REGEX "_test.cpp")
16-
list (FILTER CPP_QUANTIZATION_SRCS EXCLUDE REGEX "_test.cpp")
1717
list (FILTER CPP_DATA_CELL_SRCS EXCLUDE REGEX "_test.cpp")
1818
list (FILTER CPP_ALGORITHM_SRCS EXCLUDE REGEX "_test.cpp")
1919

2020
set (VSAG_SRCS ${CPP_SRCS} ${CPP_FACTORY_SRCS} ${CPP_INDEX_SRCS} ${CPP_CONJUGATE_GRAPH_SRCS}
21-
${CPP_HNSWLIB_SRCS} ${CPP_QUANTIZATION_SRCS} ${CPP_DATA_CELL_SRCS} ${CPP_ALGORITHM_SRCS})
21+
${CPP_HNSWLIB_SRCS} ${CPP_DATA_CELL_SRCS} ${CPP_ALGORITHM_SRCS})
2222
add_library (vsag SHARED ${VSAG_SRCS})
2323
add_library (vsag_static STATIC ${VSAG_SRCS})
2424

25-
set (VSAG_DEP_LIBS diskann pthread m dl simd fmt::fmt-header-only nlohmann_json::nlohmann_json roaring)
25+
set (VSAG_DEP_LIBS diskann pthread m dl simd io quantizer fmt::fmt-header-only nlohmann_json::nlohmann_json roaring)
2626
target_link_libraries (vsag ${VSAG_DEP_LIBS} coverage_config)
2727
target_link_libraries (vsag_static ${VSAG_DEP_LIBS} coverage_config)
28+
2829
maybe_add_dependencies (vsag spdlog roaring openblas boost mkl)
2930
maybe_add_dependencies (vsag_static spdlog roaring openblas boost mkl)

‎src/algorithm/hgraph.cpp

Lines changed: 24 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121

2222
#include "common.h"
2323
#include "data_cell/sparse_graph_datacell.h"
24-
#include "index/hgraph_zparameters.h"
24+
#include "index/hgraph_index_zparameters.h"
2525

2626
namespace vsag {
2727
static BinarySet
@@ -50,69 +50,33 @@ next_multiple_of_power_of_two(uint64_t x, uint64_t n) {
5050
return result;
5151
}
5252

53-
HGraph::HGraph(const JsonType& index_param, const vsag::IndexCommonParam& common_param) noexcept
54-
: index_param_(index_param),
55-
common_param_(common_param),
53+
HGraph::HGraph(const HGraphParameter& hgraph_param,
54+
const vsag::IndexCommonParam& common_param) noexcept
55+
: common_param_(common_param),
56+
dim_(common_param.dim_),
57+
metric_(common_param.metric_),
58+
allocator_(common_param.allocator_.get()),
5659
label_lookup_(common_param.allocator_.get()),
5760
neighbors_mutex_(0, common_param.allocator_.get()),
5861
route_graphs_(common_param.allocator_.get()),
59-
labels_(common_param.allocator_.get()) {
60-
this->dim_ = common_param.dim_;
61-
this->metric_ = common_param.metric_;
62-
this->allocator_ = common_param.allocator_.get();
63-
}
64-
65-
tl::expected<void, Error>
66-
HGraph::Init() {
67-
try {
68-
CHECK_ARGUMENT(this->index_param_.contains(HGRAPH_USE_REORDER_KEY),
69-
fmt::format("hgraph parameters must contains {}", HGRAPH_USE_REORDER_KEY));
70-
this->use_reorder_ = this->index_param_[HGRAPH_USE_REORDER_KEY];
71-
72-
CHECK_ARGUMENT(this->index_param_.contains(HGRAPH_BASE_CODES_KEY),
73-
fmt::format("hgraph parameters must contains {}", HGRAPH_BASE_CODES_KEY));
74-
const auto& base_codes_json_obj = this->index_param_[HGRAPH_BASE_CODES_KEY];
75-
this->basic_flatten_codes_ =
76-
FlattenInterface::MakeInstance(base_codes_json_obj, common_param_);
77-
78-
if (this->use_reorder_) {
79-
CHECK_ARGUMENT(
80-
this->index_param_.contains(HGRAPH_PRECISE_CODES_KEY),
81-
fmt::format("hgraph parameters must contains {}", HGRAPH_PRECISE_CODES_KEY));
82-
const auto& precise_codes_json_obj = this->index_param_[HGRAPH_PRECISE_CODES_KEY];
83-
this->high_precise_codes_ =
84-
FlattenInterface::MakeInstance(precise_codes_json_obj, common_param_);
85-
}
86-
87-
CHECK_ARGUMENT(this->index_param_.contains(HGRAPH_GRAPH_KEY),
88-
fmt::format("hgraph parameters must contains {}", HGRAPH_GRAPH_KEY));
89-
const auto& graph_json_obj = this->index_param_[HGRAPH_GRAPH_KEY];
90-
this->bottom_graph_ = GraphInterface::MakeInstance(graph_json_obj, common_param_);
91-
92-
mult_ = 1 / log(1.0 * static_cast<double>(this->bottom_graph_->MaximumDegree()));
93-
94-
resize(bottom_graph_->max_capacity_);
95-
96-
if (this->index_param_.contains(BUILD_PARAMS_KEY)) {
97-
auto& build_params = this->index_param_[BUILD_PARAMS_KEY];
98-
if (build_params.contains(BUILD_EF_CONSTRUCTION)) {
99-
this->ef_construct_ = build_params[BUILD_EF_CONSTRUCTION];
100-
}
101-
if (build_params.contains(BUILD_THREAD_COUNT)) {
102-
this->build_thread_count_ = build_params[BUILD_THREAD_COUNT];
103-
}
104-
}
105-
106-
if (this->build_thread_count_ > 1) {
107-
this->build_pool_ = std::make_unique<progschj::ThreadPool>(this->build_thread_count_);
108-
}
109-
110-
this->init_features();
111-
} catch (const std::invalid_argument& e) {
112-
LOG_ERROR_AND_RETURNS(
113-
ErrorType::INVALID_ARGUMENT, "failed to init(invalid argument): ", e.what());
62+
labels_(common_param.allocator_.get()),
63+
use_reorder_(hgraph_param.use_reorder_),
64+
ef_construct_(hgraph_param.ef_construction_),
65+
build_thread_count_(hgraph_param.build_thread_count_) {
66+
this->basic_flatten_codes_ =
67+
FlattenInterface::MakeInstance(hgraph_param.base_codes_param_, common_param);
68+
if (use_reorder_) {
69+
this->high_precise_codes_ =
70+
FlattenInterface::MakeInstance(hgraph_param.precise_codes_param_, common_param);
11471
}
115-
return {};
72+
this->bottom_graph_ =
73+
GraphInterface::MakeInstance(hgraph_param.bottom_graph_param_, common_param);
74+
mult_ = 1 / log(1.0 * static_cast<double>(this->bottom_graph_->MaximumDegree()));
75+
resize(bottom_graph_->max_capacity_);
76+
if (this->build_thread_count_ > 1) {
77+
this->build_pool_ = std::make_unique<progschj::ThreadPool>(this->build_thread_count_);
78+
}
79+
this->init_features();
11680
}
11781

11882
tl::expected<std::vector<int64_t>, Error>

‎src/algorithm/hgraph.h

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
#include "common.h"
2727
#include "data_cell/flatten_interface.h"
2828
#include "data_cell/graph_interface.h"
29+
#include "hgraph_parameter.h"
2930
#include "index/index_common_param.h"
3031
#include "index_feature_list.h"
3132
#include "typing.h"
@@ -47,10 +48,7 @@ class HGraph {
4748
Vector<std::pair<float, InnerIdType>>,
4849
CompareByFirst>;
4950

50-
HGraph(const JsonType& index_param, const IndexCommonParam& common_param) noexcept;
51-
52-
tl::expected<void, Error>
53-
Init();
51+
HGraph(const HGraphParameter& param, const IndexCommonParam& common_param) noexcept;
5452

5553
tl::expected<std::vector<int64_t>, Error>
5654
Build(const DatasetPtr& data);
@@ -208,7 +206,6 @@ class HGraph {
208206
int64_t dim_{0};
209207
MetricType metric_{MetricType::METRIC_TYPE_L2SQR};
210208

211-
const JsonType index_param_{};
212209
const IndexCommonParam common_param_{};
213210

214211
std::default_random_engine level_generator_{2021};

‎src/algorithm/hgraph_parameter.cpp

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
2+
// Copyright 2024-present the vsag project
3+
//
4+
// Licensed under the Apache License, Version 2.0 (the "License");
5+
// you may not use this file except in compliance with the License.
6+
// You may obtain a copy of the License at
7+
//
8+
// http://www.apache.org/licenses/LICENSE-2.0
9+
//
10+
// Unless required by applicable law or agreed to in writing, software
11+
// distributed under the License is distributed on an "AS IS" BASIS,
12+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
// See the License for the specific language governing permissions and
14+
// limitations under the License.
15+
16+
#include "hgraph_parameter.h"
17+
18+
#include <fmt/format-inl.h>
19+
20+
#include "data_cell/graph_interface_parameter.h"
21+
#include "inner_string_params.h"
22+
23+
namespace vsag {
24+
25+
HGraphParameter::HGraphParameter(const JsonType& json) : HGraphParameter() {
26+
this->FromJson(json);
27+
}
28+
29+
HGraphParameter::HGraphParameter() : name_(INDEX_TYPE_HGRAPH) {
30+
}
31+
32+
void
33+
HGraphParameter::FromJson(const JsonType& json) {
34+
CHECK_ARGUMENT(json.contains(HGRAPH_USE_REORDER_KEY),
35+
fmt::format("hgraph parameters must contains {}", HGRAPH_USE_REORDER_KEY));
36+
this->use_reorder_ = json[HGRAPH_USE_REORDER_KEY];
37+
38+
CHECK_ARGUMENT(json.contains(HGRAPH_BASE_CODES_KEY),
39+
fmt::format("hgraph parameters must contains {}", HGRAPH_BASE_CODES_KEY));
40+
const auto& base_codes_json = json[HGRAPH_BASE_CODES_KEY];
41+
this->base_codes_param_ = std::make_shared<FlattenDataCellParameter>();
42+
this->base_codes_param_->FromJson(base_codes_json);
43+
44+
if (use_reorder_) {
45+
CHECK_ARGUMENT(json.contains(HGRAPH_PRECISE_CODES_KEY),
46+
fmt::format("hgraph parameters must contains {}", HGRAPH_PRECISE_CODES_KEY));
47+
const auto& precise_codes_json = json[HGRAPH_PRECISE_CODES_KEY];
48+
this->precise_codes_param_ = std::make_shared<FlattenDataCellParameter>();
49+
this->precise_codes_param_->FromJson(precise_codes_json);
50+
}
51+
52+
CHECK_ARGUMENT(json.contains(HGRAPH_GRAPH_KEY),
53+
fmt::format("hgraph parameters must contains {}", HGRAPH_GRAPH_KEY));
54+
const auto& graph_json = json[HGRAPH_GRAPH_KEY];
55+
this->bottom_graph_param_ = GraphInterfaceParameter::GetGraphParameterByJson(graph_json);
56+
57+
if (json.contains(BUILD_PARAMS_KEY)) {
58+
auto& build_params = json[BUILD_PARAMS_KEY];
59+
if (build_params.contains(BUILD_EF_CONSTRUCTION)) {
60+
this->ef_construction_ = build_params[BUILD_EF_CONSTRUCTION];
61+
}
62+
if (build_params.contains(BUILD_THREAD_COUNT)) {
63+
this->build_thread_count_ = build_params[BUILD_THREAD_COUNT];
64+
}
65+
}
66+
}
67+
68+
JsonType
69+
HGraphParameter::ToJson() {
70+
JsonType json;
71+
json["type"] = INDEX_TYPE_HGRAPH;
72+
73+
json[HGRAPH_USE_REORDER_KEY] = this->use_reorder_;
74+
json[HGRAPH_BASE_CODES_KEY] = this->base_codes_param_->ToJson();
75+
if (use_reorder_) {
76+
json[HGRAPH_PRECISE_CODES_KEY] = this->precise_codes_param_->ToJson();
77+
}
78+
json[HGRAPH_GRAPH_KEY] = this->bottom_graph_param_->ToJson();
79+
80+
json[BUILD_PARAMS_KEY][BUILD_EF_CONSTRUCTION] = this->ef_construction_;
81+
json[BUILD_PARAMS_KEY][BUILD_THREAD_COUNT] = this->build_thread_count_;
82+
return json;
83+
}
84+
85+
} // namespace vsag

‎src/algorithm/hgraph_parameter.h

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
2+
// Copyright 2024-present the vsag project
3+
//
4+
// Licensed under the Apache License, Version 2.0 (the "License");
5+
// you may not use this file except in compliance with the License.
6+
// You may obtain a copy of the License at
7+
//
8+
// http://www.apache.org/licenses/LICENSE-2.0
9+
//
10+
// Unless required by applicable law or agreed to in writing, software
11+
// distributed under the License is distributed on an "AS IS" BASIS,
12+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
// See the License for the specific language governing permissions and
14+
// limitations under the License.
15+
16+
#pragma once
17+
18+
#include "data_cell/flatten_datacell_parameter.h"
19+
#include "data_cell/graph_interface_parameter.h"
20+
#include "parameter.h"
21+
22+
namespace vsag {
23+
24+
class HGraphParameter : public Parameter {
25+
public:
26+
explicit HGraphParameter(const JsonType& json);
27+
28+
HGraphParameter();
29+
30+
void
31+
FromJson(const JsonType& json) override;
32+
33+
JsonType
34+
ToJson() override;
35+
36+
public:
37+
FlattenDataCellParamPtr base_codes_param_{nullptr};
38+
FlattenDataCellParamPtr precise_codes_param_{nullptr};
39+
GraphInterfaceParamPtr bottom_graph_param_{nullptr};
40+
41+
bool use_reorder_{false};
42+
uint64_t ef_construction_{400};
43+
uint64_t build_thread_count_{100};
44+
45+
std::string name_;
46+
};
47+
48+
} // namespace vsag

‎src/data_cell/flatten_datacell.h

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,8 @@ class FlattenDataCell : public FlattenInterface {
3030
public:
3131
FlattenDataCell() = default;
3232

33-
explicit FlattenDataCell(const JsonType& quantization_param,
34-
const JsonType& io_param,
33+
explicit FlattenDataCell(const QuantizerParamPtr& quantization_param,
34+
const IOParamPtr& io_param,
3535
const IndexCommonParam& common_param);
3636

3737
void
@@ -127,8 +127,8 @@ class FlattenDataCell : public FlattenInterface {
127127
};
128128

129129
template <typename QuantTmpl, typename IOTmpl>
130-
FlattenDataCell<QuantTmpl, IOTmpl>::FlattenDataCell(const JsonType& quantization_param,
131-
const JsonType& io_param,
130+
FlattenDataCell<QuantTmpl, IOTmpl>::FlattenDataCell(const QuantizerParamPtr& quantization_param,
131+
const IOParamPtr& io_param,
132132
const IndexCommonParam& common_param)
133133
: allocator_(common_param.allocator_.get()) {
134134
this->quantizer_ = std::make_shared<QuantTmpl>(quantization_param, common_param);

‎src/data_cell/flatten_datacell_parameter.cpp

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
2+
// Copyright 2024-present the vsag project
3+
//
4+
// Licensed under the Apache License, Version 2.0 (the "License");
5+
// you may not use this file except in compliance with the License.
6+
// You may obtain a copy of the License at
7+
//
8+
// http://www.apache.org/licenses/LICENSE-2.0
9+
//
10+
// Unless required by applicable law or agreed to in writing, software
11+
// distributed under the License is distributed on an "AS IS" BASIS,
12+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
// See the License for the specific language governing permissions and
14+
// limitations under the License.
15+
16+
#include "flatten_datacell_parameter.h"
17+
18+
#include <fmt/format-inl.h>
19+
20+
#include "inner_string_params.h"
21+
22+
namespace vsag {
23+
FlattenDataCellParameter::FlattenDataCellParameter() {
24+
}
25+
26+
void
27+
FlattenDataCellParameter::FromJson(const JsonType& json) {
28+
CHECK_ARGUMENT(json.contains(IO_PARAMS_KEY),
29+
fmt::format("flatten interface parameters must contains {}", IO_PARAMS_KEY));
30+
this->io_parameter_ = IOParameter::GetIOParameterByJson(json[IO_PARAMS_KEY]);
31+
32+
CHECK_ARGUMENT(
33+
json.contains(QUANTIZATION_PARAMS_KEY),
34+
fmt::format("flatten interface parameters must contains {}", QUANTIZATION_PARAMS_KEY));
35+
this->quantizer_parameter_ =
36+
QuantizerParameter::GetQuantizerParameterByJson(json[QUANTIZATION_PARAMS_KEY]);
37+
}
38+
39+
JsonType
40+
FlattenDataCellParameter::ToJson() {
41+
JsonType json;
42+
json[IO_PARAMS_KEY] = this->io_parameter_->ToJson();
43+
json[QUANTIZATION_PARAMS_KEY] = this->quantizer_parameter_->ToJson();
44+
return json;
45+
}
46+
} // namespace vsag

0 commit comments

Comments
 (0)