Merge pull request #175 from eraft-io/feature_20230610_vector

Feature 20230610 vector

CMakeLists.txt

@@ -246,6 +246,21 @@ target_link_libraries(eraftkv_server_test PUBLIC
     ${Protobuf_LIBRARY}
 )
 
+# build eraftmeta_server_tests
+add_executable(eraftmeta_server_test 
+    src/eraftmetaserver_test.cc 
+    src/eraftkv.pb.cc 
+    src/eraftkv.grpc.pb.cc
+    src/util.cc
+)
+target_link_libraries(eraftmeta_server_test PUBLIC
+    ${GTEST_LIBRARIES}
+    rocksdb
+    pthread
+    gRPC::grpc++
+    ${Protobuf_LIBRARY}
+)
+
 add_executable(rocksdb_storage_impl_tests 
     src/rocksdb_storage_impl_tests.cc 
     src/util.cc 

Dockerfile

@@ -46,7 +46,10 @@ FROM eraft/eraftkv:v0.0.4
 #        && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 20 \
 #        && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 20
 
-# RUN git clone https://github.com/grpc/grpc.git && cd grpc && git checkout v1.28.0 && git submodule update --init && mkdir .build && cd .build && cmake .. -DgRPC_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release && make install -j4 && cd .. && rm -rf .build/CMakeCache.txt && cd .build && cmake .. -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_PROTOBUF_PROVIDER=package -DgRPC_ZLIB_PROVIDER=package -DgRPC_CARES_PROVIDER=package -DgRPC_SSL_PROVIDER=package -DCMAKE_BUILD_TYPE=Release && make install -j4
+# RUN git clone https://github.com/grpc/grpc.git && cd grpc && git checkout v1.28.0 && git submodule update --init && \
+# mkdir .build && cd .build && cmake .. -DgRPC_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release \
+# && make install -j4 && cd .. && rm -rf .build/CMakeCache.txt && cd .build && 
+# cmake .. -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_PROTOBUF_PROVIDER=package -DgRPC_ZLIB_PROVIDER=package -DgRPC_CARES_PROVIDER=package -DgRPC_SSL_PROVIDER=package -DCMAKE_BUILD_TYPE=Release && make install -j4
 
 # intstall google benchmark
 # RUN git clone https://github.com/google/benchmark.git && git clone https://github.com/google/googletest.git benchmark/googletest && cd benchmark && cmake -E make_directory "build" && cmake -E chdir "build" cmake -DCMAKE_BUILD_TYPE=Release ../ && cmake --build "build" --config Release --target install

Makefile

@@ -52,10 +52,10 @@ rm-net:
 
 run-demo:
 	docker run --name kvserver-node1 --network mytestnetwork --ip 172.18.0.2 -d --rm -v $(realpath .):/eraft eraft/eraftkv:v0.0.6 /eraft/build/eraftkv 0 /tmp/kv_db0 /tmp/log_db0 172.18.0.2:8088,172.18.0.3:8089,172.18.0.4:8090
-	sleep 4s
+	sleep 5
 	docker run --name kvserver-node2 --network mytestnetwork --ip 172.18.0.3 -d --rm -v $(realpath .):/eraft eraft/eraftkv:v0.0.6 /eraft/build/eraftkv 1 /tmp/kv_db1 /tmp/log_db1 172.18.0.2:8088,172.18.0.3:8089,172.18.0.4:8090
 	docker run --name kvserver-node3 --network mytestnetwork --ip 172.18.0.4 -d --rm -v $(realpath .):/eraft eraft/eraftkv:v0.0.6 /eraft/build/eraftkv 2 /tmp/kv_db2 /tmp/log_db2 172.18.0.2:8088,172.18.0.3:8089,172.18.0.4:8090
-	sleep 10s
+	sleep 10
 	docker run --name vdbserver-node --network mytestnetwork --ip 172.18.0.6 -d --rm -v $(realpath .):/eraft eraft/eraftkv:v0.0.6 /eraft/build/eraft-vdb 172.18.0.6:12306 172.18.0.2:8088,172.18.0.3:8089,172.18.0.4:8090
 
 stop-demo:
@@ -67,3 +67,12 @@ run-demo-bench:
 run-vdb-tests:
 	chmod +x utils/run-vdb-tests.sh
 	docker run --name vdbserver-node-tests --network mytestnetwork --ip 172.18.0.8 -it --rm -v $(realpath .):/eraft eraft/eraftkv:v0.0.6 /eraft/utils/run-vdb-tests.sh
+
+run-metaserver-tests:
+	docker run --name metaserver-node1 --network mytestnetwork --ip 172.18.0.2 -d --rm -v $(realpath .):/eraft eraft/eraftkv:v0.0.6 /eraft/build/eraftmeta 0 /tmp/meta_db0 /tmp/log_db0 172.18.0.2:8088,172.18.0.3:8089,172.18.0.4:8090
+	docker run --name metaserver-node2 --network mytestnetwork --ip 172.18.0.3 -d --rm -v $(realpath .):/eraft eraft/eraftkv:v0.0.6 /eraft/build/eraftmeta 1 /tmp/meta_db1 /tmp/log_db1 172.18.0.2:8088,172.18.0.3:8089,172.18.0.4:8090
+	docker run --name metaserver-node3 --network mytestnetwork --ip 172.18.0.4 -d --rm -v $(realpath .):/eraft eraft/eraftkv:v0.0.6 /eraft/build/eraftmeta 2 /tmp/meta_db2 /tmp/log_db2 172.18.0.2:8088,172.18.0.3:8089,172.18.0.4:8090
+	sleep 10
+	docker run --name metaserver-tests --network mytestnetwork --ip 172.18.0.8 -it --rm -v $(realpath .):/eraft eraft/eraftkv:v0.0.6 /eraft/build/eraftmeta_server_test
+	sleep 2
+	docker stop metaserver-node1 metaserver-node2 metaserver-node3

doc/meta-server.md

@@ -0,0 +1,17 @@
+### Meta key design
+
+K -> V
+
+[ ['S'] ['G'] ['M'] ['E'] ['T'] ['A'] [ID (int64)]] ->  ShardGroup protobuf serialized message
+
+- Join
+
+example:
+
+[ ['S'] ['G'] ['M'] ['E'] ['T'] ['A'] [1] ] -> 
+{ id: 1, slots: [], servers: ['0-127.0.0.1:8088-UP,1-127.0.0.1:8089-UP,2-127.0.0.1:8090-UP'], leader_id: 0}
+
+- Leave
+
+
+- Query

protocol/eraftkv.proto

@@ -81,19 +81,17 @@ message SnapshotResp {
   bool   is_last_chunk = 5;
 }
 
-enum KeyRangeStatus {
+enum SlotStatus {
   Running = 0;
   Migrating = 1;
   Importing = 2;
   Init = 3;
 }
 
-message KeyRange {
-  KeyRangeStatus key_range_status = 1;
-  int64          shard_id = 2;
+message Slot {
+  int64          id = 1;
+  SlotStatus     slot_status = 2;
   int64          status_modify_time = 3;
-  string         start = 4;
-  string         end = 5;
 }
 
 enum ServerStatus {
@@ -109,24 +107,36 @@ message Server {
 
 message ShardGroup {
   int64           id = 1;
-  KeyRange        key_range = 2;
+  repeated Slot   slots = 2;
   repeated Server servers = 3;
   int64           leader_id = 4;
 }
 
-enum ClusterConfigChangeType {
+enum ChangeType {
   ClusterInit = 0;
-  AddServer = 1;
-  RemoveServer = 2;
-  Query = 3;
+  ShardJoin = 1;
+  ShardLeave = 2;
+  ShardsQuery = 3;
+  SlotMove = 4;
+  ServerJoin = 5;
+  ServerLeave = 6;
+  MetaMembersQuery = 7;
+}
+
+enum HandleServerType {
+  MetaServer = 0;
+  DataServer = 1;
 }
 
 message ClusterConfigChangeReq {
-  ClusterConfigChangeType change_type = 1;
-  int64                   shard_id = 2;
-  Server                  server = 3;
-  int64                   config_version = 4;
-  int64                   op_count = 5;
+  ChangeType              change_type = 1;
+  HandleServerType        handle_server_type = 2;
+  int64                   shard_id = 3;
+  Server                  server = 4;
+  int64                   config_version = 5;
+  int64                   op_count = 6;
+  int64                   command_id = 7;
+  ShardGroup              shard_group = 8;
 }
 
 message ClusterConfigChangeResp {
@@ -161,34 +171,6 @@ message ClientOperationResp {
   repeated KvOpPair ops = 2;
 }
 
-message SysCatalog {
-  uint64  vecset_count = 1;
-  uint64  max_vecset_id = 2;
-  uint64  mem_used = 3;
-  uint64  disk_used = 4;
-  map<string, uint64> vecset_name2id = 5;
-}
-
-message Vecset {
-  uint64  id = 1;
-  string  name = 2;
-  uint64  vec_count = 3;
-  uint64  max_vec_id = 4;
-  uint64  used_disk_capacity = 5;
-  uint64  used_mem_capacity = 6;
-  uint64  c_time = 7;
-  uint64  m_time = 8;
-}
-
-message Vec {
-  uint64  id = 1;
-  uint64  dim = 2;
-  repeated  double vdata = 3;
-  string  vlabel = 4;
-  uint64  c_time = 5;
-  uint64  m_time = 6;
-}
-
 service ERaftKv {
   rpc RequestVote(RequestVoteReq) returns (RequestVoteResp);
   rpc AppendEntries(AppendEntriesReq) returns (AppendEntriesResp);

src/async_buffer.cc

@@ -1,6 +1,6 @@
 /**
  * @file async_buffer.cc
- * @author ERaftGroup
+ * @author https://github.com/loveyacper/Qedis
  * @brief
  * @version 0.1
  * @date 2023-06-17

src/async_buffer.h

@@ -1,6 +1,6 @@
 /**
  * @file async_buffer.h
- * @author ERaftGroup
+ * @author https://github.com/loveyacper/Qedis
  * @brief
  * @version 0.1
  * @date 2023-06-17

src/buffer.h

@@ -1,6 +1,6 @@
 /**
  * @file buffer.h
- * @author ERaftGroup
+ * @author https://github.com/loveyacper/Qedis
  * @brief
  * @version 0.1
  * @date 2023-06-17

src/client.cc

@@ -63,7 +63,7 @@ void Client::OnConnect() {}
 std::string Client::GetLeaderAddr() {
   ClientContext                   context;
   eraftkv::ClusterConfigChangeReq req;
-  req.set_change_type(eraftkv::ClusterConfigChangeType::Query);
+  req.set_change_type(eraftkv::ChangeType::ShardsQuery);
   eraftkv::ClusterConfigChangeResp resp;
   auto                             status =
       this->stubs_.begin()->second->ClusterConfigChange(&context, req, &resp);

src/consts.h

@@ -0,0 +1,14 @@
+/**
+ * @file consts.h
+ * @author [email protected]
+ * @brief 
+ * @version 0.1
+ * @date 2023-07-01
+ * 
+ * @copyright Copyright (c) 2023
+ * 
+ */
+
+#pragma once
+
+#define SG_META_PREFIX "SG_META"