add string type encode

README.md

@@ -50,49 +50,21 @@ sudo make run-demo
 command output
 ```
 docker run --name kvserver-node1 --network mytestnetwork --ip 172.18.0.10 -d --rm -v /home/colin/eraft:/eraft eraft/eraftkv:v0.0.6 /eraft/build/eraftkv 0 /tmp/kv_db0 /tmp/log_db0 172.18.0.10:8088,172.18.0.11:8089,172.18.0.12:8090
-eca081a545a9eb8dbf9b05c2a307f38c74b4fea2910776e85c806c1b70cedf20
+bef74b85fcf9c0a2dedb15399b1f53010791e329f0c60d69fcd097e0843cbb86
 sleep 2
 docker run --name kvserver-node2 --network mytestnetwork --ip 172.18.0.11 -d --rm -v /home/colin/eraft:/eraft eraft/eraftkv:v0.0.6 /eraft/build/eraftkv 1 /tmp/kv_db1 /tmp/log_db1 172.18.0.10:8088,172.18.0.11:8089,172.18.0.12:8090
-74d14edf114f47889b50f0ed20ea810af7cd383de26ebdd3d1e36078290674e7
+333c02093fcb8c974cc1dc491fc7d2e19f474e3fda354fc130cf6be6d8920c85
 docker run --name kvserver-node3 --network mytestnetwork --ip 172.18.0.12 -d --rm -v /home/colin/eraft:/eraft eraft/eraftkv:v0.0.6 /eraft/build/eraftkv 2 /tmp/kv_db2 /tmp/log_db2 172.18.0.10:8088,172.18.0.11:8089,172.18.0.12:8090
-36bd437e67d00f019732e95e31b7f7ab9c19739a0f10676f31e9c0a7fad98a6c
+9856291dd34776cea94ab957780f7a244cb387bd0d74388b5a9d440175d6d28e
 sleep 1
 docker run --name metaserver-node1 --network mytestnetwork --ip 172.18.0.2 -d --rm -v /home/colin/eraft:/eraft eraft/eraftkv:v0.0.6 /eraft/build/eraftmeta 0 /tmp/meta_db0 /tmp/log_db0 172.18.0.2:8088,172.18.0.3:8089,172.18.0.4:8090
-f8a1382542f41d14e645ddeb285e8b93afc4367b8537e5bc4030487116d8f5cd
+09f9f12bc74212d1ae09a89bfecbc5a991f1b46cd9e8ba43fc278f775dd6176d
 sleep 3
 docker run --name metaserver-node2 --network mytestnetwork --ip 172.18.0.3 -d --rm -v /home/colin/eraft:/eraft eraft/eraftkv:v0.0.6 /eraft/build/eraftmeta 1 /tmp/meta_db1 /tmp/log_db1 172.18.0.2:8088,172.18.0.3:8089,172.18.0.4:8090
-7f5385341bc1f990f50020bd09526eaba3eeec56ab3c67fab325d313ab4ceaea
+3b98b3f317e834263ddb81c0bc5b245ac31b69cd47f495415a3d70e951c13900
 docker run --name metaserver-node3 --network mytestnetwork --ip 172.18.0.4 -d --rm -v /home/colin/eraft:/eraft eraft/eraftkv:v0.0.6 /eraft/build/eraftmeta 2 /tmp/meta_db2 /tmp/log_db2 172.18.0.2:8088,172.18.0.3:8089,172.18.0.4:8090
-666732b5a9b10cd828e9c0829bd97159b3ac9a7d39d1d3f2dcbbc2e5af654373
+10269f84d95e9f82f75d3c60f0d7b0dc0efe5efe643366e615b7644fb8851f04
 sleep 16
-docker run --name metaserver-tests --network mytestnetwork --ip 172.18.0.8 -it --rm -v /home/colin/eraft:/eraft eraft/eraftkv:v0.0.6 /eraft/build/eraftmeta_server_test
-[==========] Running 1 test from 1 test suite.
-[----------] Global test environment set-up.
-[----------] 1 test from EraftMetaServerTests
-[ RUN ] EraftMetaServerTests.TestMetaBasicOp
-DEBUG: cluster config resp -> success: true
-shard_group {
- id: 1
- servers {
- address: "172.18.0.10:8088"
- }
- servers {
- id: 1
- address: "172.18.0.11:8089"
- }
- servers {
- id: 2
- address: "172.18.0.12:8090"
- }
-}
-
-[ OK ] EraftMetaServerTests.TestMetaBasicOp (4028 ms)
-[----------] 1 test from EraftMetaServerTests (4028 ms total)
-
-[----------] Global test environment tear-down
-[==========] 1 test from 1 test suite ran. (4028 ms total)
-[ PASSED ] 1 test.
-sleep 2
 docker run --name vdbserver-node --network mytestnetwork --ip 172.18.0.6 -it --rm -v /home/colin/eraft:/eraft eraft/eraftkv:v0.0.6 /eraft/build/eraft-vdb 172.18.0.6:12306 172.18.0.2:8088,172.18.0.3:8089,172.18.0.4:8090
 run server success!
 ```
@@ -106,6 +78,9 @@ command output
 ```
 chmod +x utils/run-vdb-tests.sh
 docker run --name vdbserver-node-tests --network mytestnetwork --ip 172.18.0.9 -it --rm -v /home/colin/eraft:/eraft eraft/eraftkv:v0.0.6 /eraft/utils/run-vdb-tests.sh
+
+... (config change log)
+
 + redis-cli -h 172.18.0.6 -p 12306 info
 server_id: 0,server_address: 172.18.0.10:8088,status: Running,Role: Leader
 server_id: 1,server_address: 172.18.0.11:8089,status: Running,Role: Follower

protocol/eraftkv.proto

@@ -155,8 +155,8 @@ enum ClientOpType {
 
 message KvOpPair {
  ClientOpType op_type = 1;
- string key = 2;
- string value = 3;
+ bytes  key = 2;
+ bytes  value = 3;
  bool success = 4;
  int64 op_count = 5;
 }

src/client.cc

@@ -93,7 +93,8 @@ std::string Client::GetMetaLeaderAddr() {
  return leader_address;
 }
 
-std::string Client::GetShardLeaderAddr(std::string partion_key) {
+std::string Client::GetShardLeaderAddrAndSlot(std::string partion_key,
+ uint16_t *slot) {
  std::string leader_address;
  int64_t key_slot = -1;
  key_slot = HashUtil::CRC64(0, partion_key.c_str(), partion_key.size()) % 1024;

src/client.h

@@ -35,7 +35,7 @@ class Client : public StreamSocket {
  friend class ShardGroupCommandHandler;
 
  private:
- PacketLength _HandlePacket(const char *msg, std::size_t len) override;
+ PacketLength _HandlePacket(const char* msg, std::size_t len) override;
 
  UnboundedBuffer reply_;
 
@@ -60,7 +60,8 @@ class Client : public StreamSocket {
  * @param partion_key
  * @return std::string
  */
- std::string GetShardLeaderAddr(std::string partion_key);
+ std::string GetShardLeaderAddrAndSlot(std::string partion_key,
+ uint16_t* slot);
 
  /**
  * @brief Get the meta server leader address

src/get_command_handler.cc

@@ -10,26 +10,34 @@
  */
 
 #include "command_handler.h"
+#include "key_encode.h"
 
 EStatus GetCommandHandler::Execute(const std::vector<std::string>& params,
  Client* cli) {
  std::string leader_addr;
- leader_addr = cli->GetShardLeaderAddr(params[1]);
+ uint16_t slot;
+ leader_addr = cli->GetShardLeaderAddrAndSlot(params[1], &slot);
  ClientContext op_context;
  eraftkv::ClientOperationReq op_req;
  auto kv_pair_ = op_req.add_kvs();
- kv_pair_->set_key(params[1]);
+ kv_pair_->set_key(EncodeStringKey(slot, params[1]));
  kv_pair_->set_op_type(eraftkv::ClientOpType::Get);
  eraftkv::ClientOperationResp op_resp;
  std::string reply_buf;
+ std::string* user_val = new std::string();
  if (cli->kv_stubs_[leader_addr] != nullptr) {
  auto status_ = cli->kv_stubs_[leader_addr]->ProcessRWOperation(
  &op_context, op_req, &op_resp);
  if (status_.ok()) {
  reply_buf += "$";
- reply_buf += std::to_string(op_resp.ops(0).value().size());
+ // decode string val
+ uint8_t flag;
+ uint32_t expire;
+ std::string enc_val = op_resp.ops(0).value();
+ DecodeStringVal(enc_val, &flag, &expire, user_val);
+ reply_buf += std::to_string(user_val->size());
  reply_buf += "\r\n";
- reply_buf += op_resp.ops(0).value();
+ reply_buf += *user_val;
  reply_buf += "\r\n";
  } else {
  reply_buf += "-ERR Server error\r\n";
@@ -40,6 +48,7 @@ EStatus GetCommandHandler::Execute(const std::vector<std::string>& params,
  cli->reply_.PushData(reply_buf.c_str(), reply_buf.size());
  cli->SendPacket(cli->reply_);
  cli->_Reset();
+ delete user_val;
  return EStatus::kOk;
 }
 

src/key_encode.h

@@ -0,0 +1,83 @@
+/**
+ * @file key_encode.h
+ * @author [email protected]
+ * @brief
+ * @version 0.1
+ * @date 2023-07-24
+ *
+ * @copyright Copyright (c) 2023
+ *
+ */
+
+#include <string>
+#include <vector>
+
+#include "util.h"
+
+enum Flag {
+ TYPE_STRING = 0x01,
+ TYPE_HASH = 0x02,
+ TYPE_LIST = 0x03,
+ TYPE_SET = 0x04,
+ TYPE_ZSET = 0x05,
+};
+
+/**
+ * @brief
+ *
+ * | slot (2B) | user_key (n B) |
+ * @return std::string
+ */
+static std::string EncodeStringKey(uint16_t key_slot, std::string user_key) {
+ std::string dst;
+ EncodeDecodeTool::PutFixed16(&dst, key_slot);
+ dst.append(user_key);
+ return dst;
+}
+
+/**
+ * @brief decode an encode key
+ *
+ * @param enc_key
+ * @param key_slot
+ * @param user_key
+ */
+static void DecodeStringKey(std::string enc_key,
+ uint16_t* key_slot,
+ std::string* user_key) {
+ std::vector<uint8_t> enc_key_seq = {enc_key.begin(), enc_key.end()};
+ *key_slot = EncodeDecodeTool::DecodeFixed16(&enc_key_seq[0]);
+ *user_key = std::string(enc_key_seq.begin() + 2, enc_key_seq.end());
+}
+
+/**
+ * @brief
+ *
+ * | flags (1B) | expire (4B) | user_value (nB) |
+ * @return std::string
+ */
+static std::string EncodeStringVal(uint32_t expire, std::string user_val) {
+ std::string dst;
+ dst.append(Flag::TYPE_STRING, 1);
+ EncodeDecodeTool::PutFixed32(&dst, expire);
+ dst.append(user_val);
+ return dst;
+}
+
+/**
+ * @brief decode an encode value
+ *
+ * @param enc_val
+ * @param flag
+ * @param expire
+ * @param user_val
+ */
+static void DecodeStringVal(std::string enc_val,
+ uint8_t* flag,
+ uint32_t* expire,
+ std::string* user_val) {
+ std::vector<uint8_t> enc_val_seq = {enc_val.begin(), enc_val.end()};
+ *flag = enc_val_seq[0];
+ *expire = EncodeDecodeTool::DecodeFixed32(&enc_val_seq[1]);
+ *user_val = std::string(enc_val_seq.begin() + 5, enc_val_seq.end());
+}