Skip to content

Commit b27c892

Browse files
hzchenwei7YunhuiChen
hzchenwei7
authored and
YunhuiChen
committed
rewrite snap tool
Change-Id: Idc1378a65dc5c84332b32d24e5803fa9789c13fb
1 parent da625e4 commit b27c892

28 files changed

+1998
-168
lines changed

‎curve-ansible/deploy_curve.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,8 @@
9393
tags: ["generate_config", "generage_tools_conf"] }
9494
- { role: generate_config, template_name: topo.json, conf_path: "{{ topo_file_path }}",
9595
tags: ["generate_config", "generage_topo_json"] }
96+
- { role: generate_config, template_name: snapshot_tools.conf, conf_path: "{{ snap_tool_config_path }}",
97+
tags: ["generate_config", "generage_snaptool_conf"] }
9698

9799
- name: start mds
98100
hosts: mds

‎curve-ansible/roles/generate_config/templates/snapshot_tools.conf.j2

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#
2+
#server options
3+
#
4+
# for snapshot
5+
server.address={{ snapshot_nginx_vip }}:{{ nginx_docker_external_port }}

‎curve-ansible/roles/install_package/tasks/include/install_curve-tools.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,3 +22,11 @@
2222
file_mode: 0755
2323
include_tasks: copy_file_to_remote.yml
2424

25+
- name: pip install requests
26+
ignore_errors: true
27+
shell: sudo pip install requests
28+
register: pip_requests_res
29+
30+
- name: pip2 install requests
31+
shell: sudo pip2 install requests
32+
when: pip_requests_res.failed

‎curve-ansible/server.ini

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ snapshot_nginx_lua_conf_path=/etc/curve/nginx/app/etc/config.lua
9696
nginx_docker_internal_port=80
9797
nginx_docker_external_port=5555
9898
snapshot_nginx_name=curve-snapshotclone-nginx
99+
snapshot_nginx_vip=127.0.0.1
99100

100101
[all:vars]
101102
need_confirm=true

‎mk-deb.sh

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,13 @@ if [ $? -ne 0 ]
198198
then
199199
exit
200200
fi
201+
cp -r tools/snaptool build/curve-tools/usr/bin/snaptool-lib
202+
cp tools/snaptool/snaptool build/curve-tools/usr/bin/snaptool
203+
chmod a+x build/curve-tools/usr/bin/snaptool
204+
if [ $? -ne 0 ]
205+
then
206+
exit
207+
fi
201208
cp ./bazel-bin/src/tools/curve_tool \
202209
build/curve-tools/usr/bin/curve_ops_tool
203210
if [ $? -ne 0 ]

‎mk-tar.sh

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,13 @@ if [ $? -ne 0 ]
167167
then
168168
exit
169169
fi
170+
cp -r tools/snaptool build/curve/curve-tools/bin/snaptool-lib
171+
cp tools/snaptool/snaptool build/curve/curve-tools/bin/snaptool
172+
chmod a+x build/curve/curve-tools/bin/snaptool
173+
if [ $? -ne 0 ]
174+
then
175+
exit
176+
fi
170177
# curve-chunkserver
171178
mkdir -p build/curve/curve-chunkserver/bin
172179
if [ $? -ne 0 ]

‎src/snapshotcloneserver/clone/clone_service_manager.cpp

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -286,6 +286,70 @@ int CloneServiceManager::GetCloneTaskInfoByName(
286286
return GetCloneTaskInfoInner(cloneInfos, user, info);
287287
}
288288

289+
int CloneServiceManager::GetCloneTaskInfoByFilter(
290+
const CloneFilterCondition &filter,
291+
std::vector<TaskCloneInfo> *info) {
292+
std::vector<CloneInfo> cloneInfos;
293+
int ret = cloneCore_->GetCloneInfoList(&cloneInfos);
294+
if (ret < 0) {
295+
LOG(ERROR) << "GetCloneInfoList fail"
296+
<< ", ret = " << ret;
297+
return kErrCodeFileNotExist;
298+
}
299+
return GetCloneTaskInfoInner(cloneInfos, filter, info);
300+
}
301+
302+
int CloneServiceManager::GetCloneTaskInfoInner(
303+
std::vector<CloneInfo> cloneInfos,
304+
CloneFilterCondition filter,
305+
std::vector<TaskCloneInfo> *info) {
306+
int ret = kErrCodeSuccess;
307+
for (auto &cloneInfo : cloneInfos) {
308+
if (filter.IsMatchCondition(cloneInfo)) {
309+
switch (cloneInfo.GetStatus()) {
310+
case CloneStatus::done : {
311+
info->emplace_back(cloneInfo, kProgressCloneComplete);
312+
break;
313+
}
314+
case CloneStatus::cleaning:
315+
case CloneStatus::errorCleaning:
316+
case CloneStatus::error:
317+
case CloneStatus::retrying: {
318+
info->emplace_back(cloneInfo, kProgressCloneError);
319+
break;
320+
}
321+
case CloneStatus::cloning:
322+
case CloneStatus::recovering: {
323+
TaskIdType taskId = cloneInfo.GetTaskId();
324+
std::shared_ptr<CloneTaskBase> task =
325+
cloneTaskMgr_->GetTask(taskId);
326+
if (task != nullptr) {
327+
info->emplace_back(cloneInfo,
328+
task->GetTaskInfo()->GetProgress());
329+
} else {
330+
TaskCloneInfo tcInfo;
331+
ret = GetFinishedCloneTask(taskId, &tcInfo);
332+
if (ret < 0) {
333+
return ret;
334+
}
335+
info->emplace_back(tcInfo);
336+
}
337+
break;
338+
}
339+
case CloneStatus::metaInstalled: {
340+
info->emplace_back(cloneInfo, kProgressMetaInstalled);
341+
break;
342+
}
343+
default:
344+
LOG(ERROR) << "can not reach here!, status = "
345+
<< static_cast<int>(cloneInfo.GetStatus());
346+
return kErrCodeInternalError;
347+
}
348+
}
349+
}
350+
return kErrCodeSuccess;
351+
}
352+
289353
int CloneServiceManager::GetCloneTaskInfoInner(
290354
std::vector<CloneInfo> cloneInfos,
291355
const std::string &user,
@@ -337,6 +401,35 @@ int CloneServiceManager::GetCloneTaskInfoInner(
337401
return kErrCodeSuccess;
338402
}
339403

404+
bool CloneFilterCondition::IsMatchCondition(const CloneInfo &cloneInfo) {
405+
if (user_ != nullptr && *user_ != cloneInfo.GetUser()) {
406+
return false;
407+
}
408+
409+
if (source_ != nullptr && *source_ != cloneInfo.GetSrc()) {
410+
return false;
411+
}
412+
413+
if (destination_ != nullptr && *destination_ != cloneInfo.GetDest()) {
414+
return false;
415+
}
416+
417+
if (uuid_ != nullptr && *uuid_ != cloneInfo.GetTaskId()) {
418+
return false;
419+
}
420+
421+
if (status_ != nullptr
422+
&& std::stoi(*status_) != static_cast<int>(cloneInfo.GetStatus())) {
423+
return false;
424+
}
425+
426+
if (type_ != nullptr
427+
&& std::stoi(*type_) != static_cast<int>(cloneInfo.GetTaskType())) {
428+
return false;
429+
}
430+
return true;
431+
}
432+
340433
int CloneServiceManager::GetFinishedCloneTask(
341434
const TaskIdType &taskId,
342435
TaskCloneInfo *taskCloneInfoOut) {

‎src/snapshotcloneserver/clone/clone_service_manager.h

Lines changed: 82 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,11 +67,16 @@ class TaskCloneInfo {
6767
cloneTaskObj["UUID"] = info.GetTaskId();
6868
cloneTaskObj["User"] = info.GetUser();
6969
cloneTaskObj["File"] = info.GetDest();
70+
cloneTaskObj["Src"] = info.GetSrc();
7071
cloneTaskObj["TaskType"] = static_cast<int> (
7172
info.GetTaskType());
7273
cloneTaskObj["TaskStatus"] = static_cast<int> (
7374
info.GetStatus());
75+
cloneTaskObj["IsLazy"] = info.GetIsLazy();
76+
cloneTaskObj["NextStep"] = static_cast<int> (info.GetNextStep());
7477
cloneTaskObj["Time"] = info.GetTime();
78+
cloneTaskObj["Progress"] = GetCloneProgress();
79+
cloneTaskObj["FileType"] = static_cast<int> (info.GetFileType());
7580
return cloneTaskObj;
7681
}
7782

@@ -80,11 +85,16 @@ class TaskCloneInfo {
8085
info.SetTaskId(jsonObj["UUID"].asString());
8186
info.SetUser(jsonObj["User"].asString());
8287
info.SetDest(jsonObj["File"].asString());
88+
info.SetSrc(jsonObj["Src"].asString());
8389
info.SetTaskType(static_cast<CloneTaskType>(
8490
jsonObj["TaskType"].asInt()));
8591
info.SetStatus(static_cast<CloneStatus>(
8692
jsonObj["TaskStatus"].asInt()));
93+
info.SetIsLazy(jsonObj["IsLazy"].asBool());
94+
info.SetNextStep(static_cast<CloneStep>(jsonObj["NextStep"].asInt()));
8795
info.SetTime(jsonObj["Time"].asUInt64());
96+
info.SetFileType(static_cast<CloneFileType>(
97+
jsonObj["FileType"].asInt()));
8898
SetCloneInfo(info);
8999
}
90100

@@ -93,6 +103,54 @@ class TaskCloneInfo {
93103
uint32_t cloneProgress_;
94104
};
95105

106+
class CloneFilterCondition {
107+
public:
108+
CloneFilterCondition()
109+
: uuid_(nullptr),
110+
source_(nullptr),
111+
destination_(nullptr),
112+
user_(nullptr),
113+
status_(nullptr),
114+
type_(nullptr) {}
115+
116+
CloneFilterCondition(const std::string *uuid, const std::string *source,
117+
const std::string *destination, const std::string *user,
118+
const std::string *status, const std::string *type)
119+
: uuid_(uuid),
120+
source_(source),
121+
destination_(destination),
122+
user_(user),
123+
status_(status),
124+
type_(type) {}
125+
bool IsMatchCondition(const CloneInfo &cloneInfo);
126+
127+
private:
128+
void SetUuid(const std::string *uuid) {
129+
uuid_ = uuid;
130+
}
131+
void SetSource(const std::string *source) {
132+
source_ = source;
133+
}
134+
void SetDestination(const std::string *destination) {
135+
destination_ = destination;
136+
}
137+
void SetUser(const std::string *user) {
138+
user_ = user;
139+
}
140+
void SetStatus(const std::string *status) {
141+
status_ = status;
142+
}
143+
void SetType(const std::string *type) {
144+
type_ = type;
145+
}
146+
const std::string *uuid_;
147+
const std::string *source_;
148+
const std::string *destination_;
149+
const std::string *user_;
150+
const std::string *status_;
151+
const std::string *type_;
152+
};
153+
96154
class CloneServiceManager {
97155
public:
98156
CloneServiceManager(
@@ -178,8 +236,6 @@ class CloneServiceManager {
178236
* @brief 查询某个用户的克隆/恢复任务信息
179237
*
180238
* @param user 用户名
181-
* @param taskId 指定的任务Id, 为nullptr时不指定
182-
* @param fileName 指定的文件名, 为nullptr时不指定
183239
* @param info 克隆/恢复任务信息
184240
*
185241
* @return 错误码
@@ -215,6 +271,17 @@ class CloneServiceManager {
215271
const std::string &fileName,
216272
std::vector<TaskCloneInfo> *info);
217273

274+
/**
275+
* @brief 通过过滤条件查询某个用户的克隆/恢复任务信息
276+
*
277+
* @param filter 过滤条件
278+
* @param info 克隆/恢复任务信息
279+
*
280+
* @return 错误码
281+
*/
282+
virtual int GetCloneTaskInfoByFilter(const CloneFilterCondition &filter,
283+
std::vector<TaskCloneInfo> *info);
284+
218285
/**
219286
* @brief 清除失败的clone/Recover任务、状态、文件
220287
*
@@ -247,6 +314,19 @@ class CloneServiceManager {
247314
const std::string &user,
248315
std::vector<TaskCloneInfo> *info);
249316

317+
/**
318+
* @brief 从给定的任务列表中获取符合过滤条件的任务集
319+
*
320+
* @param cloneInfos 克隆/恢复信息
321+
* @param filter 过滤条件
322+
* @param[out] info 克隆/恢复任务信息
323+
*
324+
* @return 错误码
325+
*/
326+
int GetCloneTaskInfoInner(std::vector<CloneInfo> cloneInfos,
327+
CloneFilterCondition filter,
328+
std::vector<TaskCloneInfo> *info);
329+
250330
/**
251331
* @brief 获取已经完成任务信息
252332
*

‎src/snapshotcloneserver/common/define.cpp

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ const char* kRecoverAction = "Recover";
3838
const char* kGetCloneTasksAction = "GetCloneTasks";
3939
const char* kCleanCloneTaskAction = "CleanCloneTask";
4040
const char* kFlattenAction = "Flatten";
41+
const char* kGetFileSnapshotListAction = "GetFileSnapshotList";
42+
const char* kGetCloneTaskListAction = "GetCloneTaskList";
4143

4244
const char* kActionStr = "Action";
4345
const char* kVersionStr = "Version";
@@ -50,6 +52,8 @@ const char* kOffsetStr = "Offset";
5052
const char* kSourceStr = "Source";
5153
const char* kDestinationStr = "Destination";
5254
const char* kLazyStr = "Lazy";
55+
const char* kStatusStr = "Status";
56+
const char* kTypeStr = "Type";
5357

5458
const char* kCodeStr = "Code";
5559
const char* kMessageStr = "Message";
@@ -79,7 +83,8 @@ std::map<int, std::string> code2Msg = {
7983
{kErrCodeCannotCleanCloneUnfinished, "Cannot clean task unfinished."},
8084
{kErrCodeSnapshotCountReachLimit, "Snapshot count reach the limit."},
8185
{kErrCodeFileExist, "File exist."},
82-
{kErrCodeTaskIsFull, "Task is full."}
86+
{kErrCodeTaskIsFull, "Task is full."},
87+
{kErrCodeNotSupport, "Not support."},
8388
};
8489

8590
std::string BuildErrorMessage(

‎src/snapshotcloneserver/common/define.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@ extern const char* kRecoverAction;
4141
extern const char* kGetCloneTasksAction;
4242
extern const char* kCleanCloneTaskAction;
4343
extern const char* kFlattenAction;
44+
extern const char* kGetFileSnapshotListAction;
45+
extern const char* kGetCloneTaskListAction;
4446

4547
// param
4648
extern const char* kActionStr;
@@ -54,6 +56,8 @@ extern const char* kOffsetStr;
5456
extern const char* kSourceStr;
5557
extern const char* kDestinationStr;
5658
extern const char* kLazyStr;
59+
extern const char* kStatusStr;
60+
extern const char* kTypeStr;
5761

5862
// json key
5963
extern const char* kCodeStr;

0 commit comments

Comments
 (0)