Skip to content

Commit 801403c

Browse files
charisubai-charisu
charisu
authored and
bai-charisu
committed
curve_ops_tool prove multiple pool CLDCFS-2617
Change-Id: I183eaec20cf765f94312ba3c81b680c995695672
1 parent ad4a2df commit 801403c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+2244
-940
lines changed

‎proto/nameserver2.proto

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -448,9 +448,18 @@ message GetAllocatedSizeResponse {
448448
required StatusCode statusCode = 1;
449449
// 文件或目录的分配大小
450450
optional uint64 allocatedSize = 2;
451-
// 物理上分配的时间，因为对分配了1GB空间的文件来说
452-
// 如果是三副本，实际上相当于分配了3GB物理空间
453-
optional uint64 physicalAllocatedSize = 3;
451+
// key是逻辑池id，value是分配大小
452+
map<uint32, uint64> allocSizeMap = 3;
453+
}
454+
455+
message GetFileSizeRequest {
456+
required string fileName = 1;
457+
}
458+
459+
message GetFileSizeResponse {
460+
required StatusCode statusCode = 1;
461+
// 文件或目录的file length
462+
optional uint64 fileSize = 2;
454463
}
455464

456465
message ClientInfo {
@@ -512,6 +521,7 @@ service CurveFSService {
512521

513522
// curve status
514523
rpc GetAllocatedSize(GetAllocatedSizeRequest) returns (GetAllocatedSizeResponse);
524+
rpc GetFileSize(GetFileSizeRequest) returns (GetFileSizeResponse);
515525
rpc ListClient(ListClientRequest) returns (ListClientResponse);
516526
rpc FindFileMountPoint(FindFileMountPointRequest) returns (FindFileMountPointResponse);
517527
}

‎src/mds/nameserver2/curvefs.cpp

Lines changed: 78 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -303,16 +303,18 @@ StatusCode CurveFS::GetFileInfo(const std::string & filename,
303303
}
304304

305305
AllocatedSize& AllocatedSize::operator+=(const AllocatedSize& rhs) {
306-
allocatedSize += rhs.allocatedSize;
307-
physicalAllocatedSize += rhs.physicalAllocatedSize;
306+
total += rhs.total;
307+
for (const auto& item : rhs.allocSizeMap) {
308+
allocSizeMap[item.first] += item.second;
309+
}
308310
return *this;
309311
}
310312

311313
StatusCode CurveFS::GetAllocatedSize(const std::string& fileName,
312314
AllocatedSize* allocatedSize) {
313315
assert(allocatedSize != nullptr);
314-
allocatedSize->allocatedSize = 0;
315-
allocatedSize->physicalAllocatedSize = 0;
316+
allocatedSize->total = 0;
317+
allocatedSize->allocSizeMap.clear();
316318
FileInfo fileInfo;
317319
auto ret = GetFileInfo(fileName, &fileInfo);
318320
if (ret != StatusCode::kOK) {
@@ -349,18 +351,10 @@ StatusCode CurveFS::GetFileAllocSize(const std::string& fileName,
349351
return StatusCode::kStorageError;
350352
}
351353
for (const auto& segment : segments) {
352-
const auto & logicPoolId = segment.logicalpoolid();
353-
LogicalPool logicPool;
354-
if (!topology_->GetLogicalPool(logicPoolId, &logicPool)) {
355-
LOG(ERROR) << "Get logical pool " << logicPoolId
356-
<< " from topology failed!";
357-
return StatusCode::KInternalError;
358-
}
359-
uint64_t replicasNum = logicPool.GetReplicaNum();
360-
allocSize->physicalAllocatedSize +=
361-
fileInfo.segmentsize() * replicasNum;
354+
const auto & poolId = segment.logicalpoolid();
355+
allocSize->allocSizeMap[poolId] += fileInfo.segmentsize();
362356
}
363-
allocSize->allocatedSize = fileInfo.segmentsize() * segments.size();
357+
allocSize->total = fileInfo.segmentsize() * segments.size();
364358
return StatusCode::kOK;
365359
}
366360

@@ -391,6 +385,75 @@ StatusCode CurveFS::GetDirAllocSize(const std::string& fileName,
391385
return StatusCode::kOK;
392386
}
393387

388+
StatusCode CurveFS::GetFileSize(const std::string& fileName, uint64_t* size) {
389+
assert(size != nullptr);
390+
*size = 0;
391+
FileInfo fileInfo;
392+
auto ret = GetFileInfo(fileName, &fileInfo);
393+
if (ret != StatusCode::kOK) {
394+
return ret;
395+
}
396+
397+
if (fileInfo.filetype() != curve::mds::FileType::INODE_DIRECTORY &&
398+
fileInfo.filetype() != curve::mds::FileType::INODE_PAGEFILE) {
399+
LOG(ERROR) << "GetFileSize not support file type : "
400+
<< fileInfo.filetype() << ", fileName = " << fileName;
401+
return StatusCode::kNotSupported;
402+
}
403+
return GetFileSize(fileName, fileInfo, size);
404+
}
405+
406+
StatusCode CurveFS::GetFileSize(const std::string& fileName,
407+
const FileInfo& fileInfo,
408+
uint64_t* fileSize) {
409+
// 如果是文件的话直接返回file length
410+
switch (fileInfo.filetype()) {
411+
case FileType::INODE_PAGEFILE: {
412+
*fileSize = fileInfo.length();
413+
return StatusCode::kOK;
414+
}
415+
case FileType::INODE_SNAPSHOT_PAGEFILE: {
416+
// 快照文件不统计file size，所以使file size为0
417+
*fileSize = 0;
418+
return StatusCode::kOK;
419+
}
420+
case FileType::INODE_DIRECTORY: {
421+
break;
422+
}
423+
default: {
424+
LOG(ERROR) << "Get file size of type "
425+
<< FileType_Name(fileInfo.filetype())
426+
<< " not supported";
427+
return StatusCode::kNotSupported;
428+
}
429+
}
430+
// 如果是目录，则list dir，并递归计算file size
431+
std::vector<FileInfo> files;
432+
StatusCode ret = ReadDir(fileName, &files);
433+
if (ret != StatusCode::kOK) {
434+
LOG(ERROR) << "ReadDir Fail, fileName: " << fileName
435+
<< ", error code: " << ret;
436+
return ret;
437+
}
438+
for (auto& file : files) {
439+
std::string fullPathName;
440+
if (fileName == "/") {
441+
fullPathName = fileName + file.filename();
442+
} else {
443+
fullPathName = fileName + "/" + file.filename();
444+
}
445+
uint64_t size = 0;
446+
ret = GetFileSize(fullPathName, file, &size);
447+
if (ret != StatusCode::kOK) {
448+
LOG(ERROR) << "Get file size of " << fullPathName
449+
<< " fail, error code: " << ret;
450+
return ret;
451+
}
452+
*fileSize += size;
453+
}
454+
return StatusCode::kOK;
455+
}
456+
394457
StatusCode CurveFS::isDirectoryEmpty(const FileInfo &fileInfo, bool *result) {
395458
assert(fileInfo.filetype() == FileType::INODE_DIRECTORY);
396459
std::vector<FileInfo> fileInfoList;

‎src/mds/nameserver2/curvefs.h

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
#include <memory>
3030
#include <thread> //NOLINT
3131
#include <chrono>
32+
#include <unordered_map>
3233
#include "proto/nameserver2.pb.h"
3334
#include "src/mds/nameserver2/namespace_storage.h"
3435
#include "src/mds/common/mds_define.h"
@@ -57,10 +58,10 @@ struct CurveFSOption {
5758

5859
struct AllocatedSize {
5960
// mds给文件分配的segment的大小
60-
uint64_t allocatedSize;
61-
// 实际会占用的底层空间
62-
uint64_t physicalAllocatedSize;
63-
AllocatedSize() : allocatedSize(0), physicalAllocatedSize(0) {}
61+
uint64_t total;
62+
// 在每个池子里的分配大小
63+
std::unordered_map<PoolIdType, uint64_t> allocSizeMap;
64+
AllocatedSize() : total(0) {}
6465
AllocatedSize& operator+=(const AllocatedSize& rhs);
6566
};
6667

@@ -141,6 +142,14 @@ class CurveFS {
141142
StatusCode GetAllocatedSize(const std::string& fileName,
142143
AllocatedSize* allocatedSize);
143144

145+
/**
146+
* @brief 获取文件或目录的大小
147+
* @brief fileName：文件名
148+
* @param[out]: size 文件或目录fileLength大小
149+
* @return 是否成功，成功返回StatusCode::kOK
150+
*/
151+
StatusCode GetFileSize(const std::string& fileName, uint64_t* size);
152+
144153
/**
145154
* @brief 删除文件
146155
* @param[in] filename:文件名
@@ -573,6 +582,17 @@ class CurveFS {
573582
const FileInfo& fileInfo,
574583
AllocatedSize* allocSize);
575584

585+
/**
586+
* @brief 获取文件或目录的大小
587+
* @param: fileName 文件名
588+
* @param: fileInfo 文件信息
589+
* @param[out]: fileSize 文件或目录的大小
590+
* @return 是否成功，成功返回StatusCode::kOK
591+
*/
592+
StatusCode GetFileSize(const std::string& fileName,
593+
const FileInfo& fileInfo,
594+
uint64_t* fileSize);
595+
576596
private:
577597
FileInfo rootFileInfo_;
578598
std::shared_ptr<NameServerStorage> storage_;

‎src/mds/nameserver2/namespace_service.cpp

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1681,8 +1681,10 @@ void NameSpaceService::GetAllocatedSize(
16811681
return;
16821682
} else {
16831683
response->set_statuscode(StatusCode::kOK);
1684-
response->set_allocatedsize(allocSize.allocatedSize);
1685-
response->set_physicalallocatedsize(allocSize.physicalAllocatedSize);
1684+
response->set_allocatedsize(allocSize.total);
1685+
for (const auto& item : allocSize.allocSizeMap) {
1686+
response->mutable_allocsizemap()->insert({item.first, item.second});
1687+
}
16861688
LOG(INFO) << "logid = " << cntl->log_id()
16871689
<< ", GetAllocatedSize ok, fileName = " << request->filename()
16881690
<< ", allocatedSize = " << response->allocatedsize() / kGB
@@ -1692,6 +1694,38 @@ void NameSpaceService::GetAllocatedSize(
16921694
return;
16931695
}
16941696

1697+
void NameSpaceService::GetFileSize(
1698+
::google::protobuf::RpcController* controller,
1699+
const ::curve::mds::GetFileSizeRequest* request,
1700+
::curve::mds::GetFileSizeResponse* response,
1701+
::google::protobuf::Closure* done) {
1702+
brpc::ClosureGuard doneGuard(done);
1703+
brpc::Controller* cntl = static_cast<brpc::Controller*>(controller);
1704+
1705+
1706+
LOG(INFO) << "logid = " << cntl->log_id()
1707+
<< ", GetFileSize request, fileName = " << request->filename();
1708+
1709+
StatusCode retCode;
1710+
uint64_t fileSize = 0;
1711+
retCode = kCurveFS.GetFileSize(request->filename(), &fileSize);
1712+
if (retCode != StatusCode::kOK) {
1713+
response->set_statuscode(retCode);
1714+
LOG(ERROR) << "logid = " << cntl->log_id()
1715+
<< ", GetFileSize fail, fileName = " << request->filename()
1716+
<< ", statusCode = " << retCode
1717+
<< ", StatusCode_Name = " << StatusCode_Name(retCode);
1718+
return;
1719+
} else {
1720+
response->set_statuscode(StatusCode::kOK);
1721+
response->set_filesize(fileSize);
1722+
LOG(INFO) << "logid = " << cntl->log_id()
1723+
<< ", GetFileSize ok, fileName = " << request->filename()
1724+
<< ", fileSize = " << response->filesize() / kGB << "GB";
1725+
}
1726+
return;
1727+
}
1728+
16951729
void NameSpaceService::ListClient(
16961730
::google::protobuf::RpcController* controller,
16971731
const ::curve::mds::ListClientRequest* request,

‎src/mds/nameserver2/namespace_service.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,10 @@ class NameSpaceService: public CurveFSService {
154154
const ::curve::mds::GetAllocatedSizeRequest* request,
155155
::curve::mds::GetAllocatedSizeResponse* response,
156156
::google::protobuf::Closure* done) override;
157+
void GetFileSize(::google::protobuf::RpcController* controller,
158+
const ::curve::mds::GetFileSizeRequest* request,
159+
::curve::mds::GetFileSizeResponse* response,
160+
::google::protobuf::Closure* done) override;
157161
void ListClient(::google::protobuf::RpcController* controller,
158162
const ::curve::mds::ListClientRequest* request,
159163
::curve::mds::ListClientResponse* response,

‎src/mds/topology/topology_metric.cpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,13 +196,18 @@ void TopologyMetricService::UpdateTopologyMetrics() {
196196
it->second->diskUsed.set_value(totalDiskUsed);
197197
int64_t diskAlloc = 0;
198198
allocStatistic_->GetAllocByLogicalPool(pid, &diskAlloc);
199+
it->second->logicalAlloc.set_value(diskAlloc);
199200
// 需乘以副本数
200201
it->second->diskAlloc.set_value(diskAlloc * pool.GetReplicaNum());
201202

202203
it->second->chunkSizeUsedBytes.set_value(totalChunkSizeUsedBytes);
203204
it->second->chunkSizeLeftBytes.set_value(totalChunkSizeLeftBytes);
204205
it->second->chunkSizeTrashedBytes.set_value(totalChunkSizeTrashedBytes);
205206
it->second->chunkSizeTotalBytes.set_value(totalChunkSizeBytes);
207+
if (pool.GetReplicaNum() != 0) {
208+
it->second->logicalCapacity.set_value(
209+
totalChunkSizeBytes / pool.GetReplicaNum());
210+
}
206211
}
207212
// 移除已经不存在的逻辑池metric
208213
for (auto iy = gLogicalPoolMetrics.begin();

‎src/mds/topology/topology_metric.h

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,10 @@ struct LogicalPoolMetric {
170170
bvar::Status<uint64_t> chunkSizeTrashedBytes;
171171
// 总容量
172172
bvar::Status<uint64_t> chunkSizeTotalBytes;
173+
// 逻辑总容量
174+
bvar::Status<uint64_t> logicalCapacity;
175+
// 已分配的字节
176+
bvar::Status<uint64_t> logicalAlloc;
173177

174178
explicit LogicalPoolMetric(const std::string &logicalPoolName) :
175179
chunkServerNum(kTopologyLogicalPoolMetricPrefix,
@@ -225,7 +229,11 @@ struct LogicalPoolMetric {
225229
chunkSizeTrashedBytes(kTopologyLogicalPoolMetricPrefix,
226230
logicalPoolName + "_chunkSizeTrashedBytes", 0),
227231
chunkSizeTotalBytes(kTopologyLogicalPoolMetricPrefix,
228-
logicalPoolName + "_chunkSizeTotalBytes", 0) {}
232+
logicalPoolName + "_chunkSizeTotalBytes", 0),
233+
logicalCapacity(kTopologyLogicalPoolMetricPrefix,
234+
logicalPoolName + "_logicalCapacity", 0),
235+
logicalAlloc(kTopologyLogicalPoolMetricPrefix,
236+
logicalPoolName + "_logicalAlloc", 0) {}
229237
};
230238
using LogicalPoolMetricPtr = std::unique_ptr<LogicalPoolMetric>;
231239

‎src/tools/curve_tool_define.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,8 @@ const char kChunkserverStatusCmd[] = "chunkserver-status";
4646
const char kMdsStatusCmd[] = "mds-status";
4747
const char kEtcdStatusCmd[] = "etcd-status";
4848
const char kChunkserverListCmd[] = "chunkserver-list";
49+
const char kServerListCmd[] = "server-list";
50+
const char kLogicalPoolList[] = "logical-pool-list";
4951
const char kClientStatusCmd[] = "client-status";
5052
const char kClientListCmd[] = "client-list";
5153
const char kSnapshotCloneStatusCmd[] = "snapshot-clone-status";

‎src/tools/curve_tool_factory.cpp

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,6 @@ std::shared_ptr<CurveTool> CurveToolFactory::GenerateCurveTool(
4747
std::shared_ptr<StatusTool> CurveToolFactory::GenerateStatusTool() {
4848
auto mdsClient = std::make_shared<MDSClient>();
4949
auto etcdClient = std::make_shared<EtcdClient>();
50-
auto nameSpaceTool =
51-
std::make_shared<NameSpaceToolCore>(mdsClient);
5250
auto csClient = std::make_shared<ChunkServerClient>();
5351
auto copysetCheck =
5452
std::make_shared<CopysetCheckCore>(mdsClient, csClient);
@@ -58,9 +56,8 @@ std::shared_ptr<StatusTool> CurveToolFactory::GenerateStatusTool() {
5856
auto versionTool = std::make_shared<VersionTool>(mdsClient, metricClient,
5957
snapshotCloneClient);
6058
return std::make_shared<StatusTool>(mdsClient, etcdClient,
61-
nameSpaceTool, copysetCheck,
62-
versionTool, metricClient,
63-
snapshotCloneClient);
59+
copysetCheck, versionTool,
60+
metricClient, snapshotCloneClient);
6461
}
6562

6663
std::shared_ptr<NameSpaceTool> CurveToolFactory::GenerateNameSpaceTool() {

‎src/tools/etcd_client.cpp

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -91,8 +91,9 @@ int EtcdClient::GetAndCheckEtcdVersion(std::string* version,
9191
cntl.http_request().uri() = addr + kEtcdVersionUri;
9292
httpChannel.CallMethod(NULL, &cntl, NULL, NULL, NULL);
9393
if (cntl.Failed()) {
94-
std::cout << "Access " << addr + kEtcdVersionUri << " failed"
95-
<< std::endl;;
94+
std::cout << "Access " << addr + kEtcdVersionUri
95+
<< " failed, error text: " << cntl.ErrorText()
96+
<< std::endl;
9697
failedList->emplace_back(addr);
9798
continue;
9899
}

0 commit comments

Comments
 (0)