Skip to content

Commit 424a7fd

Browse files
rock-gitchuandew
rock-git
authored and
chuandew
committed
[feat][mdsv2] Optimize meta codec.
1 parent a43509a commit 424a7fd

24 files changed

+1399
-937
lines changed

‎src/mdsv2/background/gc.cc

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -278,10 +278,22 @@ Status GcProcessor::LaunchGc() {
278278
return Status(pb::error::EINTERNAL, "not own lock");
279279
}
280280

281-
ScanDelSlice();
282-
ScanDelFile();
283-
ScanExpiredFileSession();
281+
auto fses = file_system_set_->GetAllFileSystem();
284282

283+
// delslice
284+
for (auto& fs : fses) {
285+
ScanDelSlice(fs->FsId());
286+
}
287+
288+
// delfile
289+
for (auto& fs : fses) {
290+
ScanDelFile(fs->FsId());
291+
}
292+
293+
// filesession
294+
for (auto& fs : fses) {
295+
ScanExpiredFileSession(fs->FsId());
296+
}
285297
return Status::OK();
286298
}
287299

@@ -319,10 +331,10 @@ Status GcProcessor::GetClientList(std::set<std::string>& clients) {
319331
return Status::OK();
320332
}
321333

322-
void GcProcessor::ScanDelSlice() {
334+
void GcProcessor::ScanDelSlice(uint32_t fs_id) {
323335
Trace trace;
324336
uint32_t count = 0, exec_count = 0;
325-
ScanDelSliceOperation operation(trace, [&](const std::string& key, const std::string& value) -> bool {
337+
ScanDelSliceOperation operation(trace, fs_id, [&](const std::string& key, const std::string& value) -> bool {
326338
++count;
327339

328340
uint32_t fs_id = 0;
@@ -348,10 +360,10 @@ void GcProcessor::ScanDelSlice() {
348360
status.error_str());
349361
}
350362

351-
void GcProcessor::ScanDelFile() {
363+
void GcProcessor::ScanDelFile(uint32_t fs_id) {
352364
Trace trace;
353365
uint32_t count = 0, exec_count = 0;
354-
ScanDelFileOperation operation(trace, [&](const std::string& key, const std::string& value) -> bool {
366+
ScanDelFileOperation operation(trace, fs_id, [&](const std::string& key, const std::string& value) -> bool {
355367
++count;
356368

357369
uint32_t fs_id = 0;
@@ -381,7 +393,7 @@ void GcProcessor::ScanDelFile() {
381393
status.error_str());
382394
}
383395

384-
void GcProcessor::ScanExpiredFileSession() {
396+
void GcProcessor::ScanExpiredFileSession(uint32_t fs_id) {
385397
// get alive clients
386398
// to dead clients, we will clean their file sessions
387399
std::set<std::string> alive_clients;
@@ -394,7 +406,7 @@ void GcProcessor::ScanExpiredFileSession() {
394406
Trace trace;
395407
uint32_t count = 0, exec_count = 0;
396408
std::vector<FileSessionEntry> file_sessions;
397-
ScanFileSessionOperation operation(trace, [&](const FileSessionEntry& file_session) -> bool {
409+
ScanFileSessionOperation operation(trace, fs_id, [&](const FileSessionEntry& file_session) -> bool {
398410
++count;
399411

400412
if (ShouldCleanFileSession(file_session, alive_clients)) {

‎src/mdsv2/background/gc.h

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -158,9 +158,9 @@ class GcProcessor {
158158

159159
Status GetClientList(std::set<std::string>& clients);
160160

161-
void ScanDelSlice();
162-
void ScanDelFile();
163-
void ScanExpiredFileSession();
161+
void ScanDelSlice(uint32_t fs_id);
162+
void ScanDelFile(uint32_t fs_id);
163+
void ScanExpiredFileSession(uint32_t fs_id);
164164

165165
static bool ShouldDeleteFile(const AttrType& attr);
166166
static bool ShouldCleanFileSession(const FileSessionEntry& file_session, const std::set<std::string>& alive_clients);

‎src/mdsv2/client/main.cc

Lines changed: 7 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -49,15 +49,8 @@ DEFINE_uint64(parent, 0, "parent");
4949
DEFINE_string(parents, "", "parents");
5050
DEFINE_uint32(num, 1, "num");
5151

52-
DEFINE_string(lock_table_name, "dingofs-lock", "lock table name");
53-
DEFINE_string(autoincrement_table_name, "dingofs-autoincrement", "autoincrement table name");
54-
DEFINE_string(heartbeat_table_name, "dingofs-heartbeat", "heartbeat table name");
55-
DEFINE_string(fs_table_name, "dingofs-fs", "fs table name");
56-
DEFINE_string(quota_table_name, "dingofs-quota", "quota table name");
57-
DEFINE_string(stats_table_name, "dingofs-stats", "stats table name");
58-
DEFINE_string(filesession_table_name, "dingofs-filesession", "file session table name");
59-
DEFINE_string(del_slice_table_name, "dingofs-delslice", "del slice table name");
60-
DEFINE_string(del_file_table_name, "dingofs-delfile", "del file table name");
52+
DEFINE_string(meta_table_name, "dingofs-meta", "meta table name");
53+
DEFINE_string(fsstats_table_name, "dingofs-fsstats", "fs stats table name");
6154

6255
DEFINE_uint32(max_bytes, 1024 * 1024 * 1024, "max bytes");
6356
DEFINE_uint32(max_inodes, 1000000, "max inodes");
@@ -260,43 +253,15 @@ int main(int argc, char* argv[]) {
260253
return -1;
261254
}
262255

263-
if (lower_cmd == Helper::ToLowerCase("CreateLockTable")) {
264-
store_client.CreateLockTable(FLAGS_lock_table_name);
265-
266-
} else if (lower_cmd == Helper::ToLowerCase("CreateAutoIncrementTable")) {
267-
store_client.CreateAutoIncrementTable(FLAGS_autoincrement_table_name);
268-
269-
} else if (lower_cmd == Helper::ToLowerCase("CreateHeartbeatTable")) {
270-
store_client.CreateHeartbeatTable(FLAGS_heartbeat_table_name);
271-
272-
} else if (lower_cmd == Helper::ToLowerCase("CreateFsTable")) {
273-
store_client.CreateFsTable(FLAGS_fs_table_name);
274-
275-
} else if (lower_cmd == Helper::ToLowerCase("CreateFsQuotaTable")) {
276-
store_client.CreateFsQuotaTable(FLAGS_quota_table_name);
256+
if (lower_cmd == Helper::ToLowerCase("CreateMetaTable")) {
257+
store_client.CreateMetaTable(FLAGS_meta_table_name);
277258

278259
} else if (lower_cmd == Helper::ToLowerCase("CreateFsStatsTable")) {
279-
store_client.CreateFsStatsTable(FLAGS_stats_table_name);
280-
281-
} else if (lower_cmd == Helper::ToLowerCase("CreateFileSessionTable")) {
282-
store_client.CreateFileSessionTable(FLAGS_filesession_table_name);
283-
284-
} else if (lower_cmd == Helper::ToLowerCase("CreateDelSliceTable")) {
285-
store_client.CreateDelSliceTable(FLAGS_del_slice_table_name);
286-
287-
} else if (lower_cmd == Helper::ToLowerCase("CreateDelFileTable")) {
288-
store_client.CreateDelFileTable(FLAGS_del_file_table_name);
260+
store_client.CreateFsStatsTable(FLAGS_fsstats_table_name);
289261

290262
} else if (lower_cmd == Helper::ToLowerCase("CreateAllTable")) {
291-
store_client.CreateLockTable(FLAGS_lock_table_name);
292-
store_client.CreateAutoIncrementTable(FLAGS_autoincrement_table_name);
293-
store_client.CreateHeartbeatTable(FLAGS_heartbeat_table_name);
294-
store_client.CreateFsTable(FLAGS_fs_table_name);
295-
store_client.CreateFsQuotaTable(FLAGS_quota_table_name);
296-
store_client.CreateFsStatsTable(FLAGS_stats_table_name);
297-
store_client.CreateFileSessionTable(FLAGS_filesession_table_name);
298-
store_client.CreateDelSliceTable(FLAGS_del_slice_table_name);
299-
store_client.CreateDelFileTable(FLAGS_del_file_table_name);
263+
store_client.CreateMetaTable(FLAGS_meta_table_name);
264+
store_client.CreateFsStatsTable(FLAGS_fsstats_table_name);
300265

301266
} else if (lower_cmd == Helper::ToLowerCase("tree")) {
302267
store_client.PrintDentryTree(FLAGS_fs_id, true);

‎src/mdsv2/client/store.cc

Lines changed: 7 additions & 119 deletions
Original file line numberDiff line numberDiff line change
@@ -46,90 +46,26 @@ bool StoreClient::Init(const std::string& coor_addr) {
4646
return kv_storage_->Init(store_addrs);
4747
}
4848

49-
bool StoreClient::CreateLockTable(const std::string& name) {
49+
bool StoreClient::CreateMetaTable(const std::string& name) {
5050
int64_t table_id = 0;
51-
KVStorage::TableOption option;
52-
MetaCodec::GetLockTableRange(option.start_key, option.end_key);
51+
Range range = MetaCodec::GetMetaTableRange();
52+
KVStorage::TableOption option = {.start_key = range.start, .end_key = range.end};
5353
auto status = kv_storage_->CreateTable(name, option, table_id);
5454
if (!status.ok()) {
55-
DINGO_LOG(ERROR) << fmt::format("create lock table fail, error: {}.", status.error_str());
55+
DINGO_LOG(ERROR) << fmt::format("create meta table fail, error: {}.", status.error_str());
5656
return false;
5757
}
5858

59-
DINGO_LOG(INFO) << fmt::format("create lock table success, start_key({}), end_key({}).",
60-
Helper::StringToHex(option.start_key), Helper::StringToHex(option.end_key));
61-
62-
return true;
63-
}
64-
65-
bool StoreClient::CreateAutoIncrementTable(const std::string& name) {
66-
int64_t table_id = 0;
67-
KVStorage::TableOption option;
68-
MetaCodec::GetAutoIncrementTableRange(option.start_key, option.end_key);
69-
auto status = kv_storage_->CreateTable(name, option, table_id);
70-
if (!status.ok()) {
71-
DINGO_LOG(ERROR) << fmt::format("create autoincrement table fail, error: {}.", status.error_str());
72-
return false;
73-
}
74-
75-
DINGO_LOG(INFO) << fmt::format("create autoincrement table success, start_key({}), end_key({}).",
76-
Helper::StringToHex(option.start_key), Helper::StringToHex(option.end_key));
77-
78-
return true;
79-
}
80-
81-
bool StoreClient::CreateHeartbeatTable(const std::string& name) {
82-
int64_t table_id = 0;
83-
KVStorage::TableOption option;
84-
MetaCodec::GetHeartbeatTableRange(option.start_key, option.end_key);
85-
auto status = kv_storage_->CreateTable(name, option, table_id);
86-
if (!status.ok()) {
87-
DINGO_LOG(ERROR) << fmt::format("create heartbeat table fail, error: {}.", status.error_str());
88-
return false;
89-
}
90-
91-
DINGO_LOG(INFO) << fmt::format("create heartbeat table success, start_key({}), end_key({}).",
92-
Helper::StringToHex(option.start_key), Helper::StringToHex(option.end_key));
93-
94-
return true;
95-
}
96-
97-
bool StoreClient::CreateFsTable(const std::string& name) {
98-
int64_t table_id = 0;
99-
KVStorage::TableOption option;
100-
MetaCodec::GetFsTableRange(option.start_key, option.end_key);
101-
auto status = kv_storage_->CreateTable(name, option, table_id);
102-
if (!status.ok()) {
103-
DINGO_LOG(ERROR) << fmt::format("create fs table fail, error: {}.", status.error_str());
104-
return false;
105-
}
106-
107-
DINGO_LOG(INFO) << fmt::format("create fs table success, start_key({}), end_key({}).",
108-
Helper::StringToHex(option.start_key), Helper::StringToHex(option.end_key));
109-
110-
return true;
111-
}
112-
113-
bool StoreClient::CreateFsQuotaTable(const std::string& name) {
114-
int64_t table_id = 0;
115-
KVStorage::TableOption option;
116-
MetaCodec::GetQuotaTableRange(option.start_key, option.end_key);
117-
auto status = kv_storage_->CreateTable(name, option, table_id);
118-
if (!status.ok()) {
119-
DINGO_LOG(ERROR) << fmt::format("create fs quota table fail, error: {}.", status.error_str());
120-
return false;
121-
}
122-
123-
DINGO_LOG(INFO) << fmt::format("create fs quota table success, start_key({}), end_key({}).",
59+
DINGO_LOG(INFO) << fmt::format("create meta table success, start_key({}), end_key({}).",
12460
Helper::StringToHex(option.start_key), Helper::StringToHex(option.end_key));
12561

12662
return true;
12763
}
12864

12965
bool StoreClient::CreateFsStatsTable(const std::string& name) {
13066
int64_t table_id = 0;
131-
KVStorage::TableOption option;
132-
MetaCodec::GetFsStatsTableRange(option.start_key, option.end_key);
67+
Range range = MetaCodec::GetFsStatsTableRange();
68+
KVStorage::TableOption option = {.start_key = range.start, .end_key = range.end};
13369
auto status = kv_storage_->CreateTable(name, option, table_id);
13470
if (!status.ok()) {
13571
DINGO_LOG(ERROR) << fmt::format("create fs stats table fail, error: {}.", status.error_str());
@@ -142,54 +78,6 @@ bool StoreClient::CreateFsStatsTable(const std::string& name) {
14278
return true;
14379
}
14480

145-
bool StoreClient::CreateFileSessionTable(const std::string& name) {
146-
int64_t table_id = 0;
147-
KVStorage::TableOption option;
148-
MetaCodec::GetFileSessionTableRange(option.start_key, option.end_key);
149-
auto status = kv_storage_->CreateTable(name, option, table_id);
150-
if (!status.ok()) {
151-
DINGO_LOG(ERROR) << fmt::format("create file session table fail, error: {}.", status.error_str());
152-
return false;
153-
}
154-
155-
DINGO_LOG(INFO) << fmt::format("create file session table success, start_key({}), end_key({}).",
156-
Helper::StringToHex(option.start_key), Helper::StringToHex(option.end_key));
157-
158-
return true;
159-
}
160-
161-
bool StoreClient::CreateDelSliceTable(const std::string& name) {
162-
int64_t table_id = 0;
163-
KVStorage::TableOption option;
164-
MetaCodec::GetDelSliceTableRange(option.start_key, option.end_key);
165-
auto status = kv_storage_->CreateTable(name, option, table_id);
166-
if (!status.ok()) {
167-
DINGO_LOG(ERROR) << fmt::format("create trash chunk table fail, error: {}.", status.error_str());
168-
return false;
169-
}
170-
171-
DINGO_LOG(INFO) << fmt::format("create trash chunk table success, start_key({}), end_key({}).",
172-
Helper::StringToHex(option.start_key), Helper::StringToHex(option.end_key));
173-
174-
return true;
175-
}
176-
177-
bool StoreClient::CreateDelFileTable(const std::string& name) {
178-
int64_t table_id = 0;
179-
KVStorage::TableOption option;
180-
MetaCodec::GetDelFileTableRange(option.start_key, option.end_key);
181-
auto status = kv_storage_->CreateTable(name, option, table_id);
182-
if (!status.ok()) {
183-
DINGO_LOG(ERROR) << fmt::format("create del file table fail, error: {}.", status.error_str());
184-
return false;
185-
}
186-
187-
DINGO_LOG(INFO) << fmt::format("create del file table success, start_key({}), end_key({}).",
188-
Helper::StringToHex(option.start_key), Helper::StringToHex(option.end_key));
189-
190-
return true;
191-
}
192-
19381
static std::string FormatTime(uint64_t time_ns) { return Helper::FormatMsTime(time_ns / 1000000, "%H:%M:%S"); }
19482

19583
static void TraversePrint(FsTreeNode* item, bool is_details, int level) {

‎src/mdsv2/client/store.h

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -30,15 +30,8 @@ class StoreClient {
3030

3131
bool Init(const std::string& coor_addr);
3232

33-
bool CreateLockTable(const std::string& name);
34-
bool CreateAutoIncrementTable(const std::string& name);
35-
bool CreateHeartbeatTable(const std::string& name);
36-
bool CreateFsTable(const std::string& name);
37-
bool CreateFsQuotaTable(const std::string& name);
33+
bool CreateMetaTable(const std::string& name);
3834
bool CreateFsStatsTable(const std::string& name);
39-
bool CreateFileSessionTable(const std::string& name);
40-
bool CreateDelSliceTable(const std::string& name);
41-
bool CreateDelFileTable(const std::string& name);
4235

4336
// print fs dentry tree
4437
void PrintDentryTree(uint32_t fs_id, bool is_details);

0 commit comments

Comments
 (0)