Skip to content

Commit 1d8bffd

Browse files
rock-gitchuandew
rock-git
authored and
chuandew
committed
[Refactor][mdsv2] Refactor scan meta code.
1 parent 424a7fd commit 1d8bffd

File tree

8 files changed

+534
-126
lines changed

8 files changed

+534
-126
lines changed

‎src/mdsv2/common/codec.cc

Lines changed: 260 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -442,7 +442,6 @@ Range MetaCodec::GetFsStatsRange(uint32_t fs_id) {
442442
}
443443

444444
// lock format: ${prefix} kTableMeta kMetaLock {name}
445-
446445
static const uint32_t kLockKeyHeaderSize = kPrefixSize + 2; // prefix + table id + meta type
447446

448447
bool MetaCodec::IsLockKey(const std::string& key) {
@@ -1103,5 +1102,265 @@ FsStatsDataEntry MetaCodec::DecodeFsStatsValue(const std::string& value) {
11031102
return std::move(stats);
11041103
}
11051104

1105+
bool MetaCodec::IsMetaTableKey(const std::string& key) {
1106+
if (key.size() <= kPrefixSize + 1) {
1107+
return false;
1108+
}
1109+
1110+
if (key.substr(0, kPrefixSize) != kPrefix || key.at(kPrefixSize) != kTableMeta) {
1111+
return false;
1112+
}
1113+
1114+
return true;
1115+
}
1116+
1117+
bool MetaCodec::IsFsStatsTableKey(const std::string& key) {
1118+
if (key.size() <= kPrefixSize + 1) {
1119+
return false;
1120+
}
1121+
1122+
if (key.substr(0, kPrefixSize) != kPrefix || key.at(kPrefixSize) != kTableFsStats) {
1123+
return false;
1124+
}
1125+
1126+
return true;
1127+
}
1128+
1129+
bool MetaCodec::IsFsMetaTableKey(const std::string& key) {
1130+
if (key.size() <= kPrefixSize + 1) {
1131+
return false;
1132+
}
1133+
1134+
if (key.substr(0, kPrefixSize) != kPrefix || key.at(kPrefixSize) != kTableFsMeta) {
1135+
return false;
1136+
}
1137+
1138+
return true;
1139+
}
1140+
1141+
std::pair<std::string, std::string> MetaCodec::ParseMetaTableKey(const std::string& key, const std::string& value) {
1142+
std::string key_desc, value_desc;
1143+
1144+
MetaType meta_type = static_cast<MetaType>(key.at(kPrefixSize + 1));
1145+
switch (meta_type) {
1146+
case kMetaLock: {
1147+
std::string name;
1148+
DecodeLockKey(key, name);
1149+
1150+
key_desc = fmt::format("{} kTableMeta kMetaLock {}", kPrefix, name);
1151+
1152+
int64_t mds_id;
1153+
uint64_t epoch;
1154+
uint64_t expire_time_ms;
1155+
DecodeLockValue(value, mds_id, epoch, expire_time_ms);
1156+
1157+
value_desc = fmt::format("{} {} {}", mds_id, epoch, expire_time_ms);
1158+
} break;
1159+
1160+
case kMetaAutoIncrement: {
1161+
std::string name;
1162+
DecodeAutoIncrementIDKey(key, name);
1163+
1164+
key_desc = fmt::format("{} kTableMeta kMetaAutoIncrement {}", kPrefix, name);
1165+
1166+
uint64_t next_id;
1167+
DecodeAutoIncrementIDValue(value, next_id);
1168+
1169+
value_desc = fmt::format("{} {}", next_id);
1170+
} break;
1171+
1172+
case kMetaHeartbeat: {
1173+
pb::mdsv2::Role role = static_cast<pb::mdsv2::Role>(key.at(kPrefixSize + 2));
1174+
if (role == pb::mdsv2::ROLE_MDS) {
1175+
int64_t mds_id;
1176+
DecodeHeartbeatKey(key, mds_id);
1177+
1178+
key_desc = fmt::format("{} kTableMeta kMetaHeartbeat kRoleMds {}", kPrefix, mds_id);
1179+
1180+
auto mds_info = DecodeHeartbeatMdsValue(value);
1181+
value_desc = mds_info.ShortDebugString();
1182+
1183+
} else if (role == pb::mdsv2::ROLE_CLIENT) {
1184+
std::string client_id;
1185+
DecodeHeartbeatKey(key, client_id);
1186+
1187+
key_desc = fmt::format("{} kTableMeta kMetaHeartbeat kRoleClient {}", kPrefix, client_id);
1188+
1189+
auto client_info = DecodeHeartbeatClientValue(value);
1190+
value_desc = client_info.ShortDebugString();
1191+
}
1192+
1193+
} break;
1194+
1195+
case kMetaFs: {
1196+
std::string name;
1197+
DecodeFsKey(key, name);
1198+
key_desc = fmt::format("{} kTableMeta kMetaFs {}", kPrefix, name);
1199+
1200+
auto fs_info = DecodeFsValue(value);
1201+
value_desc = fs_info.ShortDebugString();
1202+
1203+
} break;
1204+
1205+
case kMetaFsQuota: {
1206+
uint32_t fs_id;
1207+
DecodeFsQuotaKey(key, fs_id);
1208+
key_desc = fmt::format("{} kTableMeta kMetaFsQuota {}", kPrefix, fs_id);
1209+
1210+
auto quota = DecodeFsQuotaValue(value);
1211+
value_desc = quota.ShortDebugString();
1212+
1213+
} break;
1214+
1215+
default:
1216+
CHECK(false) << fmt::format("invalid meta type({}) key({}).", static_cast<int>(meta_type),
1217+
Helper::StringToHex(key));
1218+
}
1219+
1220+
return std::make_pair(std::move(key_desc), std::move(value_desc));
1221+
}
1222+
1223+
std::pair<std::string, std::string> MetaCodec::ParseFsStatsTableKey(const std::string& key, const std::string& value) {
1224+
CHECK(IsFsStatsKey(key)) << fmt::format("invalid fs stats key({}).", Helper::StringToHex(key));
1225+
1226+
std::string key_desc, value_desc;
1227+
1228+
uint32_t fs_id;
1229+
uint64_t time_ns;
1230+
DecodeFsStatsKey(key, fs_id, time_ns);
1231+
key_desc = fmt::format("{} kTableFsStats kMetaFsStats {} {}", kPrefix, fs_id, time_ns);
1232+
1233+
auto fs_stats = DecodeFsStatsValue(value);
1234+
value_desc = fs_stats.ShortDebugString();
1235+
1236+
return std::make_pair(std::move(key_desc), std::move(value_desc));
1237+
}
1238+
1239+
std::pair<std::string, std::string> MetaCodec::ParseFsMetaTableKey(const std::string& key, const std::string& value) {
1240+
CHECK(IsFsMetaTableKey(key)) << fmt::format("invalid fs meta key({}).", Helper::StringToHex(key));
1241+
1242+
std::string key_desc, value_desc;
1243+
1244+
MetaType meta_type = static_cast<MetaType>(key.at(kPrefixSize + 1 + 4));
1245+
switch (meta_type) {
1246+
case kMetaFsInode: {
1247+
FsInodeType inode_type = static_cast<FsInodeType>(key.at(kPrefixSize + 1 + 4 + 1 + 8));
1248+
switch (inode_type) {
1249+
case kFsInodeAttr: {
1250+
uint32_t fs_id;
1251+
Ino ino;
1252+
DecodeInodeKey(key, fs_id, ino);
1253+
1254+
key_desc = fmt::format("{} kTableFsMeta {} kMetaFsInode {} kFsInodeAttr", kPrefix, fs_id, ino);
1255+
1256+
auto attr = DecodeInodeValue(value);
1257+
value_desc = attr.ShortDebugString();
1258+
} break;
1259+
1260+
case kFsInodeDentry: {
1261+
uint32_t fs_id;
1262+
Ino ino;
1263+
std::string name;
1264+
DecodeDentryKey(key, fs_id, ino, name);
1265+
1266+
key_desc = fmt::format("{} kTableFsMeta {} kMetaFsInode {} kFsInodeDentry {}", kPrefix, fs_id, ino, name);
1267+
1268+
auto dentry = DecodeDentryValue(value);
1269+
value_desc = dentry.ShortDebugString();
1270+
} break;
1271+
1272+
case kFsInodeChunk: {
1273+
uint32_t fs_id;
1274+
Ino ino;
1275+
uint64_t chunk_index;
1276+
DecodeChunkKey(key, fs_id, ino, chunk_index);
1277+
1278+
key_desc =
1279+
fmt::format("{} kTableFsMeta {} kMetaFsInode {} kFsInodeChunk {}", kPrefix, fs_id, ino, chunk_index);
1280+
1281+
auto chunk = DecodeChunkValue(value);
1282+
value_desc = chunk.ShortDebugString();
1283+
} break;
1284+
1285+
default:
1286+
CHECK(false) << fmt::format("invalid inode type({}) key({}).", static_cast<int>(inode_type),
1287+
Helper::StringToHex(key));
1288+
}
1289+
} break;
1290+
1291+
case kMetaFsFileSession: {
1292+
uint32_t fs_id;
1293+
Ino ino;
1294+
std::string session_id;
1295+
DecodeFileSessionKey(key, fs_id, ino, session_id);
1296+
1297+
key_desc = fmt::format("{} kTableFsMeta {} kMetaFsFileSession {} {}", kPrefix, fs_id, ino, session_id);
1298+
1299+
auto file_session = DecodeFileSessionValue(value);
1300+
value_desc = file_session.ShortDebugString();
1301+
} break;
1302+
1303+
case kMetaFsDirQuota: {
1304+
uint32_t fs_id;
1305+
Ino ino;
1306+
DecodeDirQuotaKey(key, fs_id, ino);
1307+
1308+
key_desc = fmt::format("{} kTableFsMeta {} kMetaFsDirQuota {}", kPrefix, fs_id, ino);
1309+
1310+
auto dir_quota = DecodeDirQuotaValue(value);
1311+
value_desc = dir_quota.ShortDebugString();
1312+
1313+
} break;
1314+
case kMetaFsDelSlice: {
1315+
uint32_t fs_id;
1316+
Ino ino;
1317+
uint64_t chunk_index;
1318+
uint64_t time_ns;
1319+
DecodeDelSliceKey(key, fs_id, ino, chunk_index, time_ns);
1320+
1321+
key_desc = fmt::format("{} kTableFsMeta {} kMetaFsDelSlice {} {} {}", kPrefix, fs_id, ino, chunk_index, time_ns);
1322+
1323+
auto del_slice = DecodeDelSliceValue(value);
1324+
value_desc = del_slice.ShortDebugString();
1325+
} break;
1326+
1327+
case kMetaFsDelFile: {
1328+
uint32_t fs_id;
1329+
Ino ino;
1330+
DecodeDelFileKey(key, fs_id, ino);
1331+
1332+
key_desc = fmt::format("{} kTableFsMeta {} kMetaFsDelFile {} {}", kPrefix, fs_id, ino);
1333+
1334+
auto del_file = DecodeDelFileValue(value);
1335+
value_desc = del_file.ShortDebugString();
1336+
} break;
1337+
1338+
default:
1339+
CHECK(false) << fmt::format("invalid meta type({}) key({}).", static_cast<int>(meta_type),
1340+
Helper::StringToHex(key));
1341+
}
1342+
1343+
return std::make_pair(std::move(key_desc), std::move(value_desc));
1344+
}
1345+
1346+
std::pair<std::string, std::string> MetaCodec::ParseKey(const std::string& key, const std::string& value) {
1347+
TableID table_id = static_cast<TableID>(key.at(kPrefixSize));
1348+
switch (table_id) {
1349+
case kTableMeta:
1350+
return ParseMetaTableKey(key, value);
1351+
case kTableFsStats:
1352+
1353+
return ParseFsStatsTableKey(key, value);
1354+
case kTableFsMeta:
1355+
return ParseFsMetaTableKey(key, value);
1356+
1357+
default:
1358+
CHECK(false) << fmt::format("invalid table id({}) key({}).", static_cast<int>(table_id),
1359+
Helper::StringToHex(key));
1360+
}
1361+
1362+
return {};
1363+
}
1364+
11061365
} // namespace mdsv2
11071366
} // namespace dingofs

‎src/mdsv2/common/codec.h

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
#include <cstdint>
2121
#include <string>
22+
#include <utility>
2223

2324
#include "mdsv2/common/type.h"
2425

@@ -143,7 +144,7 @@ class MetaCodec {
143144
static std::string EncodeFileSessionValue(const FileSessionEntry& file_session);
144145
static FileSessionEntry DecodeFileSessionValue(const std::string& value);
145146

146-
// dir quota format: ${prefix} kTableFsMeta {fs_id} kMetaDirQuota {ino}
147+
// dir quota format: ${prefix} kTableFsMeta {fs_id} kMetaFsDirQuota {ino}
147148
static bool IsDirQuotaKey(const std::string& key);
148149
static std::string EncodeDirQuotaKey(uint32_t fs_id, Ino ino);
149150
static void DecodeDirQuotaKey(const std::string& key, uint32_t& fs_id, Ino& ino);
@@ -171,6 +172,16 @@ class MetaCodec {
171172
static void DecodeFsStatsKey(const std::string& key, uint32_t& fs_id, uint64_t& time_ns);
172173
static std::string EncodeFsStatsValue(const FsStatsDataEntry& stats);
173174
static FsStatsDataEntry DecodeFsStatsValue(const std::string& value);
175+
176+
// check key belongs to a specific table
177+
static bool IsMetaTableKey(const std::string& key);
178+
static bool IsFsStatsTableKey(const std::string& key);
179+
static bool IsFsMetaTableKey(const std::string& key);
180+
181+
static std::pair<std::string, std::string> ParseMetaTableKey(const std::string& key, const std::string& value);
182+
static std::pair<std::string, std::string> ParseFsStatsTableKey(const std::string& key, const std::string& value);
183+
static std::pair<std::string, std::string> ParseFsMetaTableKey(const std::string& key, const std::string& value);
184+
static std::pair<std::string, std::string> ParseKey(const std::string& key, const std::string& value);
174185
};
175186

176187
} // namespace mdsv2

‎src/mdsv2/filesystem/filesystem.cc

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -364,14 +364,21 @@ InodeSPtr FileSystem::GetInodeFromCache(Ino ino) { return inode_cache_.GetInode(
364364
std::map<uint64_t, InodeSPtr> FileSystem::GetAllInodesFromCache() { return inode_cache_.GetAllInodes(); }
365365

366366
Status FileSystem::GetInodeFromStore(Ino ino, const std::string& reason, bool is_cache, InodeSPtr& out_inode) {
367-
std::string key = MetaCodec::EncodeInodeKey(fs_id_, ino);
368-
std::string value;
369-
auto status = kv_storage_->Get(key, value);
367+
Trace trace;
368+
GetInodeAttrOperation operation(trace, fs_id_, ino);
369+
370+
auto status = RunOperation(&operation);
370371
if (!status.ok()) {
371-
return Status(pb::error::ENOT_FOUND, fmt::format("not found inode({}), {}", ino, status.error_str()));
372+
if (status.error_code() != pb::error::ENOT_FOUND) {
373+
DINGO_LOG(ERROR) << fmt::format("[fs.{}] fetch inode({}) from store fail, reason({}), status({}).", fs_id_, ino,
374+
reason, status.error_str());
375+
}
376+
return status;
372377
}
373378

374-
out_inode = Inode::New(MetaCodec::DecodeInodeValue(value));
379+
auto& result = operation.GetResult();
380+
381+
out_inode = Inode::New(result.attr);
375382

376383
if (is_cache) inode_cache_.PutInode(ino, out_inode);
377384

0 commit comments

Comments
 (0)