diff --git a/src/blockaccess/block_accesser.cpp b/src/blockaccess/block_accesser.cpp index e8319b893..1a9195a15 100644 --- a/src/blockaccess/block_accesser.cpp +++ b/src/blockaccess/block_accesser.cpp @@ -36,6 +36,7 @@ static bvar::Adder block_get_async_num("block_get_async_num"); static bvar::Adder block_get_sync_num("block_get_sync_num"); using dingofs::utils::kMB; +using stub::metric::ObjectMetric; Status BlockAccesserImpl::Init() { if (options_.type == AccesserType::kS3) { @@ -106,6 +107,11 @@ Status BlockAccesserImpl::Put(const std::string& key, const char* buffer, (s.ok() ? "ok" : "fail")); }); + // object storage write metrics + BlockAccessMetricGuard metric_guard(&s, + &ObjectMetric::GetInstance().write_object, + length, butil::cpuwide_time_us()); + block_put_sync_num << 1; auto dec = ::absl::MakeCleanup([&]() { block_put_sync_num << -1; }); @@ -131,6 +137,11 @@ void BlockAccesserImpl::AsyncPut( return absl::StrFormat("async_put_block (%s, %d) : %d", ctx->key, ctx->buffer_size, ctx->ret_code); }); + // object storage write metrics + Status s = (ctx->ret_code == 0 ? Status::OK() : Status::Unknown("")); + BlockAccessMetricGuard metric_guard( + &s, &ObjectMetric::GetInstance().write_object, ctx->buffer_size, + start_us); block_put_async_num << -1; inflight_bytes_throttle_->OnComplete(ctx->buffer_size); @@ -173,6 +184,11 @@ Status BlockAccesserImpl::Get(const std::string& key, std::string* data) { (s.ok() ? "ok" : "fail")); }); + // object storage read metrics + BlockAccessMetricGuard metric_guard(&s, + &ObjectMetric::GetInstance().read_object, + data->length(), butil::cpuwide_time_us()); + block_get_sync_num << 1; auto dec = ::absl::MakeCleanup([&]() { block_get_sync_num << -1; }); @@ -198,6 +214,11 @@ void BlockAccesserImpl::AsyncGet( ctx->offset, ctx->len, ctx->ret_code); }); + // object storage read metrics + Status s = (ctx->ret_code == 0 ? Status::OK() : Status::Unknown("")); + BlockAccessMetricGuard metric_guard( + &s, &ObjectMetric::GetInstance().read_object, ctx->len, start_us); + block_get_async_num << -1; inflight_bytes_throttle_->OnComplete(ctx->len); @@ -223,6 +244,11 @@ Status BlockAccesserImpl::Range(const std::string& key, off_t offset, (s.ok() ? "ok" : "fail")); }); + // object storage read metrics + BlockAccessMetricGuard metric_guard(&s, + &ObjectMetric::GetInstance().read_object, + length, butil::cpuwide_time_us()); + block_get_sync_num << 1; auto dec = ::absl::MakeCleanup([&]() { block_get_sync_num << -1; }); diff --git a/src/blockaccess/block_accesser.h b/src/blockaccess/block_accesser.h index 68c6fcd64..81f0e0b05 100644 --- a/src/blockaccess/block_accesser.h +++ b/src/blockaccess/block_accesser.h @@ -27,6 +27,7 @@ #include "blockaccess/accesser.h" #include "blockaccess/accesser_common.h" #include "common/status.h" +#include "stub/metric/metric.h" #include "utils/throttle.h" namespace dingofs { @@ -38,7 +39,7 @@ enum class RetryStrategy : uint8_t { }; using RetryCallback = std::function; - +using ::dingofs::stub::metric::InterfaceMetric; // BlockAccesser is a class that provides a way to access block from a data // source. It is a base class for all data access classes. class BlockAccesser { @@ -153,6 +154,28 @@ class BlockAccesserImpl : public BlockAccesser { std::unique_ptr inflight_bytes_throttle_; }; +struct BlockAccessMetricGuard { + explicit BlockAccessMetricGuard(Status* status, InterfaceMetric* metric, + size_t count, uint64_t start) + : status_(status), metric_(metric), count_(count), start_(start) {} + ~BlockAccessMetricGuard() { + if (status_->ok()) { + metric_->bps.count << count_; + metric_->qps.count << 1; + auto duration = butil::cpuwide_time_us() - start_; + metric_->latency << duration; + metric_->latTotal << duration; + } else { + metric_->eps.count << 1; + } + } + + Status* status_; + InterfaceMetric* metric_; + size_t count_; + uint64_t start_; +}; + using BlockAccesserSPtr = std::shared_ptr; using BlockAccesserUPtr = std::unique_ptr; diff --git a/src/blockaccess/s3/s3_accesser.cpp b/src/blockaccess/s3/s3_accesser.cpp index 89f5c30bd..b9b5e21a1 100644 --- a/src/blockaccess/s3/s3_accesser.cpp +++ b/src/blockaccess/s3/s3_accesser.cpp @@ -26,9 +26,6 @@ namespace dingofs { namespace blockaccess { -using stub::metric::MetricGuard; -using stub::metric::S3Metric; - bool S3Accesser::Init() { const auto& s3_info = options_.s3_info; LOG(INFO) << fmt::format( @@ -57,12 +54,7 @@ bool S3Accesser::ContainerExist() { return client_->BucketExist(); } Status S3Accesser::Put(const std::string& key, const char* buffer, size_t length) { - int rc = 0; - // write s3 metrics - auto start = butil::cpuwide_time_us(); - MetricGuard guard(&rc, &S3Metric::GetInstance().write_s3, length, start); - - rc = client_->PutObject(S3Key(key), buffer, length); + int rc = client_->PutObject(S3Key(key), buffer, length); if (rc < 0) { LOG(ERROR) << fmt::format("[accesser] put object({}) fail, retcode: {}.", key, rc); @@ -77,8 +69,6 @@ void S3Accesser::AsyncPut(std::shared_ptr context) { auto start_time = butil::cpuwide_time_us(); context->cb = [&, start_time, origin_cb](const std::shared_ptr& ctx) { - MetricGuard guard(&ctx->ret_code, &S3Metric::GetInstance().write_s3, - ctx->buffer_size, start_time); ctx->cb = origin_cb; ctx->cb(ctx); }; @@ -86,12 +76,7 @@ void S3Accesser::AsyncPut(std::shared_ptr context) { } Status S3Accesser::Get(const std::string& key, std::string* data) { - int rc; // read s3 metrics - auto start = butil::cpuwide_time_us(); - MetricGuard guard(&rc, &S3Metric::GetInstance().read_s3, data->length(), - start); - - rc = client_->GetObject(S3Key(key), data); + int rc = client_->GetObject(S3Key(key), data); if (rc < 0) { if (!client_->ObjectExist(S3Key(key))) { // TODO: more efficient LOG(WARNING) << fmt::format("[accesser] object({}) not found.", key); @@ -108,11 +93,7 @@ Status S3Accesser::Get(const std::string& key, std::string* data) { Status S3Accesser::Range(const std::string& key, off_t offset, size_t length, char* buffer) { - int rc; // read s3 metrics - auto start = butil::cpuwide_time_us(); - MetricGuard guard(&rc, &S3Metric::GetInstance().read_s3, length, start); - - rc = client_->RangeObject(S3Key(key), buffer, offset, length); + int rc = client_->RangeObject(S3Key(key), buffer, offset, length); if (rc < 0) { if (!client_->ObjectExist(S3Key(key))) { // TODO: more efficient LOG(WARNING) << fmt::format("[accesser] object({}) not found.", key); @@ -132,8 +113,6 @@ void S3Accesser::AsyncGet(std::shared_ptr context) { auto start_time = butil::cpuwide_time_us(); context->cb = [&, start_time, origin_cb](const std::shared_ptr& ctx) { - MetricGuard guard(&ctx->ret_code, &S3Metric::GetInstance().read_s3, - ctx->len, start_time); ctx->cb = origin_cb; ctx->cb(ctx); }; diff --git a/src/client/vfs/CMakeLists.txt b/src/client/vfs/CMakeLists.txt index b7ddc1e3f..90e7e84e5 100644 --- a/src/client/vfs/CMakeLists.txt +++ b/src/client/vfs/CMakeLists.txt @@ -17,6 +17,7 @@ add_subdirectory(data) add_subdirectory(handle) add_subdirectory(hub) add_subdirectory(meta) +add_subdirectory(statistics) add_library(vfs_lib vfs_impl.cpp @@ -30,6 +31,7 @@ target_link_libraries(vfs_lib vfs_data vfs_hub vfs_handle + vfs_statistics glog::glog gflags::gflags fmt::fmt diff --git a/src/client/vfs/meta/v2/filesystem.cpp b/src/client/vfs/meta/v2/filesystem.cpp index 3ecedc61a..4dee271a2 100644 --- a/src/client/vfs/meta/v2/filesystem.cpp +++ b/src/client/vfs/meta/v2/filesystem.cpp @@ -17,11 +17,14 @@ #include #include +#include #include #include #include +#include "client/common/dynamic_config.h" #include "client/vfs/meta/v2/client_id.h" +#include "client/vfs/statistics/fs_stats_manager.h" #include "client/vfs/vfs_meta.h" #include "common/status.h" #include "dingofs/error.pb.h" @@ -36,6 +39,10 @@ namespace client { namespace vfs { namespace v2 { +USING_FLAG(push_metric_interval_millsecond) + +using FsStatsManager = dingofs::client::vfs::FsStatsManager; + const uint32_t kMaxHostNameLength = 255; const uint32_t kMaxXAttrNameLength = 255; @@ -133,12 +140,14 @@ std::string FileSessionMap::Get(uint64_t fh) { MDSV2FileSystem::MDSV2FileSystem(mdsv2::FsInfoPtr fs_info, const ClientId& client_id, MDSDiscoveryPtr mds_discovery, - MDSClientPtr mds_client) + MDSClientPtr mds_client, + FsStatsManagerUPtr fs_stats_manager) : name_(fs_info->GetName()), client_id_(client_id), fs_info_(fs_info), mds_discovery_(mds_discovery), - mds_client_(mds_client) {} + mds_client_(mds_client), + fs_stats_manager_(std::move(fs_stats_manager)) {} MDSV2FileSystem::~MDSV2FileSystem() {} // NOLINT @@ -271,6 +280,8 @@ void MDSV2FileSystem::Heartbeat() { } } +void MDSV2FileSystem::PushFsStatsToMDS() { fs_stats_manager_->PushFsStats(); } + bool MDSV2FileSystem::InitCrontab() { // Add heartbeat crontab crontab_configs_.push_back({ @@ -280,6 +291,14 @@ bool MDSV2FileSystem::InitCrontab() { [this](void*) { this->Heartbeat(); }, }); + // Add push fs stats crontab + crontab_configs_.push_back({ + "PUSH_FSSTATS", + FLAGS_push_metric_interval_millsecond, + true, + [this](void*) { this->PushFsStatsToMDS(); }, + }); + crontab_manager_.AddCrontab(crontab_configs_); return true; @@ -646,8 +665,12 @@ MDSV2FileSystemUPtr MDSV2FileSystem::Build(const std::string& fs_name, return nullptr; } + // create fs stats manager + auto fs_stats_manager = FsStatsManager::New(fs_name, mds_client); + // create filesystem - return MDSV2FileSystem::New(fs_info, client_id, mds_discovery, mds_client); + return MDSV2FileSystem::New(fs_info, client_id, mds_discovery, mds_client, + std::move(fs_stats_manager)); } } // namespace v2 diff --git a/src/client/vfs/meta/v2/filesystem.h b/src/client/vfs/meta/v2/filesystem.h index afdce453c..f361d4714 100644 --- a/src/client/vfs/meta/v2/filesystem.h +++ b/src/client/vfs/meta/v2/filesystem.h @@ -24,6 +24,7 @@ #include "client/vfs/meta/v2/client_id.h" #include "client/vfs/meta/v2/mds_client.h" #include "client/vfs/meta/v2/mds_discovery.h" +#include "client/vfs/statistics/fs_stats_manager.h" #include "client/vfs/vfs_meta.h" #include "common/status.h" #include "dingofs/mdsv2.pb.h" @@ -77,15 +78,18 @@ class FileSessionMap { class MDSV2FileSystem : public vfs::MetaSystem { public: MDSV2FileSystem(mdsv2::FsInfoPtr fs_info, const ClientId& client_id, - MDSDiscoveryPtr mds_discovery, MDSClientPtr mds_client); + MDSDiscoveryPtr mds_discovery, MDSClientPtr mds_client, + FsStatsManagerUPtr fs_stats_manager); ~MDSV2FileSystem() override; static MDSV2FileSystemUPtr New(mdsv2::FsInfoPtr fs_info, const ClientId& client_id, MDSDiscoveryPtr mds_discovery, - MDSClientPtr mds_client) { + MDSClientPtr mds_client, + FsStatsManagerUPtr fs_stats_manager) { return std::make_unique(fs_info, client_id, mds_discovery, - mds_client); + mds_client, + std::move(fs_stats_manager)); } static MDSV2FileSystemUPtr Build(const std::string& fs_name, @@ -155,6 +159,8 @@ class MDSV2FileSystem : public vfs::MetaSystem { void Heartbeat(); + void PushFsStatsToMDS(); + bool InitCrontab(); const std::string name_; @@ -172,6 +178,8 @@ class MDSV2FileSystem : public vfs::MetaSystem { std::vector crontab_configs_; // This is manage crontab, like heartbeat. mdsv2::CrontabManager crontab_manager_; + // Fs stats manager + FsStatsManagerUPtr fs_stats_manager_; }; } // namespace v2 diff --git a/src/client/vfs/meta/v2/mds_client.cpp b/src/client/vfs/meta/v2/mds_client.cpp index d115cdcff..2e1241e28 100644 --- a/src/client/vfs/meta/v2/mds_client.cpp +++ b/src/client/vfs/meta/v2/mds_client.cpp @@ -875,6 +875,25 @@ Status MDSClient::GetFsQuota(FsStat& fs_stat) { return Status::OK(); } +Status MDSClient::PushFsStatsToMDS(const std::string& fs_name, + const pb::mdsv2::FsStatsData& fs_stat_data) { + CHECK(fs_name != "") << "fs_name is invalid."; + + pb::mdsv2::SetFsStatsRequest request; + pb::mdsv2::SetFsStatsResponse response; + + request.set_fs_name(fs_name); + request.mutable_stats()->CopyFrom(fs_stat_data); + + auto status = + rpc_->SendRequest("MDSService", "SetFsStats", request, response); + if (!status.ok()) { + return status; + } + + return Status::OK(); +} + bool MDSClient::UpdateRouter() { pb::mdsv2::FsInfo new_fs_info; auto status = MDSClient::GetFsInfo(rpc_, fs_info_->GetName(), new_fs_info); diff --git a/src/client/vfs/meta/v2/mds_client.h b/src/client/vfs/meta/v2/mds_client.h index 7db7403dd..360c4ef77 100644 --- a/src/client/vfs/meta/v2/mds_client.h +++ b/src/client/vfs/meta/v2/mds_client.h @@ -104,6 +104,9 @@ class MDSClient { Status GetFsQuota(FsStat& fs_stat); + Status PushFsStatsToMDS(const std::string& fs_name, + const pb::mdsv2::FsStatsData& fs_stat_data); + private: EndPoint GetEndpoint(Ino ino); EndPoint GetEndpointByParent(int64_t parent); diff --git a/src/client/vfs/statistics/CMakeLists.txt b/src/client/vfs/statistics/CMakeLists.txt new file mode 100644 index 000000000..e5dfa042f --- /dev/null +++ b/src/client/vfs/statistics/CMakeLists.txt @@ -0,0 +1,23 @@ +# Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_library(vfs_statistics + fs_stats_manager.cpp +) + +target_link_libraries(vfs_statistics + dingofs_common + stub_metric + PROTO_OBJS +) \ No newline at end of file diff --git a/src/client/vfs/statistics/fs_stats_manager.cpp b/src/client/vfs/statistics/fs_stats_manager.cpp new file mode 100644 index 000000000..3ef05a961 --- /dev/null +++ b/src/client/vfs/statistics/fs_stats_manager.cpp @@ -0,0 +1,71 @@ +// Copyright (c) 2024 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "client/vfs/statistics/fs_stats_manager.h" + +#include "common/status.h" +#include "glog/logging.h" +#include "stub/metric/metric.h" + +namespace dingofs { +namespace client { +namespace vfs { + +using ::dingofs::pb::mdsv2::FsStatsData; +using ::dingofs::stub::metric::FSMetric; +using ::dingofs::stub::metric::ObjectMetric; + +void FsStatsManager::PushFsStats() { + FsStatsData current_fs_stats = GetCurrentFsStatsData(); + FsStatsData delta_fs_stats = current_fs_stats - last_fs_stats_; + if (IsEmpty(delta_fs_stats)) return; + + Status s = mds_client_->PushFsStatsToMDS(fsname_, delta_fs_stats); + if (BAIDU_LIKELY(s.ok())) { + VLOG(12) << "PushFsStats success, fs_name: " << fsname_ + << ",delta fsstats data:[" << FsStatsDataToStr(delta_fs_stats) + << "]"; + last_fs_stats_ = current_fs_stats; + + } else { + LOG(ERROR) << "PushFsStats failed, fs_name: " << fsname_ + << ", status: " << s.ToString() << ", delta metrics data:[" + << FsStatsDataToStr(delta_fs_stats) << "]"; + } +} + +FsStatsData FsStatsManager::GetCurrentFsStatsData() { + FsStatsData fs_stats_data; + + auto& fs = FSMetric::GetInstance(); + auto& object = ObjectMetric::GetInstance(); + + // filesystem metrics + fs_stats_data.set_read_bytes(fs.user_read.bps.count.get_value()); + fs_stats_data.set_read_qps(fs.user_read.qps.count.get_value()); + fs_stats_data.set_write_bytes(fs.user_write.bps.count.get_value()); + fs_stats_data.set_write_qps(fs.user_write.qps.count.get_value()); + + // s3 or rados metrics + fs_stats_data.set_s3_read_bytes(object.read_object.bps.count.get_value()); + fs_stats_data.set_s3_read_qps(object.read_object.qps.count.get_value()); + fs_stats_data.set_s3_write_bytes(object.write_object.bps.count.get_value()); + fs_stats_data.set_s3_write_qps(object.write_object.qps.count.get_value()); + + return fs_stats_data; +} + +} // namespace vfs +} // namespace client +} // namespace dingofs \ No newline at end of file diff --git a/src/client/vfs/statistics/fs_stats_manager.h b/src/client/vfs/statistics/fs_stats_manager.h new file mode 100644 index 000000000..1702470d4 --- /dev/null +++ b/src/client/vfs/statistics/fs_stats_manager.h @@ -0,0 +1,104 @@ +// Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DINGOFS_CLIENT_VFS_STATISTICS_FS_STATS_MANAGER_H_ +#define DINGOFS_CLIENT_VFS_STATISTICS_FS_STATS_MANAGER_H_ + +#include +#include +#include + +#include "client/vfs/meta/v2/mds_client.h" +#include "dingofs/mdsv2.pb.h" + +namespace dingofs { +namespace client { +namespace vfs { + +using ::dingofs::client::vfs::v2::MDSClient; +using ::dingofs::pb::mdsv2::FsStatsData; +class FsStatsManager; +using FsStatsManagerUPtr = std::unique_ptr; + +// TODO: Proto(mdsv2) FsStatsData member name s3xxxx change to objectxxxx + +inline std::string FsStatsDataToStr(const FsStatsData& fs_stat_data) { + std::ostringstream oss; + oss << "read_bytes: " << fs_stat_data.read_bytes() << ", " + << "write_bytes: " << fs_stat_data.write_bytes() << ", " + << "read_qps: " << fs_stat_data.read_qps() << ", " + << "write_qps: " << fs_stat_data.write_qps() << ", " + << "object_read_bytes: " << fs_stat_data.s3_read_bytes() << ", " + << "object_write_bytes: " << fs_stat_data.s3_write_bytes() << ", " + << "object_read_qps: " << fs_stat_data.s3_read_qps() << ", " + << "object_write_qps: " << fs_stat_data.s3_write_qps(); + + return oss.str(); +} + +inline bool IsEmpty(const FsStatsData& fs_stat_data) { + return fs_stat_data.read_bytes() == 0 && fs_stat_data.write_bytes() == 0 && + fs_stat_data.read_qps() == 0 && fs_stat_data.write_qps() == 0 && + fs_stat_data.s3_read_bytes() == 0 && + fs_stat_data.s3_write_bytes() == 0 && + fs_stat_data.s3_read_qps() == 0 && fs_stat_data.s3_write_qps() == 0; +} + +inline FsStatsData operator-(const FsStatsData& current, + const FsStatsData& last) { + FsStatsData delta; + + // filesystem delta statistics + delta.set_read_bytes(current.read_bytes() - last.read_bytes()); + delta.set_read_qps(current.read_qps() - last.read_qps()); + delta.set_write_bytes(current.write_bytes() - last.write_bytes()); + delta.set_write_qps(current.write_qps() - last.write_qps()); + + // s3、rados delta statistics + delta.set_s3_read_bytes(current.s3_read_bytes() - last.s3_read_bytes()); + delta.set_s3_read_qps(current.s3_read_qps() - last.s3_read_qps()); + delta.set_s3_write_bytes(current.s3_write_bytes() - last.s3_write_bytes()); + delta.set_s3_write_qps(current.s3_write_qps() - last.s3_write_qps()); + + return delta; +} + +class FsStatsManager { + public: + FsStatsManager(const std::string& fs_name, + std::shared_ptr mds_client) + : fsname_(fs_name), mds_client_(mds_client) {} + + virtual ~FsStatsManager() = default; + + void PushFsStats(); + + static FsStatsManagerUPtr New(const std::string& fs_name, + std::shared_ptr mds_client) { + return std::make_unique(fs_name, mds_client); + } + + private: + FsStatsData GetCurrentFsStatsData(); + + const std::string fsname_; + std::shared_ptr mds_client_; + FsStatsData last_fs_stats_; // store last fs stats +}; + +} // namespace vfs +} // namespace client +} // namespace dingofs + +#endif // DINGOFS_CLIENT_VFS_STATISTICS_FS_STATS_MANAGER_H_ \ No newline at end of file diff --git a/src/client/vfs_old/filesystem/fs_push_metric_manager.cpp b/src/client/vfs_old/filesystem/fs_push_metric_manager.cpp index 4cd377f69..4daad5e1f 100644 --- a/src/client/vfs_old/filesystem/fs_push_metric_manager.cpp +++ b/src/client/vfs_old/filesystem/fs_push_metric_manager.cpp @@ -28,7 +28,7 @@ namespace filesystem { using ::dingofs::pb::mds::FsStatsData; using ::dingofs::pb::mds::FSStatusCode; using ::dingofs::stub::metric::FSMetric; -using ::dingofs::stub::metric::S3Metric; +using ::dingofs::stub::metric::ObjectMetric; USING_FLAG(push_metric_interval_millsecond) @@ -101,13 +101,13 @@ FsStatsData FsPushMetricManager::GetClientMetrics() { FSMetric::GetInstance().user_write.qps.count.get_value()); // s3 write metrics client_metrics.set_s3readbytes( - S3Metric::GetInstance().read_s3.bps.count.get_value()); + ObjectMetric::GetInstance().read_object.bps.count.get_value()); client_metrics.set_s3readqps( - S3Metric::GetInstance().read_s3.qps.count.get_value()); + ObjectMetric::GetInstance().read_object.qps.count.get_value()); client_metrics.set_s3writebytes( - S3Metric::GetInstance().write_s3.bps.count.get_value()); + ObjectMetric::GetInstance().write_object.bps.count.get_value()); client_metrics.set_s3writeqps( - S3Metric::GetInstance().write_s3.qps.count.get_value()); + ObjectMetric::GetInstance().write_object.qps.count.get_value()); return client_metrics; } diff --git a/src/client/vfs_old/vfs_old.cpp b/src/client/vfs_old/vfs_old.cpp index e7a36a0e5..5ed12332a 100644 --- a/src/client/vfs_old/vfs_old.cpp +++ b/src/client/vfs_old/vfs_old.cpp @@ -1399,18 +1399,13 @@ Status VFSOld::Read(Ino ino, char* buf, uint64_t size, uint64_t offset, return Status::OK(); } - // fuse read metrics uint64_t r_size = 0; - FsMetricGuard guard(&stub::metric::FSMetric::GetInstance().user_read, - &r_size); - std::shared_ptr inode_wrapper; DINGOFS_ERROR ret = inode_cache_manager_->GetInode(ino, inode_wrapper); if (ret != DINGOFS_ERROR::OK) { LOG(ERROR) << "Fail get inode fail in read, rc: " << ret << ", inodeId=" << ino; - guard.Fail(); return filesystem::DingofsErrorToStatus(ret); } @@ -1425,7 +1420,6 @@ Status VFSOld::Read(Ino ino, char* buf, uint64_t size, uint64_t offset, int r_ret = s3_adapter_->Read(ino, offset, len, buf); if (r_ret < 0) { LOG(ERROR) << "Fail read for inodeId=" << ino << ", rc: " << r_ret; - guard.Fail(); return Status::Internal("read s3 fail"); } r_size = r_ret; @@ -1455,15 +1449,11 @@ Status VFSOld::Write(Ino ino, const char* buf, uint64_t size, uint64_t offset, return Status::NoSpace("check quota fail"); } - // fuse write metrics uint64_t w_size = 0; - FsMetricGuard guard(&stub::metric::FSMetric::GetInstance().user_write, - &w_size); int w_ret = s3_adapter_->Write(ino, offset, size, buf); if (w_ret < 0) { LOG(ERROR) << "Fail write for inodeId=" << ino << ", rc: " << w_ret; - guard.Fail(); return Status::Internal("write s3 fail"); } @@ -1473,7 +1463,6 @@ Status VFSOld::Write(Ino ino, const char* buf, uint64_t size, uint64_t offset, DINGOFS_ERROR ret = inode_cache_manager_->GetInode(ino, inode_wrapper); if (ret != DINGOFS_ERROR::OK) { LOG(ERROR) << "Fail get inode fail, rc: " << ret << ", inodeId=" << ino; - guard.Fail(); return filesystem::DingofsErrorToStatus(ret); } diff --git a/src/client/vfs_old/vfs_old.h b/src/client/vfs_old/vfs_old.h index cdaeb96db..75ea62d86 100644 --- a/src/client/vfs_old/vfs_old.h +++ b/src/client/vfs_old/vfs_old.h @@ -203,31 +203,6 @@ class VFSOld : public VFS { dingofs::utils::Mutex rename_mutex_; }; - -struct FsMetricGuard { - explicit FsMetricGuard(stub::metric::InterfaceMetric* metric, size_t* count) - : metric(metric), count(count), start(butil::cpuwide_time_us()) {} - - ~FsMetricGuard() { - if (!fail) { - metric->bps.count << *count; - metric->qps.count << 1; - auto duration = butil::cpuwide_time_us() - start; - metric->latency << duration; - metric->latTotal << duration; - } else { - metric->eps.count << 1; - } - } - - void Fail() { fail = true; } - - bool fail{false}; - stub::metric::InterfaceMetric* metric; - size_t* count; - uint64_t start; -}; - } // namespace vfs } // namespace client } // namespace dingofs diff --git a/src/client/vfs_old/warmup/warmup_manager.cpp b/src/client/vfs_old/warmup/warmup_manager.cpp index ab8e67e78..40060ba50 100644 --- a/src/client/vfs_old/warmup/warmup_manager.cpp +++ b/src/client/vfs_old/warmup/warmup_manager.cpp @@ -49,7 +49,7 @@ using base::filepath::PathSplit; using common::ClientOption; using common::WarmupStorageType; using stub::metric::MetricGuard; -using stub::metric::S3Metric; +using stub::metric::ObjectMetric; using utils::ReadLockGuard; using utils::WriteLockGuard; @@ -499,7 +499,8 @@ void WarmupManagerS3Impl::WarmUpAllObjs( blockaccess::GetObjectAsyncCallBack cb = [&](const std::shared_ptr& context) { // metrics for async get data from s3 - MetricGuard guard(&context->ret_code, &S3Metric::GetInstance().read_s3, + MetricGuard guard(&context->ret_code, + &ObjectMetric::GetInstance().read_object, context->len, start); if (bgFetchStop_.load(std::memory_order_acquire)) { VLOG(9) << "need stop warmup"; diff --git a/src/client/vfs_wrapper/vfs_wrapper.cpp b/src/client/vfs_wrapper/vfs_wrapper.cpp index 114ef0f2d..6550a8d47 100644 --- a/src/client/vfs_wrapper/vfs_wrapper.cpp +++ b/src/client/vfs_wrapper/vfs_wrapper.cpp @@ -426,6 +426,9 @@ Status VFSWrapper::Read(Ino ino, char* buf, uint64_t size, uint64_t offset, ClientOpMetricGuard op_metric( {&client_op_metric_->opRead, &client_op_metric_->opAll}); + FsMetricGuard fs_metric(&s, &stub::metric::FSMetric::GetInstance().user_read, + out_rsize, butil::cpuwide_time_us()); + s = vfs_->Read(ino, buf, size, offset, fh, out_rsize); VLOG(1) << "VFSRead end ino: " << ino << " parma_size: " << size << " offset: " << offset << " fh: " << fh @@ -449,6 +452,9 @@ Status VFSWrapper::Write(Ino ino, const char* buf, uint64_t size, ClientOpMetricGuard op_metric( {&client_op_metric_->opWrite, &client_op_metric_->opAll}); + FsMetricGuard fs_metric(&s, &stub::metric::FSMetric::GetInstance().user_write, + out_wsize, butil::cpuwide_time_us()); + s = vfs_->Write(ino, buf, size, offset, fh, out_wsize); VLOG(1) << "VFSWrite end ino: " << ino << " size: " << size << " offset: " << offset << " fh: " << fh diff --git a/src/client/vfs_wrapper/vfs_wrapper.h b/src/client/vfs_wrapper/vfs_wrapper.h index f72b5030f..df30bd6c9 100644 --- a/src/client/vfs_wrapper/vfs_wrapper.h +++ b/src/client/vfs_wrapper/vfs_wrapper.h @@ -158,6 +158,29 @@ struct ClientOpMetricGuard { uint64_t start; }; +struct FsMetricGuard { + explicit FsMetricGuard(Status* status, stub::metric::InterfaceMetric* metric, + size_t* count, uint64_t start) + : status_(status), metric_(metric), count_(count), start_(start) {} + + ~FsMetricGuard() { + if (status_->ok()) { + metric_->bps.count << *count_; + metric_->qps.count << 1; + auto duration = butil::cpuwide_time_us() - start_; + metric_->latency << duration; + metric_->latTotal << duration; + } else { + metric_->eps.count << 1; + } + } + + Status* status_; + stub::metric::InterfaceMetric* metric_; + size_t* count_; + uint64_t start_; +}; + } // namespace vfs } // namespace client } // namespace dingofs diff --git a/src/stub/metric/metric.cpp b/src/stub/metric/metric.cpp index 7e57889a3..78cfb9335 100644 --- a/src/stub/metric/metric.cpp +++ b/src/stub/metric/metric.cpp @@ -34,7 +34,7 @@ const std::string ClientOpMetric::prefix = "dingofs_fuse"; // NOLINT const std::string S3MultiManagerMetric::prefix = "dingofs_client_manager"; // NOLINT const std::string FSMetric::prefix = "dingofs_filesystem"; // NOLINT -const std::string S3Metric::prefix = "dingofs_s3"; // NOLINT +const std::string ObjectMetric::prefix = "dingofs_object"; // NOLINT const std::string DiskCacheMetric::prefix = "dingofs_diskcache"; // NOLINT const std::string KVClientMetric::prefix = "dingofs_kvclient"; // NOLINT const std::string S3ChunkInfoMetric::prefix = "inode_s3_chunk_info"; // NOLINT diff --git a/src/stub/metric/metric.h b/src/stub/metric/metric.h index 69ab43834..34c81116f 100644 --- a/src/stub/metric/metric.h +++ b/src/stub/metric/metric.h @@ -268,23 +268,25 @@ struct FSMetric { } }; -struct S3Metric { +// s3 or rados read write metric +struct ObjectMetric { static const std::string prefix; - InterfaceMetric write_s3; - InterfaceMetric read_s3; + InterfaceMetric write_object; + InterfaceMetric read_object; private: - explicit S3Metric() - : write_s3(prefix, "_write_s3"), read_s3(prefix, "_read_s3") {} + explicit ObjectMetric() + : write_object(prefix, "_write_object"), + read_object(prefix, "_read_object") {} - S3Metric(const S3Metric&) = delete; + ObjectMetric(const ObjectMetric&) = delete; - S3Metric& operator=(const S3Metric&) = delete; + ObjectMetric& operator=(const ObjectMetric&) = delete; public: - static S3Metric& GetInstance() { - static S3Metric instance; + static ObjectMetric& GetInstance() { + static ObjectMetric instance; return instance; } }; diff --git a/test/stub/metric/client_metric_test.cpp b/test/stub/metric/client_metric_test.cpp index 45d281044..d5a310d2d 100644 --- a/test/stub/metric/client_metric_test.cpp +++ b/test/stub/metric/client_metric_test.cpp @@ -35,8 +35,8 @@ using ::dingofs::stub::metric::FSMetric; using ::dingofs::stub::metric::KVClientMetric; using ::dingofs::stub::metric::MDSClientMetric; using ::dingofs::stub::metric::MetaServerClientMetric; +using ::dingofs::stub::metric::ObjectMetric; using ::dingofs::stub::metric::S3ChunkInfoMetric; -using ::dingofs::stub::metric::S3Metric; using ::dingofs::stub::metric::S3MultiManagerMetric; using ::dingofs::stub::metric::WarmupManagerS3Metric; @@ -73,8 +73,8 @@ TEST_F(ClientMetricTest, test_prefix) { } { - const char* prefix = "dingofs_s3"; - ASSERT_EQ(0, ::strcmp(S3Metric::prefix.c_str(), prefix)); + const char* prefix = "dingofs_object"; + ASSERT_EQ(0, ::strcmp(ObjectMetric::prefix.c_str(), prefix)); } {