feat: Add model_load_time metric (#397)

Model Load time Metric added

Author: indrajit96

src/constants.h

@@ -85,6 +85,7 @@ constexpr char kInitialStateFolder[] = "initial_state";
 
 // Metric names
 constexpr char kPendingRequestMetric[] = "inf_pending_request_count";
+constexpr char kModelLoadTimeMetric[] = "model_load_time";
 
 constexpr uint64_t NANOS_PER_SECOND = 1000000000;
 constexpr uint64_t NANOS_PER_MILLIS = 1000000;

src/metric_model_reporter.cc

@@ -251,6 +251,7 @@ MetricModelReporter::InitializeGauges(
 {
   // Always setup these inference request metrics, regardless of config
   gauge_families_[kPendingRequestMetric] = &Metrics::FamilyInferenceQueueSize();
+  gauge_families_[kModelLoadTimeMetric] = &Metrics::FamilyModelLoadTime();
 
   for (auto& iter : gauge_families_) {
     const auto& name = iter.first;
@@ -392,6 +393,15 @@ MetricModelReporter::IncrementGauge(const std::string& name, double value)
   }
 }
 
+void
+MetricModelReporter::SetGauge(const std::string& name, double value)
+{
+  auto gauge = GetGauge(name);
+  if (gauge) {
+    gauge->Set(value);
+  }
+}
+
 void
 MetricModelReporter::DecrementGauge(const std::string& name, double value)
 {

src/metric_model_reporter.h

@@ -87,6 +87,8 @@ class MetricModelReporter {
   const MetricReporterConfig& Config();
   // Lookup counter metric by name, and increment it by value if it exists.
   void IncrementCounter(const std::string& name, double value);
+  // Overwrite gauge to value
+  void SetGauge(const std::string& name, double value);
   // Increase gauge by value.
   void IncrementGauge(const std::string& name, double value);
   // Decrease gauge by value.

src/metrics.cc

@@ -109,6 +109,11 @@ Metrics::Metrics()
                     "execution per-model.")
               .Register(*registry_)),
 
+      model_load_time_family_(prometheus::BuildGauge()
+                                  .Name("nv_model_load_duration_secs")
+                                  .Help("Model load time in seconds")
+                                  .Register(*registry_)),
+
       pinned_memory_pool_total_family_(
           prometheus::BuildGauge()
               .Name("nv_pinned_memory_pool_total_bytes")

src/metrics.h

@@ -215,6 +215,12 @@ class Metrics {
     return GetSingleton()->inf_pending_request_count_family_;
   }
 
+  // Metric family of load time per model
+  static prometheus::Family<prometheus::Gauge>& FamilyModelLoadTime()
+  {
+    return GetSingleton()->model_load_time_family_;
+  }
+
   // Metric families of per-model response cache metrics
   // NOTE: These are used in infer_stats for perf_analyzer
   static prometheus::Family<prometheus::Counter>& FamilyCacheHitCount()
@@ -300,6 +306,7 @@ class Metrics {
   prometheus::Family<prometheus::Counter>&
       inf_compute_output_duration_us_family_;
   prometheus::Family<prometheus::Gauge>& inf_pending_request_count_family_;
+  prometheus::Family<prometheus::Gauge>& model_load_time_family_;
 
   prometheus::Family<prometheus::Gauge>& pinned_memory_pool_total_family_;
   prometheus::Family<prometheus::Gauge>& pinned_memory_pool_used_family_;

src/model_repository_manager/model_lifecycle.cc

@@ -36,6 +36,7 @@
 #include "backend_model.h"
 #include "constants.h"
 #include "filesystem/api.h"
+#include "metrics.h"
 #include "model.h"
 #include "model_config_utils.h"
 #include "repo_agent.h"
@@ -559,11 +560,20 @@ ModelLifeCycle::CreateModel(
   // backend.
   if (!model_config.backend().empty()) {
     std::unique_ptr<TritonModel> model;
+    const uint64_t model_load_ns =
+        std::chrono::duration_cast<std::chrono::nanoseconds>(
+            std::chrono::steady_clock::now().time_since_epoch())
+            .count();
     status = TritonModel::Create(
         server_, model_info->model_path_, options_.backend_cmdline_config_map,
         options_.host_policy_map, model_id, version, model_config,
         is_config_provided, &model);
     is.reset(model.release());
+    if (status.IsOk()) {
+#ifdef TRITON_ENABLE_METRICS
+      CalculateAndReportLoadTime(model_load_ns, &is);
+#endif  // TRITON_ENABLE_METRICS
+    }
   } else {
 #ifdef TRITON_ENABLE_ENSEMBLE
     if (model_info->is_ensemble_) {
@@ -799,10 +809,8 @@ ModelLifeCycle::OnLoadFinal(
     // Mark current versions ready and track info in foreground
     for (auto& loaded : load_tracker->load_set_) {
       std::lock_guard<std::mutex> curr_info_lk(loaded.second->mtx_);
-
       loaded.second->state_ = ModelReadyState::READY;
       loaded.second->state_reason_.clear();
-
       auto bit = background_models_.find((uintptr_t)loaded.second);
       // Check if the version model is loaded in background, if so,
       // replace and unload the current serving version
@@ -847,4 +855,35 @@ ModelLifeCycle::OnLoadFinal(
   }
 }
 
+void
+ModelLifeCycle::CalculateAndReportLoadTime(
+    uint64_t load_start_ns_, std::unique_ptr<Model>* model)
+{
+#ifdef TRITON_ENABLE_METRICS
+  auto reporter = (*model)->MetricReporter();
+  const uint64_t now_ns =
+      std::chrono::duration_cast<std::chrono::nanoseconds>(
+          std::chrono::steady_clock::now().time_since_epoch())
+          .count();
+  uint64_t time_to_load_ns = now_ns - load_start_ns_;
+  std::chrono::duration<double> time_to_load =
+      std::chrono::duration_cast<std::chrono::duration<double>>(
+          std::chrono::nanoseconds(time_to_load_ns));
+  ReportModelLoadTime(reporter, time_to_load);
+#endif  // TRITON_ENABLE_METRICS
+}
+
+void
+ModelLifeCycle::ReportModelLoadTime(
+    std::shared_ptr<MetricModelReporter> reporter,
+    const std::chrono::duration<double>& time_to_load)
+{
+#ifdef TRITON_ENABLE_METRICS
+  if (reporter) {
+    double load_time_in_seconds = time_to_load.count();
+    reporter->SetGauge(kModelLoadTimeMetric, load_time_in_seconds);
+  }
+#endif  // TRITON_ENABLE_METRICS
+}
+
 }}  // namespace triton::core

src/model_repository_manager/model_lifecycle.h

@@ -31,6 +31,7 @@
 #include <map>
 #include <mutex>
 
+#include "backend_model.h"
 #include "infer_parameter.h"
 #include "model.h"
 #include "model_config.pb.h"
@@ -313,6 +314,13 @@ class ModelLifeCycle {
       ModelInfo* model_info, const bool is_update,
       const std::function<void(Status)>& OnComplete,
       std::shared_ptr<LoadTracker> load_tracker);
+  // Calculate time to load model
+  void CalculateAndReportLoadTime(
+      uint64_t load_start_ns_, std::unique_ptr<Model>* model);
+  // Report Load time per model metrics
+  void ReportModelLoadTime(
+      std::shared_ptr<MetricModelReporter> reporter,
+      const std::chrono::duration<double>& time_to_load);
   // Helper function for 'OnLoadComplete()' to finish final operations after
   // loading **all** model versions.
   void OnLoadFinal(