Merge CMSSW_15_1_X into CMSSW_15_1_FASTPU_X.

Author: cmsbuild

FWCore/AbstractServices/interface/IntrusiveMonitorBase.h

@@ -0,0 +1,39 @@
+#ifndef FWCore_AbstractServices_IntrusiveMonitorBase_h
+#define FWCore_AbstractServices_IntrusiveMonitorBase_h
+
+#include <string_view>
+
+namespace edm {
+  class IntrusiveMonitorBase {
+  public:
+    IntrusiveMonitorBase() = default;
+    IntrusiveMonitorBase(IntrusiveMonitorBase const&) = delete;
+    IntrusiveMonitorBase& operator=(IntrusiveMonitorBase const&) = delete;
+    IntrusiveMonitorBase(IntrusiveMonitorBase&&) = delete;
+    IntrusiveMonitorBase& operator=(IntrusiveMonitorBase&&) = delete;
+    virtual ~IntrusiveMonitorBase();
+
+    class Guard {
+    public:
+      Guard(IntrusiveMonitorBase& mon, std::string_view name) : monitor_(mon), name_(name) { monitor_.start(); }
+      Guard(Guard const&) = delete;
+      Guard& operator=(Guard const&) = delete;
+      Guard(Guard&&) = delete;
+      Guard& operator=(Guard&&) = delete;
+
+      ~Guard() { monitor_.stop(name_); }
+
+    private:
+      IntrusiveMonitorBase& monitor_;
+      std::string_view name_;
+    };
+
+    Guard startMonitoring(std::string_view name) { return Guard(*this, name); }
+
+  private:
+    virtual void start() = 0;
+    virtual void stop(std::string_view name) = 0;
+  };
+}  // namespace edm
+
+#endif

FWCore/AbstractServices/src/IntrusiveMonitorBase.cc

@@ -0,0 +1,5 @@
+#include "FWCore/AbstractServices/interface/IntrusiveMonitorBase.h"
+
+namespace edm {
+  IntrusiveMonitorBase::~IntrusiveMonitorBase() = default;
+}

IOPool/TFileAdaptor/src/TStorageFactoryFile.cc

@@ -303,6 +303,13 @@ Bool_t TStorageFactoryFile::ReadBuffer(char *buf, Int_t len) {
           GetRelOffset(),
           GetSize());
   }
+
+  // Update ROOT's statistics
+  fBytesRead += n;
+  fgBytesRead += n;
+  fReadCalls++;
+  fgReadCalls++;
+
   return n ? kFALSE : kTRUE;
 }
 
@@ -410,6 +417,12 @@ Bool_t TStorageFactoryFile::ReadBuffersSync(char *buf, Long64_t *pos, Int_t *len
     xstats.tick(io_buffer_used);
     repacker.unpack(current_buffer);
 
+    // Update ROOT's statistics
+    fBytesRead += result;
+    fgBytesRead += result;
+    fReadCalls++;
+    fgReadCalls++;
+
     // Update the location of the unused part of the input buffer.
     remaining_buffer_size -= real_bytes_processed;
     current_buffer += real_bytes_processed;

PerfTools/AllocMonitor/README.md

@@ -132,3 +132,36 @@ service in the configuration. The parameters are
 The beginning of the file contains a description of the structure and contents of the file. The file can be analyzed with the helper script `edmModuleEventAllocMonitorAnalyze.py`. The script can be used to find modules where the memory is being retained between Events as well as modules where the memory appears to be growing Event to Event. Use `--help` with the script for a full description.
 
 This service is multi-thread safe.
+
+
+### IntrusiveAllocMonitor
+
+This service registers a monitor when the service is created (after python parsing is finished, but before any modules have been loaded into cmsRun). The user can then start the monitoring in their code as shown in the example below. The monitoring stops, in an RAII fashion, at the end of the scope and the results of the memory operations are printed with the [MessageLogger](../../FWCore/MessageService/Readme.md) with an `IntrusiveAllocMonitor` message category.
+
+```cpp
+#include "FWCore/AbstractServices/interface/IntrusiveMonitorBase.h"
+#include "FWCore/ServiceRegistry/interface/Service.h"
+
+
+void someFunction() {
+  {
+    edm::Service<IntrusiveMonitorBase> monitor;
+    auto guard = monitor.startMeasurement("Measurement description");
+
+    // more code doing memory allocations
+  }
+}
+
+```
+
+The message will print
+| Field | Description |
+|-------|-------------|
+| `requested` | Total number of requested bytes |
+| `added ` | Total number of bytes added (can be more than `requested`) |
+| `max alloc` | Largest single memory allocation |
+| `peak` | Maximum amount of allocated memory at any given moment |
+| `nAlloc` | Number of memory allocations |
+| `nDealloc` | Number of memory deallocations |
+
+This service is multi-thread safe, and concurrent measurements can be done in different threads. Nested measurements are not supported (i.e. one thread can be doing only one measurement at a time), and neither are measurements started in one thread and ended in different thread.

PerfTools/AllocMonitor/plugins/IntrusiveAllocMonitor.cc

@@ -0,0 +1,91 @@
+#include "FWCore/AbstractServices/interface/IntrusiveMonitorBase.h"
+#include "FWCore/MessageLogger/interface/MessageLogger.h"
+#include "FWCore/ServiceRegistry/interface/ServiceMaker.h"
+
+#include "PerfTools/AllocMonitor/interface/AllocMonitorBase.h"
+#include "PerfTools/AllocMonitor/interface/AllocMonitorRegistry.h"
+
+#include "ThreadAllocInfo.h"
+#include "ThreadTracker.h"
+
+namespace {
+  using namespace edm::service::moduleAlloc;
+  class MonitorAdaptor : public cms::perftools::AllocMonitorBase {
+  public:
+    static void startOnThread() { threadAllocInfo().reset(); }
+    static ThreadAllocInfo const& stopOnThread() {
+      auto& t = threadAllocInfo();
+      if (not t.active_) {
+        t.reset();
+      } else {
+        t.deactivate();
+      }
+      return t;
+    }
+
+    static ThreadAllocInfo& threadAllocInfo() {
+      using namespace cms::perftools::allocMon;
+      static ThreadAllocInfo s_info[ThreadTracker::kTotalEntries];
+      return s_info[ThreadTracker::instance().thread_index()];
+    }
+
+  private:
+    void allocCalled(size_t iRequested, size_t iActual, void const*) final {
+      auto& allocInfo = threadAllocInfo();
+      if (not allocInfo.active_) {
+        return;
+      }
+      allocInfo.nAllocations_ += 1;
+      allocInfo.requested_ += iRequested;
+
+      if (allocInfo.maxSingleAlloc_ < iRequested) {
+        allocInfo.maxSingleAlloc_ = iRequested;
+      }
+
+      allocInfo.presentActual_ += iActual;
+      if (allocInfo.presentActual_ > static_cast<long long>(allocInfo.maxActual_)) {
+        allocInfo.maxActual_ = allocInfo.presentActual_;
+      }
+    }
+    void deallocCalled(size_t iActual, void const*) final {
+      auto& allocInfo = threadAllocInfo();
+      if (not allocInfo.active_) {
+        return;
+      }
+
+      allocInfo.nDeallocations_ += 1;
+      allocInfo.presentActual_ -= iActual;
+      if (allocInfo.presentActual_ < 0) {
+        if (allocInfo.minActual_ == 0 or allocInfo.minActual_ > allocInfo.presentActual_) {
+          allocInfo.minActual_ = allocInfo.presentActual_;
+        }
+      }
+    }
+  };
+}  // namespace
+
+class IntrusiveAllocMonitor : public edm::IntrusiveMonitorBase {
+public:
+  IntrusiveAllocMonitor() {
+    (void)cms::perftools::AllocMonitorRegistry::instance().createAndRegisterMonitor<MonitorAdaptor>();
+  };
+  ~IntrusiveAllocMonitor() override = default;
+
+  void start() final { MonitorAdaptor::startOnThread(); }
+  void stop(std::string_view name) final {
+    MonitorAdaptor::stopOnThread();
+    auto& info = MonitorAdaptor::threadAllocInfo();
+    edm::LogSystem("IntrusiveAllocMonitor")
+        .format("{}: requested {} added {} max alloc {} peak {} nAlloc {} nDealloc {}",
+                name,
+                info.requested_,
+                info.presentActual_,
+                info.maxSingleAlloc_,
+                info.maxActual_,
+                info.nAllocations_,
+                info.nDeallocations_);
+  }
+};
+
+typedef edm::serviceregistry::NoArgsMaker<edm::IntrusiveMonitorBase, IntrusiveAllocMonitor> IntrusiveAllocMonitorMaker;
+DEFINE_FWK_SERVICE_MAKER(IntrusiveAllocMonitor, IntrusiveAllocMonitorMaker);

PerfTools/AllocMonitor/test/BuildFile.xml

@@ -23,4 +23,18 @@
     <flags CXXFLAGS="-O0"/>
   </bin>
   <test name="TestPerfToolsModuleAllocMonitor" command="runModuleAlloc.sh"/>
+
+  <!-- can't be part of the catch2 tests above because of ASAN exclusion -->
+  <bin file="test_intrusiveAllocMonitor.cc" name="testIntrusiveAllocMonitor">
+    <use name="FWCore/AbstractServices"/>
+    <use name="FWCore/ParameterSet"/>
+    <use name="FWCore/ParameterSetReader"/>
+    <use name="FWCore/PluginManager"/>
+    <use name="FWCore/ServiceRegistry"/>
+    <use name="PerfTools/AllocMonitor"/>
+    <use name="PerfTools/AllocMonitorPreload"/>
+  </bin>
+  <test name="testIntrusiveAllocMonitorOutput" command="testIntrusiveAllocMonitor 2>&amp;1 | grep -q 'Vector fill: requested'">
+    <flags PRE_TEST="testIntrusiveAllocMonitor"/>
+  </test>
 </ifrelease>

PerfTools/AllocMonitor/test/test_intrusiveAllocMonitor.cc

@@ -0,0 +1,46 @@
+#include "FWCore/AbstractServices/interface/IntrusiveMonitorBase.h"
+#include "FWCore/MessageLogger/interface/MessageLogger.h"
+#include "FWCore/ParameterSet/interface/ParameterSet.h"
+#include "FWCore/ParameterSetReader/interface/ParameterSetReader.h"
+#include "FWCore/PluginManager/interface/PluginManager.h"
+#include "FWCore/PluginManager/interface/standard.h"
+#include "FWCore/ServiceRegistry/interface/Service.h"
+#include "FWCore/ServiceRegistry/interface/ServiceRegistry.h"
+#include "FWCore/ServiceRegistry/interface/ServiceToken.h"
+
+#include <cassert>
+#include <vector>
+
+int main() {
+  edmplugin::PluginManager::configure(edmplugin::standard::config());
+
+  std::string const config = R"_(
+import FWCore.ParameterSet.Config as cms
+process = cms.Process('Test')
+process.add_(cms.Service('IntrusiveAllocMonitor'))
+)_";
+  std::unique_ptr<edm::ParameterSet> params;
+  edm::makeParameterSets(config, params);
+  auto token = edm::ServiceToken(edm::ServiceRegistry::createServicesFromConfig(std::move(params)));
+  edm::ServiceRegistry::Operate operate(token);
+
+  edm::Service<edm::IntrusiveMonitorBase> imb;
+  assert(imb.isAvailable());
+
+  std::vector<int> vec;
+  {
+    auto guard = imb->startMonitoring("Vector fill");
+    for (int i = 0; i < 10000; ++i) {
+      vec.push_back(i * 2 - 1);
+    }
+
+    int sum = 0;
+    for (int a : vec) {
+      sum += a;
+    }
+
+    edm::LogPrint("Test").format("Sum {}", sum);
+  }
+
+  return 0;
+}