Add instrumentation and expose measurement in prometheus format

.gitmodules

@@ -10,3 +10,6 @@
 	path = lib/googletest
 	url = https://github.com/google/googletest
 	ignore = dirty
+[submodule "lib/prometheus-cpp"]
+	path = lib/prometheus-cpp
+	url = https://github.com/jupp0r/prometheus-cpp

src/.gitignore

@@ -1,4 +1,5 @@
 init-submodules
 build-cpp-netlib
 build-yaml-cpp
+build-prometheus-cpp
 metadatad

src/Makefile

@@ -23,7 +23,10 @@ NETWORK_URI_DIR=$(CPP_NETLIB_DIR)/deps/uri
 NETWORK_URI_LIBDIR=$(NETWORK_URI_DIR)/src
 YAML_CPP_DIR=$(LIBDIR)/yaml-cpp
 YAML_CPP_LIBDIR=$(YAML_CPP_DIR)
-SUBMODULE_DIRS=$(CPP_NETLIB_DIR) $(YAML_CPP_DIR)
+PROMETHEUS_CPP_DIR=$(LIBDIR)/prometheus-cpp
+PROMETHEUS_CPP_LIBDIR=$(PROMETHEUS_CPP_DIR)
+PROMETHEUS_CPP_CORE_LIBDIR=$(PROMETHEUS_CPP_LIBDIR)/core
+SUBMODULE_DIRS=$(CPP_NETLIB_DIR) $(YAML_CPP_DIR) $(PROMETHEUS_CPP_DIR)
 
 GIT=git
 GIT_VERSION=$(shell $(GIT) --version | grep -oh '[0-9]\+\.[0-9]\+\.[0-9]\+')
@@ -33,14 +36,16 @@ CMAKE=cmake
 CPPFLAGS=\
     -DAGENT_VERSION='$(PKG_VERSION)-$(PKG_RELEASE)' \
     -DENABLE_DOCKER_METADATA \
-    -I$(CPP_NETLIB_DIR) -I$(NETWORK_URI_DIR)/include -I$(YAML_CPP_DIR)/include
+    -I$(CPP_NETLIB_DIR) -I$(NETWORK_URI_DIR)/include -I$(YAML_CPP_DIR)/include \
+    -I$(PROMETHEUS_CPP_CORE_LIBDIR)/include
 CXXFLAGS=\
     -std=c++11 -g -pthread -Wno-write-strings -Wno-deprecated
-LDFLAGS=-L$(CPP_NETLIB_LIBDIR) -L$(NETWORK_URI_LIBDIR) -L$(YAML_CPP_LIBDIR)
+LDFLAGS=-L$(CPP_NETLIB_LIBDIR) -L$(NETWORK_URI_LIBDIR) -L$(YAML_CPP_LIBDIR) \
+	-L$(PROMETHEUS_CPP_CORE_LIBDIR)
 LDLIBS=\
     -lcppnetlib-client-connections -lcppnetlib-server-parsers -lnetwork-uri \
     -lboost_program_options -lboost_system -lboost_thread -lboost_filesystem \
-    -lpthread -lyajl -lssl -lcrypto -lyaml-cpp
+    -lpthread -lyajl -lssl -lcrypto -lyaml-cpp -lprometheus-cpp-core
 SED_EXTRA=-e 's/-Wall/-Wall -Wno-deprecated/'
 
 UNAME_S=$(shell uname -s)
@@ -88,7 +93,9 @@ CPP_NETLIB_LIBS=\
     $(NETWORK_URI_LIBDIR)/libnetwork-uri.a
 YAML_CPP_LIBS=\
     $(YAML_CPP_LIBDIR)/libyaml-cpp.a
-LIBS=$(CPP_NETLIB_LIBS) $(YAML_CPP_LIBS)
+PROMETHEUS_CPP_LIBS=\
+    $(PROMETHEUS_CPP_CORE_LIBDIR)/libprometheus-cpp-core.a
+LIBS=$(CPP_NETLIB_LIBS) $(YAML_CPP_LIBS) $(PROMETHEUS_CPP_LIBS)
 
 sbindir=/opt/stackdriver/metadata/sbin
 INSTALL=/usr/bin/install
@@ -167,7 +174,7 @@ clean:
 	$(RM) metadatad $(OBJS)
 
 purge: clean
-	$(RM) -r init-submodules build-cpp-netlib build-yaml-cpp
+	$(RM) -r init-submodules build-cpp-netlib build-yaml-cpp build-prometheus-cpp
 	(cd .. && git submodule deinit -f $(SUBMODULE_DIRS:../%=%))
 
 init-submodules:
@@ -183,6 +190,8 @@ ifneq ($(findstring $(GIT_VERSION),2.7.0 2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.8
 	    deps/*/.git
 	cd $(CPP_NETLIB_DIR) && $(SED_I) -e 's@/src/.git/@../../../../../../../.git/@' \
 	    libs/network/doc/_ext/breathe/.git
+	cd $(PROMETHEUS_CPP_DIR) && $(SED_I) -e 's@/src/.git/@../../../../.git/@' \
+	    3rdparty/*/.git
 endif
 	touch init-submodules
 
@@ -217,12 +226,28 @@ build-yaml-cpp: $(YAML_CPP_DIR)/Makefile
 	$(MAKE)
 	touch build-yaml-cpp
 
+$(PROMETHEUS_CPP_DIR)/Makefile: init-submodules
+	cd $(PROMETHEUS_CPP_DIR) && \
+	$(CMAKE) -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS=-std=c++11 \
+	    -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ \
+	    -DENABLE_PUSH=OFF -DENABLE_PULL=OFF -DENABLE_COMPRESSION=OFF \
+	    -DENABLE_TESTING=OFF
+
+$(PROMETHEUS_CPP_LIBS): build-prometheus-cpp
+
+build-prometheus-cpp: $(PROMETHEUS_CPP_DIR)/Makefile
+	cd $(PROMETHEUS_CPP_DIR) && \
+	$(MAKE)
+	touch build-prometheus-cpp
+
 cpp-netlib: $(CPP_NETLIB_LIBS)
 
 yaml-cpp: $(YAML_CPP_LIBS)
 
-submodules: cpp-netlib yaml-cpp
+prometheus-cpp: $(PROMETHEUS_CPP_LIBS)
+
+submodules: cpp-netlib yaml-cpp prometheus-cpp
 
 all: submodules metadatad
 
-.PHONY: all submodules cpp-netlib yaml-cpp purge clean install deb rpm
+.PHONY: all submodules cpp-netlib yaml-cpp prometheus-cpp purge clean install deb rpm

src/agent.cc

@@ -24,13 +24,14 @@
 namespace google {
 
 MetadataAgent::MetadataAgent(const Configuration& config)
-    : config_(config), store_(config_), health_checker_(config, store_) {}
+    : config_(config), store_(config_), health_checker_(config, store_),
+      registry_(std::make_shared<prometheus::Registry>()) {}
 
 MetadataAgent::~MetadataAgent() {}
 
 void MetadataAgent::Start() {
   metadata_api_server_.reset(new MetadataApiServer(
-      config_, &health_checker_, store_, config_.MetadataApiNumThreads(),
+      config_, &health_checker_, registry_, store_, config_.MetadataApiNumThreads(),
       config_.MetadataApiBindAddress(), config_.MetadataApiPort()));
   reporter_.reset(new MetadataReporter(
       config_, &store_, config_.MetadataReporterIntervalSeconds()));

src/agent.h

@@ -17,6 +17,7 @@
 #define AGENT_H_
 
 #include <memory>
+#include <prometheus/registry.h>
 
 #include "store.h"
 #include "health_checker.h"
@@ -60,6 +61,10 @@ class MetadataAgent {
     return &health_checker_;
   }
 
+  std::shared_ptr<prometheus::Registry> prometheus_registry() {
+    return registry_;
+  }
+
  private:
   const Configuration& config_;
 
@@ -68,6 +73,8 @@ class MetadataAgent {
 
   HealthChecker health_checker_;
 
+  std::shared_ptr<prometheus::Registry> registry_;
+
   // The Metadata API server.
   std::unique_ptr<MetadataApiServer> metadata_api_server_;
   // The metadata reporter.

src/api_server.cc

@@ -17,6 +17,7 @@
 #include "api_server.h"
 
 #include <boost/range/irange.hpp>
+#include <prometheus/text_serializer.h>
 
 #include "configuration.h"
 #include "health_checker.h"
@@ -69,10 +70,12 @@ void MetadataApiServer::Dispatcher::log(const HttpServer::string_type& info) con
 
 MetadataApiServer::MetadataApiServer(const Configuration& config,
                                      const HealthChecker* health_checker,
+                                     const std::shared_ptr<prometheus::Collectable> collectable,
                                      const MetadataStore& store,
                                      int server_threads,
                                      const std::string& host, int port)
-    : config_(config), health_checker_(health_checker), store_(store),
+    : config_(config), health_checker_(health_checker), collectable_(collectable),
+      store_(store),
       dispatcher_({
         {{"GET", "/monitoredResource/"},
          [=](const HttpServer::request& request,
@@ -84,6 +87,11 @@ MetadataApiServer::MetadataApiServer(const Configuration& config,
              std::shared_ptr<HttpServer::connection> conn) {
              HandleHealthz(request, conn);
          }},
+        {{"GET", "/metrics"},
+         [=](const HttpServer::request& request,
+             std::shared_ptr<HttpServer::connection> conn) {
+             HandleMetrics(request, conn);
+         }},
       }, config_.VerboseLogging()),
       server_(
           HttpServer::options(dispatcher_)
@@ -200,4 +208,26 @@ void MetadataApiServer::HandleHealthz(
   }
 }
 
+void MetadataApiServer::HandleMetrics(
+    const HttpServer::request& request,
+    std::shared_ptr<HttpServer::connection> conn) {
+  std::string response = SerializeMetricsToPrometheusTextFormat();
+  conn->set_status(HttpServer::connection::ok);
+  conn->set_headers(std::map<std::string, std::string>({
+    {"Connection", "close"},
+    {"Content-Length", std::to_string(response.size())},
+    {"Content-Type", "application/json"},
+  }));
+  conn->write(response);
+}
+
+std::string MetadataApiServer::SerializeMetricsToPrometheusTextFormat() const {
+  if (!collectable_) {
+    return "";
+  }
+
+  prometheus::TextSerializer text_serializer;
+  return std::move(text_serializer.Serialize(collectable_->Collect()));
+}
+
 }

src/api_server.h

@@ -18,6 +18,7 @@
 
 #define BOOST_NETWORK_ENABLE_HTTPS
 #include <boost/network/protocol/http/server.hpp>
+#include <prometheus/collectable.h>
 #include <functional>
 #include <map>
 #include <memory>
@@ -43,6 +44,7 @@ class MetadataApiServer {
  public:
   MetadataApiServer(const Configuration& config,
                     const HealthChecker* health_checker,
+                    const std::shared_ptr<prometheus::Collectable> collectable,
                     const MetadataStore& store, int server_threads,
                     const std::string& host, int port);
   ~MetadataApiServer();
@@ -77,13 +79,17 @@ class MetadataApiServer {
                                std::shared_ptr<HttpServer::connection> conn);
   void HandleHealthz(const HttpServer::request& request,
                      std::shared_ptr<HttpServer::connection> conn);
+  void HandleMetrics(const HttpServer::request& request,
+                     std::shared_ptr<HttpServer::connection> conn);
+  std::string SerializeMetricsToPrometheusTextFormat() const;
 
   const Configuration& config_;
   const HealthChecker* health_checker_;
   const MetadataStore& store_;
   Dispatcher dispatcher_;
   HttpServer server_;
   std::vector<std::thread> server_pool_;
+  std::shared_ptr<prometheus::Collectable> collectable_;
 };
 
 }

test/Makefile

@@ -32,23 +32,31 @@ NETWORK_URI_DIR=$(CPP_NETLIB_DIR)/deps/uri
 NETWORK_URI_LIBDIR=$(NETWORK_URI_DIR)/src
 YAML_CPP_DIR=$(LIBDIR)/yaml-cpp
 YAML_CPP_LIBDIR=$(YAML_CPP_DIR)
+PROMETHEUS_CPP_DIR=$(LIBDIR)/prometheus-cpp
+PROMETHEUS_CPP_LIBDIR=$(PROMETHEUS_CPP_DIR)
+PROMETHEUS_CPP_CORE_LIBDIR=$(PROMETHEUS_CPP_LIBDIR)/core
 
 CPP_NETLIB_LIBS=\
     $(CPP_NETLIB_LIBDIR)/libcppnetlib-client-connections.a \
     $(CPP_NETLIB_LIBDIR)/libcppnetlib-server-parsers.a \
     $(NETWORK_URI_LIBDIR)/libnetwork-uri.a
 YAML_CPP_LIBS=$(YAML_CPP_LIBDIR)/libyaml-cpp.a
+PROMETHEUS_CPP_LIBS=\
+    $(PROMETHEUS_CPP_CORE_LIBDIR)/libprometheus-cpp-core.a
 
 CPPFLAGS+= \
     -isystem $(GTEST_DIR)/include -I$(GMOCK_DIR)/include \
-    -I$(CPP_NETLIB_DIR) -I$(NETWORK_URI_DIR)/include -I$(YAML_CPP_DIR)/include
+    -I$(CPP_NETLIB_DIR) -I$(NETWORK_URI_DIR)/include -I$(YAML_CPP_DIR)/include \
+    -I$(PROMETHEUS_CPP_CORE_LIBDIR)/include
 CXXFLAGS=\
     -std=c++11 -g -pthread -Wno-write-strings -Wno-deprecated
-LDFLAGS=-L$(CPP_NETLIB_LIBDIR) -L$(NETWORK_URI_LIBDIR) -L$(YAML_CPP_LIBDIR)
+LDFLAGS=-L$(CPP_NETLIB_LIBDIR) -L$(NETWORK_URI_LIBDIR) -L$(YAML_CPP_LIBDIR) \
+        -L$(PROMETHEUS_CPP_CORE_LIBDIR)
 LDLIBS=\
     -lcppnetlib-client-connections -lcppnetlib-server-parsers -lnetwork-uri \
     -lboost_program_options -lboost_system -lboost_thread -lboost_filesystem \
-    -lboost_regex -lpthread -lyajl -lssl -lcrypto -lyaml-cpp
+    -lboost_regex -lpthread -lyajl -lssl -lcrypto -lyaml-cpp \
+    -lprometheus-cpp-core
 
 UNAME_S=$(shell uname -s)
 ifeq ($(UNAME_S),Darwin)
@@ -115,14 +123,16 @@ init-submodules:
 $(SRC_DIR)/init-submodules:
 	cd $(SRC_DIR) && $(MAKE) $(@:$(SRC_DIR)/%=%)
 
-$(SRC_DIR)/build-cpp-netlib $(SRC_DIR)/build-yaml-cpp: $(SRC_DIR)/init-submodules
+$(SRC_DIR)/build-cpp-netlib $(SRC_DIR)/build-yaml-cpp $(SRC_DIR)/build-prometheus-cpp: $(SRC_DIR)/init-submodules
 	cd $(SRC_DIR) && $(MAKE) $(@:$(SRC_DIR)/%=%)
 
 $(CPP_NETLIB_LIBS): $(SRC_DIR)/build-cpp-netlib
 
 $(YAML_CPP_LIBS): $(SRC_DIR)/build-yaml-cpp
 
-$(SRC_DIR)/%.o: $(SRC_DIR)/build-cpp-netlib $(SRC_DIR)/build-yaml-cpp $(SRC_DIR)/%.cc
+$(PROMETHEUS_CPP_LIBS): $(SRC_DIR)/build-prometheus-cpp
+
+$(SRC_DIR)/%.o: $(SRC_DIR)/build-cpp-netlib $(SRC_DIR)/build-yaml-cpp $(SRC_DIR)/build-prometheus-cpp $(SRC_DIR)/%.cc
 	cd $(SRC_DIR) && $(MAKE) $(@:$(SRC_DIR)/%=%)
 
 $(GTEST_SOURCEDIR)/gtest-all.cc: init-submodules
@@ -139,14 +149,14 @@ gmock_main.o: $(GMOCK_SOURCEDIR)/gmock_main.cc
 $(GTEST_LIB): gtest-all.o gmock-all.o gmock_main.o
 	$(AR) $(ARFLAGS) $@ $^
 
-$(TESTS): $(GTEST_LIB) $(CPP_NETLIB_LIBS) $(YAML_CPP_LIBS)
+$(TESTS): $(GTEST_LIB) $(CPP_NETLIB_LIBS) $(YAML_CPP_LIBS) $(PROMETHEUS_CPP_LIBS)
 
 # All unittest objects depend on GTEST_LIB.
 # Some headers need CPP_NETLIB_LIBS and YAML_CPP_LIBS.
-$(TESTS:%=%.o): $(GTEST_LIB) $(CPP_NETLIB_LIBS) $(YAML_CPP_LIBS)
+$(TESTS:%=%.o): $(GTEST_LIB) $(CPP_NETLIB_LIBS) $(YAML_CPP_LIBS) $(PROMETHEUS_CPP_LIBS)
 
 api_server_unittest: api_server_unittest.o $(SRC_DIR)/api_server.o $(SRC_DIR)/configuration.o $(SRC_DIR)/store.o $(SRC_DIR)/json.o $(SRC_DIR)/resource.o $(SRC_DIR)/logging.o $(SRC_DIR)/time.o $(SRC_DIR)/health_checker.o
-	$(CXX) $(LDFLAGS) $^ $(LDLIBS) -o $@
+	$(CXX) $(LDFLAGS) $^ $(LDLIBS) -lprometheus-cpp-core -o $@
 base64_unittest: base64_unittest.o $(SRC_DIR)/base64.o
 	$(CXX) $(LDFLAGS) $^ $(LDLIBS) -o $@
 configuration_unittest: configuration_unittest.o $(SRC_DIR)/configuration.o