Add instrumentation and expose measurement in prometheus format

.gitmodules

@@ -10,3 +10,9 @@
 	path = lib/googletest
 	url = https://github.com/google/googletest
 	ignore = dirty
+[submodule "lib/prometheus-cpp"]
+	path = lib/prometheus-cpp
+	url = https://github.com/jupp0r/prometheus-cpp
+[submodule "lib/opencensus-cpp"]
+	path = lib/opencensus-cpp
+	url = https://github.com/census-instrumentation/opencensus-cpp

src/.gitignore

@@ -1,4 +1,6 @@
 init-submodules
 build-cpp-netlib
 build-yaml-cpp
+build-prometheus-cpp
+build-opencensus-cpp
 metadatad

src/Makefile

@@ -23,7 +23,15 @@ NETWORK_URI_DIR=$(CPP_NETLIB_DIR)/deps/uri
 NETWORK_URI_LIBDIR=$(NETWORK_URI_DIR)/src
 YAML_CPP_DIR=$(LIBDIR)/yaml-cpp
 YAML_CPP_LIBDIR=$(YAML_CPP_DIR)
-SUBMODULE_DIRS=$(CPP_NETLIB_DIR) $(YAML_CPP_DIR)
+PROMETHEUS_CPP_DIR=$(LIBDIR)/prometheus-cpp
+PROMETHEUS_CPP_CORE_DIR=$(PROMETHEUS_CPP_DIR)/core
+PROMETHEUS_CPP_CORE_LIBDIR=$(PROMETHEUS_CPP_CORE_DIR)
+OPENCENSUS_CPP_DIR=$(LIBDIR)/opencensus-cpp
+OPENCENSUS_CPP_OPENCENSUS_DIR=$(OPENCENSUS_CPP_DIR)/opencensus
+OPENCENSUS_CPP_OPENCENSUS_LIBDIR=$(OPENCENSUS_CPP_OPENCENSUS_DIR)
+OPENCENSUS_CPP_ABSL_INCLUDEDIR=$(OPENCENSUS_CPP_DIR)/abseil-src
+OPENCENSUS_CPP_ABSL_LIBDIR=$(OPENCENSUS_CPP_DIR)/abseil-build/absl
+SUBMODULE_DIRS=$(CPP_NETLIB_DIR) $(YAML_CPP_DIR) $(PROMETHEUS_CPP_DIR) $(OPENCENSUS_CPP_DIR)
 
 GIT=git
 GIT_VERSION=$(shell $(GIT) --version | grep -oh '[0-9]\+\.[0-9]\+\.[0-9]\+')
@@ -33,14 +41,47 @@ CMAKE=cmake
 CPPFLAGS=\
     -DAGENT_VERSION='$(PKG_VERSION)-$(PKG_RELEASE)' \
     -DENABLE_DOCKER_METADATA \
-    -I$(CPP_NETLIB_DIR) -I$(NETWORK_URI_DIR)/include -I$(YAML_CPP_DIR)/include
+    -I$(CPP_NETLIB_DIR) -I$(NETWORK_URI_DIR)/include -I$(YAML_CPP_DIR)/include \
+    -I$(PROMETHEUS_CPP_CORE_DIR)/include -I$(OPENCENSUS_CPP_DIR) \
+    -I$(OPENCENSUS_CPP_ABSL_INCLUDEDIR)
 CXXFLAGS=\
     -std=c++11 -g -pthread -Wno-write-strings -Wno-deprecated
-LDFLAGS=-L$(CPP_NETLIB_LIBDIR) -L$(NETWORK_URI_LIBDIR) -L$(YAML_CPP_LIBDIR)
+LDFLAGS=\
+    -L$(CPP_NETLIB_LIBDIR) \
+    -L$(NETWORK_URI_LIBDIR) \
+    -L$(YAML_CPP_LIBDIR) \
+    -L$(PROMETHEUS_CPP_CORE_LIBDIR) \
+    -L$(OPENCENSUS_CPP_OPENCENSUS_LIBDIR)/stats \
+    -L$(OPENCENSUS_CPP_OPENCENSUS_LIBDIR)/context \
+    -L$(OPENCENSUS_CPP_OPENCENSUS_LIBDIR)/exporters/stats/prometheus \
+    -L$(OPENCENSUS_CPP_OPENCENSUS_LIBDIR)/trace \
+    -L$(OPENCENSUS_CPP_OPENCENSUS_LIBDIR)/tags \
+    -L$(OPENCENSUS_CPP_OPENCENSUS_LIBDIR)/common/internal \
+    -L$(OPENCENSUS_CPP_ABSL_LIBDIR)/numeric \
+    -L$(OPENCENSUS_CPP_ABSL_LIBDIR)/strings \
+    -L$(OPENCENSUS_CPP_ABSL_LIBDIR)/synchronization \
+    -L$(OPENCENSUS_CPP_ABSL_LIBDIR)/time \
+    -L$(OPENCENSUS_CPP_ABSL_LIBDIR)/debugging \
+    -L$(OPENCENSUS_CPP_ABSL_LIBDIR)/base
 LDLIBS=\
     -lcppnetlib-client-connections -lcppnetlib-server-parsers -lnetwork-uri \
     -lboost_program_options -lboost_system -lboost_thread -lboost_filesystem \
-    -lpthread -lyajl -lssl -lcrypto -lyaml-cpp
+    -lpthread -lyajl -lssl -lcrypto -lyaml-cpp -lprometheus-cpp-core \
+    -lopencensus_exporters_stats_prometheus \
+    -lopencensus_exporters_stats_prometheus_utils \
+    -lopencensus_stats_core -lopencensus_stats_recording \
+    -lopencensus_tags -lopencensus_tags_context_util \
+    -lopencensus_context \
+    -lopencensus_trace_context_util \
+    -lopencensus_trace_with_span \
+    -lopencensus_trace \
+    -lopencensus_common_random \
+    -labsl_strings -labsl_synchronization -labsl_time \
+    -labsl_int128 \
+    -labsl_stacktrace -labsl_symbolize \
+    -labsl_demangle_internal -labsl_internal_debugging_internal \
+    -labsl_base -labsl_dynamic_annotations -labsl_internal_spinlock_wait \
+    -labsl_internal_malloc_internal -labsl_internal_throw_delegate
 SED_EXTRA=-e 's/-Wall/-Wall -Wno-deprecated/'
 
 UNAME_S=$(shell uname -s)
@@ -71,6 +112,8 @@ SRCS=\
      kubernetes.cc \
      resource.cc \
      oauth2.cc \
+     measures.cc \
+     internal/measures_utils.cc \
      logging.cc \
      local_stream_http.cc \
      local_stream_delegate.cc \
@@ -88,7 +131,34 @@ CPP_NETLIB_LIBS=\
     $(NETWORK_URI_LIBDIR)/libnetwork-uri.a
 YAML_CPP_LIBS=\
     $(YAML_CPP_LIBDIR)/libyaml-cpp.a
-LIBS=$(CPP_NETLIB_LIBS) $(YAML_CPP_LIBS)
+PROMETHEUS_CPP_LIBS=\
+    $(PROMETHEUS_CPP_CORE_LIBDIR)/libprometheus-cpp-core.a
+OPENCENSUS_CPP_LIBS=\
+    $(OPENCENSUS_CPP_ABSL_LIBDIR)/base/libabsl_base.a \
+    $(OPENCENSUS_CPP_ABSL_LIBDIR)/base/libabsl_internal_malloc_internal.a \
+    $(OPENCENSUS_CPP_ABSL_LIBDIR)/base/libabsl_internal_throw_delegate.a \
+    $(OPENCENSUS_CPP_ABSL_LIBDIR)/base/libabsl_internal_spinlock_wait.a \
+    $(OPENCENSUS_CPP_ABSL_LIBDIR)/base/libabsl_dynamic_annotations.a \
+    $(OPENCENSUS_CPP_ABSL_LIBDIR)/debugging/libabsl_demangle_internal.a \
+    $(OPENCENSUS_CPP_ABSL_LIBDIR)/debugging/libabsl_internal_debugging_internal.a \
+    $(OPENCENSUS_CPP_ABSL_LIBDIR)/debugging/libabsl_stacktrace.a \
+    $(OPENCENSUS_CPP_ABSL_LIBDIR)/debugging/libabsl_symbolize.a \
+    $(OPENCENSUS_CPP_ABSL_LIBDIR)/numeric/libabsl_int128.a \
+    $(OPENCENSUS_CPP_ABSL_LIBDIR)/synchronization/libabsl_synchronization.a \
+    $(OPENCENSUS_CPP_ABSL_LIBDIR)/strings/libabsl_strings.a \
+    $(OPENCENSUS_CPP_ABSL_LIBDIR)/time/libabsl_time.a \
+    $(OPENCENSUS_CPP_OPENCENSUS_LIBDIR)/exporters/stats/prometheus/libopencensus_exporters_stats_prometheus.a \
+    $(OPENCENSUS_CPP_OPENCENSUS_LIBDIR)/exporters/stats/prometheus/libopencensus_exporters_stats_prometheus_utils.a \
+    $(OPENCENSUS_CPP_OPENCENSUS_LIBDIR)/stats/libopencensus_stats_core.a \
+    $(OPENCENSUS_CPP_OPENCENSUS_LIBDIR)/context/libopencensus_context.a \
+    $(OPENCENSUS_CPP_OPENCENSUS_LIBDIR)/tags/libopencensus_tags.a \
+    $(OPENCENSUS_CPP_OPENCENSUS_LIBDIR)/tags/libopencensus_tags_context_util.a \
+    $(OPENCENSUS_CPP_OPENCENSUS_LIBDIR)/trace/libopencensus_trace.a \
+    $(OPENCENSUS_CPP_OPENCENSUS_LIBDIR)/trace/libopencensus_trace_with_span.a \
+    $(OPENCENSUS_CPP_OPENCENSUS_LIBDIR)/trace/libopencensus_trace_context_util.a \
+    $(OPENCENSUS_CPP_OPENCENSUS_LIBDIR)/context/libopencensus_context.a \
+    $(OPENCENSUS_CPP_OPENCENSUS_LIBDIR)/common/internal/libopencensus_common_random.a
+LIBS=$(CPP_NETLIB_LIBS) $(YAML_CPP_LIBS) $(PROMETHEUS_CPP_LIBS) $(OPENCENSUS_CPP_LIBS)
 
 sbindir=/opt/stackdriver/metadata/sbin
 INSTALL=/usr/bin/install
@@ -167,7 +237,7 @@ clean:
 	$(RM) metadatad $(OBJS)
 
 purge: clean
-	$(RM) -r init-submodules build-cpp-netlib build-yaml-cpp
+	$(RM) -r init-submodules build-cpp-netlib build-yaml-cpp build-prometheus-cpp build-opencensus-cpp
 	(cd .. && git submodule deinit -f $(SUBMODULE_DIRS:../%=%))
 
 init-submodules:
@@ -183,6 +253,8 @@ ifneq ($(findstring $(GIT_VERSION),2.7.0 2.7.1 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.8
 	    deps/*/.git
 	cd $(CPP_NETLIB_DIR) && $(SED_I) -e 's@/src/.git/@../../../../../../../.git/@' \
 	    libs/network/doc/_ext/breathe/.git
+	cd $(PROMETHEUS_CPP_DIR) && $(SED_I) -e 's@/src/.git/@../../../../.git/@' \
+	    3rdparty/*/.git
 endif
 	touch init-submodules
 
@@ -217,12 +289,40 @@ build-yaml-cpp: $(YAML_CPP_DIR)/Makefile
 	$(MAKE)
 	touch build-yaml-cpp
 
-cpp-netlib: $(CPP_NETLIB_LIBS)
+$(PROMETHEUS_CPP_DIR)/Makefile: init-submodules
+	cd $(PROMETHEUS_CPP_DIR) && \
+	$(CMAKE) -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS=-std=c++11 \
+	    -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ \
+	    -DENABLE_PUSH=OFF -DENABLE_PULL=OFF -DENABLE_COMPRESSION=OFF \
+	    -DENABLE_TESTING=OFF
+
+$(PROMETHEUS_CPP_LIBS): build-prometheus-cpp
 
+build-prometheus-cpp: $(PROMETHEUS_CPP_DIR)/Makefile
+	cd $(PROMETHEUS_CPP_DIR) && \
+	$(MAKE)
+	touch build-prometheus-cpp
+
+$(OPENCENSUS_CPP_DIR)/Makefile: init-submodules
+	cd $(OPENCENSUS_CPP_DIR) && \
+	$(CMAKE) -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS=-std=c++11 \
+	    -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ \
+	    -DBUILD_TESTING=OFF
+
+$(OPENCENSUS_CPP_LIBS): build-opencensus-cpp
+
+build-opencensus-cpp: $(OPENCENSUS_CPP_DIR)/Makefile
+	cd $(OPENCENSUS_CPP_DIR) && \
+	$(MAKE)
+	touch build-opencensus-cpp
+
+cpp-netlib: $(CPP_NETLIB_LIBS)
 yaml-cpp: $(YAML_CPP_LIBS)
+prometheus-cpp: $(PROMETHEUS_CPP_LIBS)
+opencensus-cpp: $(OPENCENSUS_CPP_LIBS)
 
-submodules: cpp-netlib yaml-cpp
+submodules: cpp-netlib yaml-cpp prometheus-cpp opencensus-cpp
 
 all: submodules metadatad
 
-.PHONY: all submodules cpp-netlib yaml-cpp purge clean install deb rpm
+.PHONY: all submodules cpp-netlib yaml-cpp prometheus-cpp opencensus-cpp purge clean install deb rpm

src/agent.cc

@@ -16,10 +16,11 @@
 
 #include "agent.h"
 
-#include "configuration.h"
 #include "api_server.h"
-#include "reporter.h"
+#include "configuration.h"
 #include "health_checker.h"
+#include "measures.h"
+#include "reporter.h"
 
 namespace google {
 
@@ -34,6 +35,7 @@ void MetadataAgent::Start() {
       config_.MetadataApiBindAddress(), config_.MetadataApiPort()));
   reporter_.reset(new MetadataReporter(
       config_, &store_, config_.MetadataReporterIntervalSeconds()));
+  ::google::RegisterAllViewsForExport();
 }
 
 void MetadataAgent::Stop() {

src/api_server.cc

@@ -21,6 +21,7 @@
 #include "configuration.h"
 #include "health_checker.h"
 #include "http_common.h"
+#include "internal/measures_utils.h"
 #include "logging.h"
 #include "store.h"
 
@@ -84,6 +85,11 @@ MetadataApiServer::MetadataApiServer(const Configuration& config,
              std::shared_ptr<HttpServer::connection> conn) {
              HandleHealthz(request, conn);
          }},
+        {{"GET", "/metrics"},
+         [=](const HttpServer::request& request,
+             std::shared_ptr<HttpServer::connection> conn) {
+             HandleMetrics(request, conn);
+         }},
       }, config_.VerboseLogging()),
       server_(
           HttpServer::options(dispatcher_)
@@ -200,4 +206,17 @@ void MetadataApiServer::HandleHealthz(
   }
 }
 
+void MetadataApiServer::HandleMetrics(
+    const HttpServer::request& request,
+    std::shared_ptr<HttpServer::connection> conn) {
+  std::string response = internal::SerializeMetricsToPrometheusTextFormat();
+  conn->set_status(HttpServer::connection::ok);
+  conn->set_headers(std::map<std::string, std::string>({
+    {"Connection", "close"},
+    {"Content-Length", std::to_string(response.size())},
+    {"Content-Type", "application/json"},
+  }));
+  conn->write(response);
 }
+
+} // namespace google

src/api_server.h

@@ -77,6 +77,8 @@ class MetadataApiServer {
                                std::shared_ptr<HttpServer::connection> conn);
   void HandleHealthz(const HttpServer::request& request,
                      std::shared_ptr<HttpServer::connection> conn);
+  void HandleMetrics(const HttpServer::request& request,
+                     std::shared_ptr<HttpServer::connection> conn);
 
   const Configuration& config_;
   const HealthChecker* health_checker_;

src/internal/measures_utils.cc

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2018 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "measures_utils.h"
+
+#include <opencensus/exporters/stats/prometheus/prometheus_exporter.h>
+#include <prometheus/text_serializer.h>
+
+namespace google {
+namespace internal {
+
+namespace {
+
+::prometheus::TextSerializer text_serializer;
+::opencensus::exporters::stats::PrometheusExporter exporter;
+
+} // namespace
+
+std::string SerializeMetricsToPrometheusTextFormat() {
+  return text_serializer.Serialize(exporter.Collect());
+}
+
+} // namespace internal
+} // namespace google

src/internal/measures_utils.h

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2018 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#ifndef METADATA_AGENT_INTERNAL_MEASURES_UTILS_H_
+#define METADATA_AGENT_INTERNAL_MEASURES_UTILS_H_
+
+#include <string>
+
+namespace google {
+namespace internal {
+
+std::string SerializeMetricsToPrometheusTextFormat();
+
+} // namespace internal
+} // namespace google
+
+#endif /* METADATA_AGENT_INTERNAL_MEASURES_UTILS_H_ */

src/measures.cc

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2018 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+#include "measures.h"
+
+#include <absl/strings/string_view.h>
+#include <opencensus/stats/stats.h>
+
+namespace google {
+
+namespace {
+
+constexpr char kUnitBytes[] = "By";
+constexpr char kCount[] = "1";
+
+} // namespace
+
+ABSL_CONST_INIT const absl::string_view
+    kGceApiRequestErrors =
+        "container.googleapis.com/internal/metadata_agent/gce_api_request_errors";
+
+::opencensus::stats::MeasureInt64 GceApiRequestErrors() {
+  static const auto measure =
+      ::opencensus::stats::MeasureInt64::Register(
+          kGceApiRequestErrors,
+          "Number of API request errors encountered.",
+          kCount);
+  return measure;
+}
+
+::opencensus::stats::TagKey MethodTagKey() {
+  static const auto method_tag_key =
+      ::opencensus::stats::TagKey::Register("method");
+  return method_tag_key;
+}
+
+const ::opencensus::stats::ViewDescriptor& GceApiRequestErrorsCumulative() {
+  const static ::opencensus::stats::ViewDescriptor descriptor =
+      ::opencensus::stats::ViewDescriptor()
+          .set_name("gce_api_request_errors")
+          .set_measure(kGceApiRequestErrors)
+          .set_aggregation(::opencensus::stats::Aggregation::Count())
+          .add_column(MethodTagKey());
+  return descriptor;
+}
+
+void RegisterAllViewsForExport() {
+  GceApiRequestErrorsCumulative().RegisterForExport();
+}
+
+} // namespace google