implement azuretracing metrics

Signed-off-by: Markus Blaschke <mblaschke82@gmail.com>

Author: mblaschke

README.md

@@ -94,6 +94,24 @@ available on `/metrics`
 | `azure_loganalytics_query_results`          | Number of results from query                                                   |
 | `azure_loganalytics_query_requests`         | Count of requests (eg paged subqueries) per query                              |
 
+### Azuretracing metrics
+
+(with 22.2.0 and later)
+
+Azuretracing metrics collects latency and latency from azure-sdk-for-go and creates metrics and is controllable using
+environment variables (eg. setting buckets, disabling metrics or disable autoreset).
+
+| Metric                                   | Description                                                                            |
+|------------------------------------------|----------------------------------------------------------------------------------------|
+| `azurerm_api_ratelimit`                  | Azure ratelimit metrics (only on /metrics, resets after query due to limited validity) |
+| `azurerm_api_request_*`                  | Azure request count and latency as histogram                                           |
+
+| Environment variable                     | Example                          | Description                                              |
+|------------------------------------------|----------------------------------|----------------------------------------------------------|
+| `METRIC_AZURERM_API_REQUEST_BUCKETS`     | `1, 2.5, 5, 10, 30, 60, 90, 120` | Sets buckets for `azurerm_api_request` histogram metric  |
+| `METRIC_AZURERM_API_REQUEST_DISABLE`     | `false`                          | Disables `azurerm_api_request_*` metric                  |
+| `METRIC_AZURERM_API_RATELIMIT_DISABLE`   | `false`                          | Disables `azurerm_api_ratelimit` metric                  |
+| `METRIC_AZURERM_API_RATELIMIT_AUTORESET` | `false`                          | Disables `azurerm_api_ratelimit` autoreset after fetch   |
 
 ## Examples
 

go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/sirupsen/logrus v1.8.1
 	github.com/webdevops/azure-resourcegraph-exporter v0.0.0-20220211155142-96a70b1d912f
+	github.com/webdevops/go-prometheus-common v0.0.0-20220211101814-4c9913788fe1
 )
 
 require (

go.sum

@@ -231,6 +231,7 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/webdevops/azure-resourcegraph-exporter v0.0.0-20220211155142-96a70b1d912f h1:Hbtrj8gZOSPzywrok7vxakGiM9cloDiwZApEjyNQ9Js=
 github.com/webdevops/azure-resourcegraph-exporter v0.0.0-20220211155142-96a70b1d912f/go.mod h1:7/oZePL6o+quZtWhtM97XjTh2JCEAYSPkpkx9wmNz2s=
+github.com/webdevops/go-prometheus-common v0.0.0-20220211101814-4c9913788fe1 h1:wIf6O43jGEarp8ojgInXRIt6jHeQ94JUvsy4O3wqKHY=
 github.com/webdevops/go-prometheus-common v0.0.0-20220211101814-4c9913788fe1/go.mod h1:naEkgDRh6L+Ef0qzaS9Q60m1GkwN5Mu6xaNd/Ha84Sg=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

loganalytics/prober.go

@@ -15,6 +15,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/webdevops/azure-loganalytics-exporter/config"
 	"github.com/webdevops/azure-resourcegraph-exporter/kusto"
+	"github.com/webdevops/go-prometheus-common/azuretracing"
 	"net/http"
 	"strconv"
 	"time"
@@ -28,6 +29,7 @@ type (
 	LogAnalyticsProber struct {
 		QueryConfig kusto.Config
 		Conf        config.Opts
+		UserAgent   string
 
 		Azure struct {
 			Environment          azure.Environment
@@ -133,8 +135,8 @@ func (p *LogAnalyticsProber) AddWorkspaces(workspace ...string) {
 func (p *LogAnalyticsProber) LogAnalyticsQueryClient() operationalinsights.QueryClient {
 	// Create and authorize a operationalinsights client
 	client := operationalinsights.NewQueryClientWithBaseURI(p.Azure.Environment.ResourceIdentifiers.OperationalInsights + OperationInsightsWorkspaceUrlSuffix)
+	p.decorateAzureAutoRest(&client.Client)
 	client.Authorizer = p.Azure.OpInsightsAuthorizer
-	client.ResponseInspector = p.respondDecorator(nil)
 	return client
 }
 
@@ -346,14 +348,6 @@ func (p *LogAnalyticsProber) sendQueryToWorkspace(logger *log.Entry, workspaceId
 	}
 }
 
-func (p *LogAnalyticsProber) respondDecorator(subscriptionId *string) autorest.RespondDecorator {
-	return func(p autorest.Responder) autorest.Responder {
-		return autorest.ResponderFunc(func(r *http.Response) error {
-			return nil
-		})
-	}
-}
-
 func (p *LogAnalyticsProber) parseCacheTime(r *http.Request) (time.Duration, error) {
 	durationString := r.URL.Query().Get("cache")
 	if durationString != "" {
@@ -379,3 +373,11 @@ func (p *LogAnalyticsProber) NewSizedWaitGroup() sizedwaitgroup.SizedWaitGroup {
 
 	return sizedwaitgroup.New(size)
 }
+
+func (p *LogAnalyticsProber) decorateAzureAutoRest(client *autorest.Client) {
+	client.Authorizer = p.Azure.AzureAuthorizer
+	if err := client.AddToUserAgent(p.UserAgent); err != nil {
+		log.Panic(err)
+	}
+	azuretracing.DecoreAzureAutoRest(client)
+}

loganalytics/servicediscovery.go

@@ -21,8 +21,7 @@ func (sd *LogAnalyticsServiceDiscovery) ResourcesClient(subscriptionId string) *
 	prober := sd.prober
 
 	client := operationalinsightsProfile.NewWorkspacesClientWithBaseURI(prober.Azure.Environment.ResourceManagerEndpoint, subscriptionId)
-	client.Authorizer = prober.Azure.AzureAuthorizer
-	client.ResponseInspector = prober.respondDecorator(&subscriptionId)
+	prober.decorateAzureAutoRest(&client.Client)
 
 	return &client
 }

main.go

@@ -12,6 +12,7 @@ import (
 	"github.com/webdevops/azure-loganalytics-exporter/config"
 	"github.com/webdevops/azure-loganalytics-exporter/loganalytics"
 	"github.com/webdevops/azure-resourcegraph-exporter/kusto"
+	"github.com/webdevops/go-prometheus-common/azuretracing"
 	"net/http"
 	"os"
 	"path"
@@ -22,6 +23,8 @@ import (
 
 const (
 	Author = "webdevops.io"
+
+	UserAgent = "azure-loganalytics-exporter/"
 )
 
 var (
@@ -139,7 +142,14 @@ func initAzureConnection() {
 
 // start and handle prometheus handler
 func startHttpServer() {
-	http.Handle("/metrics", promhttp.Handler())
+	// healthz
+	http.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
+		if _, err := fmt.Fprint(w, "Ok"); err != nil {
+			log.Error(err)
+		}
+	})
+
+	http.Handle("/metrics", azuretracing.RegisterAzureMetricAutoClean(promhttp.Handler()))
 
 	http.HandleFunc("/probe", handleProbeRequest)
 	http.HandleFunc("/probe/workspace", handleProbeWorkspace)

probe.go

@@ -75,6 +75,7 @@ func NewLogAnalyticsProber(w http.ResponseWriter, r *http.Request) *loganalytics
 	prober := loganalytics.NewLogAnalyticsProber(w, r)
 	prober.QueryConfig = Config
 	prober.Conf = opts
+	prober.UserAgent = UserAgent + gitTag
 	prober.Azure.AzureAuthorizer = AzureAuthorizer
 	prober.Azure.OpInsightsAuthorizer = OpInsightsAuthorizer
 	prober.Azure.Environment = AzureEnvironment