Skip to content

Commit 3ae3e57

Browse files
BugRogerBugRoger
committed
adds basic launchctl metrics
1 parent 1204d97 commit 3ae3e57

File tree

8 files changed

+78
-36
lines changed

8 files changed

+78
-36
lines changed

‎pkg/cmd/kubernikus/operator.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"github.com/golang/glog"
1313
"github.com/sapcc/kubernikus/pkg/cmd"
1414
"github.com/sapcc/kubernikus/pkg/controller"
15+
"github.com/sapcc/kubernikus/pkg/controller/metrics"
1516
"github.com/spf13/cobra"
1617
"github.com/spf13/pflag"
1718
)
@@ -97,7 +98,7 @@ func (o *Options) Run(c *cobra.Command) error {
9798
wg := &sync.WaitGroup{} // Goroutines can add themselves to this to be waited on
9899

99100
go controller.NewKubernikusOperator(&o.KubernikusOperatorOptions, logger).Run(stop, wg)
100-
go controller.ExposeMetrics(o.MetricPort, stop, wg)
101+
go metrics.ExposeMetrics(o.MetricPort, stop, wg)
101102

102103
<-sigs // Wait for signals (this hangs until a signal arrives)
103104
glog.Info("Shutting down...")

‎pkg/controller/base/instrumenting.go‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package base
2+
3+
import (
4+
"time"
5+
6+
"github.com/prometheus/client_golang/prometheus"
7+
"github.com/sapcc/kubernikus/pkg/apis/kubernikus/v1"
8+
)
9+
10+
type InstrumentingReconciler struct {
11+
Reconciler
12+
ReconciliationCount *prometheus.CounterVec
13+
ReconciliationLatency *prometheus.HistogramVec
14+
}
15+
16+
func (ir *InstrumentingReconciler) Reconcile(kluster *v1.Kluster) (requeue bool, err error) {
17+
defer func(begin time.Time) {
18+
ir.ReconciliationCount.With(
19+
prometheus.Labels{
20+
"kluster": kluster.Spec.Name,
21+
"project": kluster.Account(),
22+
}).Add(1)
23+
ir.ReconciliationLatency.With(
24+
prometheus.Labels{
25+
"kluster": kluster.Spec.Name,
26+
"project": kluster.Account(),
27+
}).Observe(time.Since(begin).Seconds())
28+
}(time.Now())
29+
return ir.Reconciler.Reconcile(kluster)
30+
}

‎pkg/controller/base/logging.go‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -19,18 +19,6 @@ type EventingReconciler struct {
1919
Reconciler
2020
}
2121

22-
type InstrumentedReconciler struct {
23-
Reconciler
24-
}
25-
26-
func (r *InstrumentedReconciler) Reconcile(kluster *v1.Kluster) (requeue bool, err error) {
27-
defer func() {
28-
RECONCILLIATION_COUNTER = RECONCILLIATION_COUNTER + 1
29-
fmt.Printf("Metrics: Reconciled %v kluster\n", RECONCILLIATION_COUNTER)
30-
}()
31-
return r.Reconciler.Reconcile(kluster)
32-
}
33-
3422
func (r *EventingReconciler) Reconcile(kluster *v1.Kluster) (requeue bool, err error) {
3523
fmt.Printf("EVENT: Reconciled %v\n", kluster.Name)
3624
return r.Reconciler.Reconcile(kluster)

‎pkg/controller/ground.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"github.com/sapcc/kubernikus/pkg/client/kubernetes"
2626
"github.com/sapcc/kubernikus/pkg/controller/config"
2727
"github.com/sapcc/kubernikus/pkg/controller/ground"
28+
"github.com/sapcc/kubernikus/pkg/controller/metrics"
2829
"github.com/sapcc/kubernikus/pkg/util"
2930
helm_util "github.com/sapcc/kubernikus/pkg/util/helm"
3031
waitutil "github.com/sapcc/kubernikus/pkg/util/wait"
@@ -136,8 +137,8 @@ func (op *GroundControl) handler(key string) error {
136137
} else {
137138
kluster := obj.(*v1.Kluster)
138139
glog.V(5).Infof("Handling kluster %v in phase %q", kluster.Name, kluster.Status.Phase)
139-
setMetricKlusterInfo(kluster.GetNamespace(),kluster.GetName(),kluster.Status.Version,kluster.Spec.Openstack.ProjectID,kluster.GetAnnotations(),kluster.GetLabels())
140-
setMetricKlusterStatusPhase(kluster.GetName(), kluster.Status.Phase)
140+
metrics.SetMetricKlusterInfo(kluster.GetNamespace(), kluster.GetName(), kluster.Status.Version, kluster.Spec.Openstack.ProjectID, kluster.GetAnnotations(), kluster.GetLabels())
141+
metrics.SetMetricKlusterStatusPhase(kluster.GetName(), kluster.Status.Phase)
141142

142143
switch phase := kluster.Status.Phase; phase {
143144
case models.KlusterPhasePending:

‎pkg/controller/launch/controller.go‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"github.com/sapcc/kubernikus/pkg/client/openstack"
77
"github.com/sapcc/kubernikus/pkg/controller/base"
88
"github.com/sapcc/kubernikus/pkg/controller/config"
9+
"github.com/sapcc/kubernikus/pkg/controller/metrics"
910
"github.com/sapcc/kubernikus/pkg/templates"
1011

1112
"github.com/go-kit/kit/log"
@@ -52,7 +53,11 @@ func NewController(factories config.Factories, clients config.Clients, recorder
5253
reconciler = &LaunchReconciler{clients, recorder, logger}
5354
reconciler = &base.LoggingReconciler{reconciler, logger}
5455
reconciler = &base.EventingReconciler{reconciler}
55-
reconciler = &base.InstrumentedReconciler{reconciler}
56+
reconciler = &base.InstrumentingReconciler{
57+
reconciler,
58+
metrics.KlusterReconcilicationCount,
59+
metrics.KlusterReconciliationLatency,
60+
}
5661

5762
return base.NewController(factories, clients, reconciler, logger)
5863
}

‎pkg/controller/metrics/launch.go‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
package metrics
2+
3+
import "github.com/prometheus/client_golang/prometheus"
4+
5+
var KlusterReconcilicationCount = prometheus.NewCounterVec(
6+
prometheus.CounterOpts{
7+
Namespace: "kubernikus",
8+
Subsystem: "launch",
9+
Name: "kluster_reconciliation_count",
10+
Help: "Number of reconcilitations."},
11+
[]string{"kluster", "project"})
12+
13+
var KlusterReconciliationLatency = prometheus.NewHistogramVec(
14+
prometheus.HistogramOpts{
15+
Namespace: "kubernikus",
16+
Subsystem: "launch",
17+
Name: "kluster_reconciliation_latency_microseconds",
18+
Help: "Total duration of reconciliation in microseconds.",
19+
},
20+
[]string{"kluster", "project"})

‎pkg/controller/metrics.go‎ renamed to ‎pkg/controller/metrics/metrics.go‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package controller
1+
package metrics
22

33
import (
44
"fmt"
@@ -28,7 +28,7 @@ var klusterInfo = prometheus.NewGaugeVec(
2828
Name: "kluster_info",
2929
Help: "detailed information on a kluster",
3030
},
31-
[]string{"kluster_namespace","kluster_name","kluster_version","creator","account","project_id"},
31+
[]string{"kluster_namespace", "kluster_name", "kluster_version", "creator", "account", "project_id"},
3232
)
3333

3434
var klusterStatusPhase = prometheus.NewGaugeVec(
@@ -58,14 +58,14 @@ var nodePoolStatus = prometheus.NewGaugeVec(
5858
[]string{"kluster_id", "node_pool", "status"},
5959
)
6060

61-
func setMetricKlusterInfo(namespace, name, version, projectID string, annotations, labels map[string]string) {
61+
func SetMetricKlusterInfo(namespace, name, version, projectID string, annotations, labels map[string]string) {
6262
promLabels := prometheus.Labels{
6363
"kluster_namespace": namespace,
64-
"kluster_name": name,
65-
"kluster_version": version,
66-
"creator": getCreatorFromAnnotations(annotations),
67-
"account": getAccountFromLabels(labels),
68-
"project_id": projectID,
64+
"kluster_name": name,
65+
"kluster_version": version,
66+
"creator": getCreatorFromAnnotations(annotations),
67+
"account": getAccountFromLabels(labels),
68+
"project_id": projectID,
6969
}
7070
klusterInfo.With(promLabels).Set(1)
7171
}
@@ -77,7 +77,7 @@ kubernikus_kluster_status_phase{"kluster_id"="<id>","phase"="running"} 0
7777
kubernikus_kluster_status_phase{"kluster_id"="<id>","phase"="pending"} 0
7878
kubernikus_kluster_status_phase{"kluster_id"="<id>","phase"="terminating"} 0
7979
*/
80-
func setMetricKlusterStatusPhase(klusterName string, klusterPhase models.KlusterPhase) {
80+
func SetMetricKlusterStatusPhase(klusterName string, klusterPhase models.KlusterPhase) {
8181
// Set current phase to 1, others to 0
8282
for _, phase := range klusterPhases {
8383
labels := prometheus.Labels{
@@ -147,6 +147,8 @@ func init() {
147147
klusterStatusPhase,
148148
nodePoolSize,
149149
nodePoolStatus,
150+
KlusterReconcilicationCount,
151+
KlusterReconciliationLatency,
150152
)
151153
}
152154

‎pkg/controller/metrics_test.go‎ renamed to ‎pkg/controller/metrics/metrics_test.go‎

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package controller
1+
package metrics
22

33
import (
44
"bytes"
@@ -17,28 +17,24 @@ import (
1717
func TestMetrics(t *testing.T) {
1818

1919
expectedMetrics := map[prometheus.Collector]string{
20-
nodePoolSize:
21-
`
20+
nodePoolSize: `
2221
# HELP kubernikus_node_pool_size size of a node pool
2322
# TYPE kubernikus_node_pool_size gauge
2423
kubernikus_node_pool_size{flavor_name="flavorName",image_name="imageName",kluster_id="klusterID",node_pool="nodePoolName"} 3
2524
`,
26-
nodePoolStatus:
27-
`
25+
nodePoolStatus: `
2826
# HELP kubernikus_node_pool_status status of the node pool and the number of nodes nodes in that status
2927
# TYPE kubernikus_node_pool_status gauge
3028
kubernikus_node_pool_status{kluster_id="klusterID",node_pool="nodePoolName",status="ready"} 2
3129
kubernikus_node_pool_status{kluster_id="klusterID",node_pool="nodePoolName",status="running"} 2
3230
kubernikus_node_pool_status{kluster_id="klusterID",node_pool="nodePoolName",status="starting"} 1
3331
`,
34-
klusterInfo:
35-
`
32+
klusterInfo: `
3633
# HELP kubernikus_kluster_info detailed information on a kluster
3734
# TYPE kubernikus_kluster_info gauge
3835
kubernikus_kluster_info{account="account",creator="D012345",kluster_name="klusterName",kluster_namespace="namespace",kluster_version="version",project_id="projectID"} 1
3936
`,
40-
klusterStatusPhase:
41-
`
37+
klusterStatusPhase: `
4238
# HELP kubernikus_kluster_status_phase the phase the kluster is currently in
4339
# TYPE kubernikus_kluster_status_phase gauge
4440
kubernikus_kluster_status_phase{kluster_id="klusterID",phase="Pending"} 0
@@ -51,10 +47,9 @@ kubernikus_kluster_status_phase{kluster_id="klusterID",phase="Terminating"} 0
5147
// call functions that update the metrics here
5248
setMetricNodePoolSize("klusterID", "nodePoolName", "imageName", "flavorName", 3)
5349
setMetricNodePoolStatus("klusterID", "nodePoolName", map[string]int64{"running": 2, "starting": 1, "ready": 2})
54-
setMetricKlusterInfo("namespace","klusterName","version","projectID",map[string]string{"creator":"D012345"},map[string]string{"account":"account"})
50+
setMetricKlusterInfo("namespace", "klusterName", "version", "projectID", map[string]string{"creator": "D012345"}, map[string]string{"account": "account"})
5551
setMetricKlusterStatusPhase("klusterID", models.KlusterPhaseRunning)
5652

57-
5853
registry := prometheus.NewPedanticRegistry()
5954
for collector, expectedMetricString := range expectedMetrics {
6055
// register the metric we're checking right now

0 commit comments

Comments
 (0)