Service monitor scrap tempo metrics for monolithic (#1274)

* Service monitor scrap tempo metrics for monolithic

Signed-off-by: Ruben Vargas <ruben.vp8510@gmail.com>

* Add mTLS to monolithic tempo

Signed-off-by: Ruben Vargas <ruben.vp8510@gmail.com>

* more work on cert rotation

Signed-off-by: Ruben Vargas <ruben.vp8510@gmail.com>

* secure gateway with mtls

Signed-off-by: Ruben Vargas <ruben.vp8510@gmail.com>

* fix tests

Signed-off-by: Ruben Vargas <ruben.vp8510@gmail.com>

* Update .chloggen/fix_monolithic_metrics.yaml

Co-authored-by: Andreas Gerstmayr <andreas@gerstmayr.me>

* fix service monitor for monolithic with mtls

Signed-off-by: Ruben Vargas <ruben.vp8510@gmail.com>

* fix test

Signed-off-by: Ruben Vargas <ruben.vp8510@gmail.com>

* address comments

Signed-off-by: Ruben Vargas <ruben.vp8510@gmail.com>

* fix openshift tests

Signed-off-by: Ruben Vargas <ruben.vp8510@gmail.com>

---------

Signed-off-by: Ruben Vargas <ruben.vp8510@gmail.com>
Co-authored-by: Andreas Gerstmayr <andreas@gerstmayr.me>

Author: rubenvp8510

.chloggen/fix_monolithic_metrics.yaml

@@ -0,0 +1,16 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: bug_fix
+
+# The name of the component, or a single word describing the area of concern, (e.g. tempostack, tempomonolithic, github action)
+component: tempomonolithic
+
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Scrape tempo metrics for monolithic.
+
+# One or more tracking issues related to the change
+issues: [1275]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:

cmd/start/main.go

@@ -60,6 +60,15 @@ func start(c *cobra.Command, args []string) {
 			setupLog.Error(err, "unable to create controller", "controller", "certrotation")
 			os.Exit(1)
 		}
+
+		if err = (&controllers.CertRotationMonolithicReconciler{
+			Client:       mgr.GetClient(),
+			Scheme:       mgr.GetScheme(),
+			FeatureGates: ctrlConfig.Gates,
+		}).SetupWithManager(mgr); err != nil {
+			setupLog.Error(err, "unable to create controller", "controller", "certrotationmonolithic")
+			os.Exit(1)
+		}
 	}
 
 	if err = (&controllers.TempoStackReconciler{

internal/certrotation/build.go

@@ -14,7 +14,7 @@ var defaultUserInfo = &user.DefaultInfo{Name: "system:tempostacks", Groups: []st
 
 // BuildAll builds all secrets and configmaps containing
 // CA certificates, CA bundles and client certificates for
-// a TempoStack.
+// a Tempo CR.
 func BuildAll(opts Options) ([]client.Object, error) {
 	res := make([]client.Object, 0)
 
@@ -40,7 +40,7 @@ func BuildAll(opts Options) ([]client.Object, error) {
 }
 
 // ApplyDefaultSettings merges the default options with the ones we give.
-func ApplyDefaultSettings(opts *Options, cfg configv1alpha1.BuiltInCertManagement) error {
+func ApplyDefaultSettings(opts *Options, cfg configv1alpha1.BuiltInCertManagement, components map[string]string) error {
 	rotation, err := ParseRotation(cfg)
 	if err != nil {
 		return err
@@ -55,11 +55,12 @@ func ApplyDefaultSettings(opts *Options, cfg configv1alpha1.BuiltInCertManagemen
 	if opts.Certificates == nil {
 		opts.Certificates = make(map[string]SelfSignedCertKey)
 	}
-	for service, name := range ComponentCertSecretNames(opts.StackName) {
+	for service, name := range components {
 		r := certificateRotation{
 			Clock:    clock,
 			UserInfo: defaultUserInfo,
 			Hostnames: []string{
+				"localhost",
 				fmt.Sprintf("%s.%s.svc.cluster.local", service, opts.StackNamespace),
 				fmt.Sprintf("%s.%s.svc", service, opts.StackNamespace),
 			},

internal/certrotation/build_test.go

@@ -29,7 +29,7 @@ func TestBuildAll(t *testing.T) {
 		StackName:      "dev",
 		StackNamespace: "ns",
 	}
-	err := ApplyDefaultSettings(&opts, cfg)
+	err := ApplyDefaultSettings(&opts, cfg, TempoStackComponentCertSecretNames(opts.StackName))
 	require.NoError(t, err)
 
 	objs, err := BuildAll(opts)
@@ -69,17 +69,18 @@ func TestApplyDefaultSettings_EmptySecrets(t *testing.T) {
 		StackNamespace: "ns",
 	}
 
-	err := ApplyDefaultSettings(&opts, cfg)
+	err := ApplyDefaultSettings(&opts, cfg, TempoStackComponentCertSecretNames(opts.StackName))
 	require.NoError(t, err)
 
-	cs := ComponentCertSecretNames(opts.StackName)
+	cs := TempoStackComponentCertSecretNames(opts.StackName)
 
 	for service, name := range cs {
 		cert, ok := opts.Certificates[name]
 		require.True(t, ok)
 		require.NotEmpty(t, cert.Rotation)
 
 		hostnames := []string{
+			"localhost",
 			fmt.Sprintf("%s.%s.svc.cluster.local", service, opts.StackNamespace),
 			fmt.Sprintf("%s.%s.svc", service, opts.StackNamespace),
 		}
@@ -114,7 +115,7 @@ func TestApplyDefaultSettings_ExistingSecrets(t *testing.T) {
 		Certificates:   ComponentCertificates{},
 	}
 
-	cs := ComponentCertSecretNames(opts.StackName)
+	cs := TempoStackComponentCertSecretNames(opts.StackName)
 
 	for _, name := range cs {
 		opts.Certificates[name] = SelfSignedCertKey{
@@ -131,7 +132,7 @@ func TestApplyDefaultSettings_ExistingSecrets(t *testing.T) {
 		}
 	}
 
-	err := ApplyDefaultSettings(&opts, cfg)
+	err := ApplyDefaultSettings(&opts, cfg, TempoStackComponentCertSecretNames(opts.StackName))
 	require.NoError(t, err)
 
 	for service, name := range cs {
@@ -140,6 +141,7 @@ func TestApplyDefaultSettings_ExistingSecrets(t *testing.T) {
 		require.NotEmpty(t, cert.Rotation)
 
 		hostnames := []string{
+			"localhost",
 			fmt.Sprintf("%s.%s.svc.cluster.local", service, opts.StackNamespace),
 			fmt.Sprintf("%s.%s.svc", service, opts.StackNamespace),
 		}

internal/certrotation/handlers/certrotation_discovery.go

@@ -14,10 +14,10 @@ import (
 
 const certRotationRequiredAtKey = "tempo.grafana.com/certRotationRequiredAt"
 
-// AnnotateForRequiredCertRotation adds/updates the `tempo.grafana.com/certRotationRequiredAt` annotation
+// AnnotateTempoStackForRequiredCertRotation adds/updates the `tempo.grafana.com/certRotationRequiredAt` annotation
 // to the named TempoStack if any of the managed client/serving/ca certificates expired. If no TempoStack
 // is found, then skip reconciliation.
-func AnnotateForRequiredCertRotation(ctx context.Context, k client.Client, name, namespace string) error {
+func AnnotateTempoStackForRequiredCertRotation(ctx context.Context, k client.Client, name, namespace string) error {
 	var s v1alpha1.TempoStack
 	key := client.ObjectKey{Name: name, Namespace: namespace}
 
@@ -43,3 +43,33 @@ func AnnotateForRequiredCertRotation(ctx context.Context, k client.Client, name,
 
 	return nil
 }
+
+// AnnotateMonolithicForRequiredCertRotation adds/updates the `tempo.grafana.com/certRotationRequiredAt` annotation
+// to the named TempoStack if any of the managed client/serving/ca certificates expired. If no TempoStack
+// is found, then skip reconciliation.
+func AnnotateMonolithicForRequiredCertRotation(ctx context.Context, k client.Client, name, namespace string) error {
+	var s v1alpha1.TempoMonolithic
+	key := client.ObjectKey{Name: name, Namespace: namespace}
+
+	if err := k.Get(ctx, key, &s); err != nil {
+		if apierrors.IsNotFound(err) {
+			// Do nothing
+			return nil
+		}
+
+		return kverrors.Wrap(err, "failed to get tempo TempoStack", "key", key)
+	}
+
+	ss := s.DeepCopy()
+	if ss.Annotations == nil {
+		ss.Annotations = make(map[string]string)
+	}
+
+	ss.Annotations[certRotationRequiredAtKey] = time.Now().UTC().Format(time.RFC3339)
+
+	if err := k.Update(ctx, ss); err != nil {
+		return kverrors.Wrap(err, fmt.Sprintf("failed to update tempo TempoStack `%s` annotation", certRotationRequiredAtKey), "key", key)
+	}
+
+	return nil
+}

internal/certrotation/handlers/check_cert_expiry.go

@@ -5,37 +5,26 @@ import (
 
 	"github.com/ViaQ/logerr/v2/kverrors"
 	"github.com/go-logr/logr"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	configv1alpha1 "github.com/grafana/tempo-operator/api/config/v1alpha1"
-	v1alpha1 "github.com/grafana/tempo-operator/api/tempo/v1alpha1"
 	"github.com/grafana/tempo-operator/internal/certrotation"
 )
 
 // CheckCertExpiry handles the case if the TempoStack managed signing CA, client and/or serving
 // certificates expired. Returns true if any of those expired and an error representing the reason
 // of expiry.
-func CheckCertExpiry(ctx context.Context, log logr.Logger, req ctrl.Request, k client.Client, fg configv1alpha1.FeatureGates) error {
-	ll := log.WithValues("tempostacks", req.String(), "event", "checkCertExpiry")
-
-	var stack v1alpha1.TempoStack
-	if err := k.Get(ctx, req.NamespacedName, &stack); err != nil {
-		if apierrors.IsNotFound(err) {
-			// maybe the user deleted it before we could react? Either way this isn't an issue
-			ll.Error(err, "could not find the requested tempo tempostacks", "name", req.String())
-			return nil
-		}
-		return kverrors.Wrap(err, "failed to lookup tempostacks", "name", req.String())
-	}
+func CheckCertExpiry(controllerName string, ctx context.Context, log logr.Logger, req ctrl.Request, k client.Client,
+	fg configv1alpha1.FeatureGates, components map[string]string) error {
+	ll := log.WithValues(controllerName, req.String(), "event", "checkCertExpiry")
 
-	opts, err := GetOptions(ctx, k, req)
+	opts, err := GetOptions(ctx, k, req, components)
 	if err != nil {
 		return kverrors.Wrap(err, "failed to lookup certificates secrets", "name", req.String())
 	}
 
-	if optErr := certrotation.ApplyDefaultSettings(&opts, fg.BuiltInCertManagement); optErr != nil {
+	if optErr := certrotation.ApplyDefaultSettings(&opts, fg.BuiltInCertManagement, components); optErr != nil {
 		ll.Error(optErr, "failed to conform options to build settings")
 		return optErr
 	}

internal/certrotation/handlers/options.go

@@ -14,7 +14,7 @@ import (
 
 // GetOptions return a certrotation options struct filled with all found client and serving certificate secrets if any found.
 // Return an error only if either the k8s client returns any other error except IsNotFound or if merging options fails.
-func GetOptions(ctx context.Context, k client.Client, req ctrl.Request) (certrotation.Options, error) {
+func GetOptions(ctx context.Context, k client.Client, req ctrl.Request, cs map[string]string) (certrotation.Options, error) {
 	name := certrotation.SigningCASecretName(req.Name)
 	ca, err := getSecret(ctx, k, name, req.Namespace)
 	if err != nil {
@@ -31,7 +31,7 @@ func GetOptions(ctx context.Context, k client.Client, req ctrl.Request) (certrot
 		}
 	}
 
-	certs, err := getCertificateOptions(ctx, k, req)
+	certs, err := getCertificateOptions(ctx, k, req, cs)
 	if err != nil {
 		return certrotation.Options{}, err
 	}
@@ -47,8 +47,7 @@ func GetOptions(ctx context.Context, k client.Client, req ctrl.Request) (certrot
 	}, nil
 }
 
-func getCertificateOptions(ctx context.Context, k client.Client, req ctrl.Request) (certrotation.ComponentCertificates, error) {
-	cs := certrotation.ComponentCertSecretNames(req.Name)
+func getCertificateOptions(ctx context.Context, k client.Client, req ctrl.Request, cs map[string]string) (certrotation.ComponentCertificates, error) {
 	certs := make(certrotation.ComponentCertificates, len(cs))
 
 	for _, name := range cs {

internal/certrotation/handlers/rotate_certs.go

@@ -23,7 +23,7 @@ import (
 // including the signing CA and a ca bundle or else returns an error. It returns only a degrade-condition-worthy
 // error if building the manifests fails for any reason.
 func CreateOrRotateCertificates(ctx context.Context, log logr.Logger,
-	req ctrl.Request, k client.Client, s *runtime.Scheme, fg configv1alpha1.FeatureGates) error {
+	req ctrl.Request, k client.Client, s *runtime.Scheme, fg configv1alpha1.FeatureGates, cs map[string]string) error {
 	ll := log.WithValues("tempostacks", req.String(), "event", "createOrRotateCerts")
 	var stack v1alpha1.TempoStack
 	if err := k.Get(ctx, req.NamespacedName, &stack); err != nil {
@@ -35,12 +35,12 @@ func CreateOrRotateCertificates(ctx context.Context, log logr.Logger,
 		return kverrors.Wrap(err, "failed to lookup tempostacks", "name", req.String())
 	}
 
-	opts, err := GetOptions(ctx, k, req)
+	opts, err := GetOptions(ctx, k, req, cs)
 	if err != nil {
 		return kverrors.Wrap(err, "failed to lookup certificates secrets", "name", req.String())
 	}
 
-	if optErr := certrotation.ApplyDefaultSettings(&opts, fg.BuiltInCertManagement); optErr != nil {
+	if optErr := certrotation.ApplyDefaultSettings(&opts, fg.BuiltInCertManagement, certrotation.TempoStackComponentCertSecretNames(opts.StackName)); optErr != nil {
 		ll.Error(optErr, "failed to conform options to build settings")
 		return kverrors.Wrap(err, "failed to conform options to build settings", "name", req.String())
 	}

internal/certrotation/target.go

@@ -51,7 +51,7 @@ func CertificatesExpired(opts Options) error {
 	return &CertExpiredError{Message: "certificates expired", Reasons: reasons}
 }
 
-// buildTargetCertKeyPairSecrets returns a slice of all rotated client and serving tempostacks certificates.
+// buildTargetCertKeyPairSecrets returns a slice of all rotated client and serving tempo certificates.
 func buildTargetCertKeyPairSecrets(opts Options) ([]client.Object, error) {
 	var (
 		res      = make([]client.Object, 0)

internal/certrotation/target_test.go

@@ -55,10 +55,10 @@ func TestCertificatesExpired(t *testing.T) {
 		},
 		RawCACerts: rawCA.Config.Certs,
 	}
-	err = ApplyDefaultSettings(&opts, cfg)
+	err = ApplyDefaultSettings(&opts, cfg, TempoStackComponentCertSecretNames(opts.StackName))
 	require.NoError(t, err)
 
-	for _, name := range ComponentCertSecretNames(stackName) {
+	for _, name := range TempoStackComponentCertSecretNames(stackName) {
 		cert := opts.Certificates[name]
 		cert.Secret = &corev1.Secret{
 			ObjectMeta: metav1.ObjectMeta{
@@ -106,7 +106,7 @@ func TestBuildTargetCertKeyPairSecrets_Create(t *testing.T) {
 		RawCACerts: rawCA.Config.Certs,
 	}
 
-	err := ApplyDefaultSettings(&opts, cfg)
+	err := ApplyDefaultSettings(&opts, cfg, TempoStackComponentCertSecretNames(opts.StackName))
 	require.NoError(t, err)
 
 	objs, err := buildTargetCertKeyPairSecrets(opts)
@@ -154,7 +154,7 @@ func TestBuildTargetCertKeyPairSecrets_Rotate(t *testing.T) {
 			},
 		},
 	}
-	err := ApplyDefaultSettings(&opts, cfg)
+	err := ApplyDefaultSettings(&opts, cfg, TempoStackComponentCertSecretNames(opts.StackName))
 	require.NoError(t, err)
 
 	objs, err := buildTargetCertKeyPairSecrets(opts)