fix: a race condition with election hijacking (#101)

* feat: Emit events when finalizers complete

* fix: a race condition with election hijacking

Author: ellistarn

go.mod

@@ -17,6 +17,7 @@ require (
 	k8s.io/apimachinery v0.31.2
 	k8s.io/client-go v0.31.2
 	k8s.io/klog/v2 v2.130.1
+	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8
 	sigs.k8s.io/controller-runtime v0.19.1
 	sigs.k8s.io/yaml v1.4.0
 )
@@ -71,7 +72,6 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiextensions-apiserver v0.31.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
-	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 )

leaderelection/leasehijacker.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"time"
 
 	"github.com/samber/lo"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -40,12 +41,21 @@ func LeaseHijacker(ctx context.Context, config *rest.Config, namespace string, n
 	if os.Getenv("HIJACK_LEASE") != "true" {
 		return nil // If not set, fallback to other controller-runtime lease settings
 	}
-	log.FromContext(ctx).Info("hijacking lease", "namespace", namespace, "name", name)
 	kubeClient := coordinationv1client.NewForConfigOrDie(config)
 	lease := lo.Must(kubeClient.Leases(namespace).Get(ctx, name, metav1.GetOptions{}))
+
+	untilElection := time.Until(lease.Spec.RenewTime.Add(time.Duration(*lease.Spec.LeaseDurationSeconds) * time.Second))
+
 	lease.Spec.HolderIdentity = lo.ToPtr(fmt.Sprintf("%s_%s", lo.Must(os.Hostname()), uuid.NewUUID()))
 	lease.Spec.AcquireTime = lo.ToPtr(metav1.NowMicro())
+	lease.Spec.RenewTime = lo.ToPtr(metav1.NowMicro())
+	*lease.Spec.LeaseDurationSeconds += 5 // Make our lease longer to guarantee we win the next election
+	*lease.Spec.LeaseTransitions += 1
 	lo.Must(kubeClient.Leases(namespace).Update(ctx, lease, metav1.UpdateOptions{}))
+
+	log.FromContext(ctx).Info(fmt.Sprintf("hijacked lease, waiting %s for election", untilElection), "namespace", namespace, "name", name)
+	time.Sleep(untilElection)
+
 	return lo.Must(resourcelock.New(
 		resourcelock.LeasesResourceLock,
 		namespace,