fix: handle pod termination during reconcile

Author: aramase

controllers/secretproviderclasspodstatus_controller.go

@@ -209,6 +209,22 @@ func (r *SecretProviderClassPodStatusReconciler) Reconcile(req ctrl.Request) (ct
 		return ctrl.Result{}, nil
 	}
 
+	// Obtain the full pod metadata. An object reference is needed for sending
+	// events and the UID is helpful for validating the SPCPS TargetPath.
+	pod := &v1.Pod{}
+	if err := r.reader.Get(ctx, client.ObjectKey{Namespace: req.Namespace, Name: spcPodStatus.Status.PodName}, pod); err != nil {
+		klog.ErrorS(err, "failed to get pod", "pod", klog.ObjectRef{Namespace: req.Namespace, Name: spcPodStatus.Status.PodName})
+		if apierrors.IsNotFound(err) {
+			return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
+		}
+		return ctrl.Result{}, err
+	}
+	// pod is being terminated so don't reconcile
+	if !pod.GetDeletionTimestamp().IsZero() {
+		klog.InfoS("pod is being terminated, skipping reconcile", "pod", klog.KObj(pod))
+		return ctrl.Result{}, nil
+	}
+
 	spcName := spcPodStatus.Status.SecretProviderClassName
 	spc := &v1alpha1.SecretProviderClass{}
 	if err := r.reader.Get(ctx, client.ObjectKey{Namespace: req.Namespace, Name: spcName}, spc); err != nil {
@@ -224,17 +240,6 @@ func (r *SecretProviderClassPodStatusReconciler) Reconcile(req ctrl.Request) (ct
 		return ctrl.Result{}, nil
 	}
 
-	// Obtain the full pod metadata. An object reference is needed for sending
-	// events and the UID is helpful for validating the SPCPS TargetPath.
-	pod := &v1.Pod{}
-	if err := r.reader.Get(ctx, client.ObjectKey{Namespace: req.Namespace, Name: spcPodStatus.Status.PodName}, pod); err != nil {
-		klog.ErrorS(err, "failed to get pod", "pod", klog.ObjectRef{Namespace: req.Namespace, Name: spcPodStatus.Status.PodName})
-		if apierrors.IsNotFound(err) {
-			return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
-		}
-		return ctrl.Result{}, err
-	}
-
 	// determine which pod volume this is associated with
 	podVol := k8sutil.SPCVolume(pod, spc.Name)
 	if podVol == nil {

pkg/rotation/reconciler.go

@@ -172,6 +172,20 @@ func (r *Reconciler) reconcile(ctx context.Context, spcps *v1alpha1.SecretProvid
 		r.reporter.reportRotationDuration(time.Since(begin).Seconds())
 	}()
 
+	// get pod from informer cache
+	podName, podNamespace := spcps.Status.PodName, spcps.Namespace
+	pod, err := r.store.GetPod(podName, podNamespace)
+	if err != nil {
+		errorReason = internalerrors.PodNotFound
+		return fmt.Errorf("failed to get pod %s/%s, err: %+v", podNamespace, podName, err)
+	}
+	// if the pod is being terminated, then skip rotation
+	// the spcps will be gc when the pod is deleted and will not show up in the next rotation cycle
+	if !pod.GetDeletionTimestamp().IsZero() {
+		klog.InfoS("pod is being terminated, skipping rotation", "pod", klog.KObj(pod))
+		return nil
+	}
+
 	spcName, spcNamespace := spcps.Status.SecretProviderClassName, spcps.Namespace
 
 	// get the secret provider class the pod status is referencing from informer cache
@@ -180,13 +194,6 @@ func (r *Reconciler) reconcile(ctx context.Context, spcps *v1alpha1.SecretProvid
 		errorReason = internalerrors.SecretProviderClassNotFound
 		return fmt.Errorf("failed to get secret provider class %s/%s, err: %+v", spcNamespace, spcName, err)
 	}
-	// get pod from informer cache
-	podName, podNamespace := spcps.Status.PodName, spcps.Namespace
-	pod, err := r.store.GetPod(podName, podNamespace)
-	if err != nil {
-		errorReason = internalerrors.PodNotFound
-		return fmt.Errorf("failed to get pod %s/%s, err: %+v", podNamespace, podName, err)
-	}
 
 	// determine which pod volume this is associated with
 	podVol := k8sutil.SPCVolume(pod, spc.Name)

pkg/rotation/reconciler_test.go

@@ -596,6 +596,17 @@ func TestReconcileNoError(t *testing.T) {
 	// 2 normal events - one for successfully updating the mounted contents and
 	// second for successfully rotating the K8s secret
 	g.Expect(len(fakeRecorder.Events)).To(BeNumerically("==", 2))
+
+	// test with pod being terminated
+	podToAdd.DeletionTimestamp = &metav1.Time{Time: time.Now()}
+	kubeClient = fake.NewSimpleClientset(podToAdd, secretToAdd)
+	testReconciler, err = newTestReconciler(scheme, kubeClient, crdClient, ctrlClient, 60*time.Second, socketPath)
+	g.Expect(err).NotTo(HaveOccurred())
+	err = testReconciler.store.Run(wait.NeverStop)
+	g.Expect(err).NotTo(HaveOccurred())
+
+	err = testReconciler.reconcile(context.TODO(), secretProviderClassPodStatusToProcess)
+	g.Expect(err).NotTo(HaveOccurred())
 }
 
 func TestPatchSecret(t *testing.T) {