Skip to content

Commit 84976e0

Browse files
shyamradhakrishnanshyamradhakrishnan
authored
Fix reconcilers where error condition leads to infinite reconciliation (#316)
* Fix reconcilers where error condition leads to infinite reconciliation without exponential backoff
1 parent e7f3ec8 commit 84976e0

File tree

3 files changed

+23
-14
lines changed

3 files changed

+23
-14
lines changed

‎exp/controllers/ocimachinepool_controller.go

+8-6
Original file line numberDiff line numberDiff line change
@@ -295,7 +295,6 @@ func (r *OCIMachinePoolReconciler) reconcileNormal(ctx context.Context, logger l
295295
// https://docs.oracle.com/en-us/iaas/api/#/en/iaas/20160918/InstanceConfiguration/
296296
if err := machinePoolScope.ReconcileInstanceConfiguration(ctx); err != nil {
297297
r.Recorder.Eventf(machinePoolScope.OCIMachinePool, corev1.EventTypeWarning, "FailedLaunchTemplateReconcile", "Failed to reconcile launch template: %v", err)
298-
machinePoolScope.Error(err, "failed to reconcile launch template")
299298
return ctrl.Result{}, err
300299
}
301300

@@ -305,13 +304,15 @@ func (r *OCIMachinePoolReconciler) reconcileNormal(ctx context.Context, logger l
305304
// Find existing Instance Pool
306305
instancePool, err := machinePoolScope.FindInstancePool(ctx)
307306
if err != nil {
308-
conditions.MarkUnknown(machinePoolScope.OCIMachinePool, infrav2exp.InstancePoolReadyCondition, infrav2exp.InstancePoolNotFoundReason, err.Error())
307+
r.Recorder.Event(machinePoolScope.OCIMachinePool, corev1.EventTypeWarning, "ReconcileError", err.Error())
308+
conditions.MarkUnknown(machinePoolScope.OCIMachinePool, infrav2exp.InstancePoolReadyCondition, infrav2exp.InstancePoolNotFoundReason, "")
309309
return ctrl.Result{}, err
310310
}
311311

312312
if instancePool == nil {
313313
if _, err := machinePoolScope.CreateInstancePool(ctx); err != nil {
314-
conditions.MarkFalse(machinePoolScope.OCIMachinePool, infrav2exp.InstancePoolReadyCondition, infrav2exp.InstancePoolProvisionFailedReason, clusterv1.ConditionSeverityError, err.Error())
314+
r.Recorder.Event(machinePoolScope.OCIMachinePool, corev1.EventTypeWarning, "ReconcileError", err.Error())
315+
conditions.MarkFalse(machinePoolScope.OCIMachinePool, infrav2exp.InstancePoolReadyCondition, infrav2exp.InstancePoolProvisionFailedReason, clusterv1.ConditionSeverityError, "")
315316
return ctrl.Result{}, err
316317
}
317318
r.Recorder.Eventf(machinePoolScope.OCIMachinePool, corev1.EventTypeNormal, "InstancePoolCreated", "Created new Instance Pool: %s", machinePoolScope.OCIMachinePool.GetName())
@@ -398,7 +399,7 @@ func (r *OCIMachinePoolReconciler) reconcileDelete(ctx context.Context, machineP
398399
if instancePool == nil {
399400
machinePoolScope.OCIMachinePool.Status.Ready = false
400401
conditions.MarkFalse(machinePoolScope.OCIMachinePool, infrav2exp.InstancePoolReadyCondition, infrav2exp.InstancePoolNotFoundReason, clusterv1.ConditionSeverityWarning, "")
401-
machinePoolScope.Info("Instance Pool may already be deleted", "displayName", instancePool.DisplayName, "id", instancePool.Id)
402+
machinePoolScope.Info("Instance Pool may already be deleted")
402403
r.Recorder.Eventf(machinePoolScope.OCIMachinePool, corev1.EventTypeNormal, infrav2exp.InstancePoolNotFoundReason, "Unable to find matching instance pool")
403404
} else {
404405
switch instancePool.LifecycleState {
@@ -462,13 +463,14 @@ func (r *OCIMachinePoolReconciler) reconcileMachines(ctx context.Context, err er
462463
}
463464
err = cloudutil.CreateMachinePoolMachinesIfNotExists(ctx, params)
464465
if err != nil {
465-
conditions.MarkFalse(machinePoolScope.OCIMachinePool, clusterv1.ReadyCondition, "FailedToDeleteOrphanedMachines", clusterv1.ConditionSeverityWarning, err.Error())
466+
r.Recorder.Event(machinePoolScope.OCIMachinePool, corev1.EventTypeWarning, "FailedToCreateNewMachines", err.Error())
467+
conditions.MarkFalse(machinePoolScope.OCIMachinePool, clusterv1.ReadyCondition, "FailedToCreateNewMachines", clusterv1.ConditionSeverityWarning, "")
466468
return errors.Wrap(err, "failed to create missing machines")
467469
}
468470

469471
err = cloudutil.DeleteOrphanedMachinePoolMachines(ctx, params)
470472
if err != nil {
471-
conditions.MarkFalse(machinePoolScope.OCIMachinePool, clusterv1.ReadyCondition, "FailedToDeleteOrphanedMachines", clusterv1.ConditionSeverityWarning, err.Error())
473+
conditions.MarkFalse(machinePoolScope.OCIMachinePool, clusterv1.ReadyCondition, "FailedToDeleteOrphanedMachines", clusterv1.ConditionSeverityWarning, "")
472474
return errors.Wrap(err, "failed to delete orphaned machines")
473475
}
474476
return nil

‎exp/controllers/ocimanaged_machinepool_controller.go

+7-4
Original file line numberDiff line numberDiff line change
@@ -270,13 +270,15 @@ func (r *OCIManagedMachinePoolReconciler) reconcileNormal(ctx context.Context, l
270270
// Find existing Node Pool
271271
nodePool, err := machinePoolScope.FindNodePool(ctx)
272272
if err != nil {
273-
conditions.MarkUnknown(machinePoolScope.OCIManagedMachinePool, infrav2exp.NodePoolReadyCondition, infrav2exp.NodePoolNotFoundReason, err.Error())
273+
r.Recorder.Event(machinePoolScope.OCIManagedMachinePool, corev1.EventTypeWarning, "ReconcileError", err.Error())
274+
conditions.MarkUnknown(machinePoolScope.OCIManagedMachinePool, infrav2exp.NodePoolReadyCondition, infrav2exp.NodePoolNotFoundReason, "")
274275
return ctrl.Result{}, err
275276
}
276277

277278
if nodePool == nil {
278279
if nodePool, err = machinePoolScope.CreateNodePool(ctx); err != nil {
279-
conditions.MarkFalse(machinePoolScope.OCIManagedMachinePool, infrav2exp.NodePoolReadyCondition, infrav2exp.NodePoolProvisionFailedReason, clusterv1.ConditionSeverityError, err.Error())
280+
r.Recorder.Event(machinePoolScope.OCIManagedMachinePool, corev1.EventTypeWarning, "ReconcileError", err.Error())
281+
conditions.MarkFalse(machinePoolScope.OCIManagedMachinePool, infrav2exp.NodePoolReadyCondition, infrav2exp.NodePoolProvisionFailedReason, clusterv1.ConditionSeverityError, "")
280282
return ctrl.Result{}, err
281283
}
282284
// record the event only when node pool is created
@@ -391,13 +393,14 @@ func (r *OCIManagedMachinePoolReconciler) reconcileManagedMachines(ctx context.C
391393
}
392394
err = cloudutil.CreateMachinePoolMachinesIfNotExists(ctx, params)
393395
if err != nil {
394-
conditions.MarkFalse(machinePoolScope.OCIManagedMachinePool, clusterv1.ReadyCondition, "FailedToDeleteOrphanedMachines", clusterv1.ConditionSeverityWarning, err.Error())
396+
r.Recorder.Event(machinePoolScope.OCIManagedMachinePool, corev1.EventTypeWarning, "FailedToCreateNewMachines", err.Error())
397+
conditions.MarkFalse(machinePoolScope.OCIManagedMachinePool, clusterv1.ReadyCondition, "FailedToCreateNewMachines", clusterv1.ConditionSeverityWarning, "")
395398
return errors.Wrap(err, "failed to create missing machines")
396399
}
397400

398401
err = cloudutil.DeleteOrphanedMachinePoolMachines(ctx, params)
399402
if err != nil {
400-
conditions.MarkFalse(machinePoolScope.OCIManagedMachinePool, clusterv1.ReadyCondition, "FailedToDeleteOrphanedMachines", clusterv1.ConditionSeverityWarning, err.Error())
403+
conditions.MarkFalse(machinePoolScope.OCIManagedMachinePool, clusterv1.ReadyCondition, "FailedToDeleteOrphanedMachines", clusterv1.ConditionSeverityWarning, "")
401404
return errors.Wrap(err, "failed to delete orphaned machines")
402405
}
403406
return nil

‎exp/controllers/ocivirtual_machinepool_controller.go

+8-4
Original file line numberDiff line numberDiff line change
@@ -269,13 +269,15 @@ func (r *OCIVirtualMachinePoolReconciler) reconcileNormal(ctx context.Context, l
269269
// Find existing Virtual Node Pool
270270
nodePool, err := machinePoolScope.FindVirtualNodePool(ctx)
271271
if err != nil {
272-
conditions.MarkUnknown(machinePoolScope.OCIVirtualMachinePool, infrav2exp.VirtualNodePoolReadyCondition, infrav2exp.VirtualNodePoolNotFoundReason, err.Error())
272+
r.Recorder.Event(machinePoolScope.OCIVirtualMachinePool, corev1.EventTypeWarning, "ReconcileError", err.Error())
273+
conditions.MarkUnknown(machinePoolScope.OCIVirtualMachinePool, infrav2exp.VirtualNodePoolReadyCondition, infrav2exp.VirtualNodePoolNotFoundReason, "")
273274
return ctrl.Result{}, err
274275
}
275276

276277
if nodePool == nil {
277278
if nodePool, err = machinePoolScope.CreateVirtualNodePool(ctx); err != nil {
278-
conditions.MarkFalse(machinePoolScope.OCIVirtualMachinePool, infrav2exp.VirtualNodePoolReadyCondition, infrav2exp.VirtualNodePoolProvisionFailedReason, clusterv1.ConditionSeverityError, err.Error())
279+
r.Recorder.Event(machinePoolScope.OCIVirtualMachinePool, corev1.EventTypeWarning, "ReconcileError", err.Error())
280+
conditions.MarkFalse(machinePoolScope.OCIVirtualMachinePool, infrav2exp.VirtualNodePoolReadyCondition, infrav2exp.VirtualNodePoolProvisionFailedReason, clusterv1.ConditionSeverityError, "")
279281
return ctrl.Result{}, err
280282
}
281283
// record the event only when node pool is created
@@ -405,13 +407,15 @@ func (r *OCIVirtualMachinePoolReconciler) reconcileVirtualMachines(ctx context.C
405407
}
406408
err = cloudutil.CreateMachinePoolMachinesIfNotExists(ctx, params)
407409
if err != nil {
408-
conditions.MarkFalse(machinePoolScope.OCIVirtualMachinePool, clusterv1.ReadyCondition, "FailedToDeleteOrphanedMachines", clusterv1.ConditionSeverityWarning, err.Error())
410+
r.Recorder.Event(machinePoolScope.OCIVirtualMachinePool, corev1.EventTypeWarning, "FailedToCreateNewMachines", err.Error())
411+
conditions.MarkFalse(machinePoolScope.OCIVirtualMachinePool, clusterv1.ReadyCondition, "FailedToCreateNewMachines", clusterv1.ConditionSeverityWarning, "")
409412
return errors.Wrap(err, "failed to create missing machines")
410413
}
411414

412415
err = cloudutil.DeleteOrphanedMachinePoolMachines(ctx, params)
413416
if err != nil {
414-
conditions.MarkFalse(machinePoolScope.OCIVirtualMachinePool, clusterv1.ReadyCondition, "FailedToDeleteOrphanedMachines", clusterv1.ConditionSeverityWarning, err.Error())
417+
r.Recorder.Event(machinePoolScope.OCIVirtualMachinePool, corev1.EventTypeWarning, "FailedToDeleteOrphanedMachines", err.Error())
418+
conditions.MarkFalse(machinePoolScope.OCIVirtualMachinePool, clusterv1.ReadyCondition, "FailedToDeleteOrphanedMachines", clusterv1.ConditionSeverityWarning, "")
415419
return errors.Wrap(err, "failed to delete orphaned machines")
416420
}
417421
return nil

0 commit comments

Comments
 (0)