feat: Preflight check opt-out (#1156)

**What problem does this PR solve?**:
Provides a way for the client to opt out of preflight checks for a
specific Cluster by using an annotation:

Examples

Opt out of two specific checks:
```
preflight.cluster.caren.nutanix.com/opt-out: "CheckOne, checktwo"
```

(Note that both case and whitespace is ignored)

Opt out of all checks:

```
preflight.cluster.caren.nutanix.com/opt-out: all
```

When a pre-flight check returns a false negative due to a bug, an issue
in some external API, or some unforseen circumstance, the client needs a
way to opt out.

**Which issue(s) this PR fixes**:
Fixes #

**How Has This Been Tested?**:
<!--
Please describe the tests that you ran to verify your changes.
Provide output from the tests and any manual steps needed to replicate
the tests.
-->

**Special notes for your reviewer**:
<!--
Use this to provide any additional information to the reviewers.
This may include:
- Best way to review the PR.
- Where the author wants the most review attention on.
- etc.
-->

Author: dlipovetsky

pkg/webhook/preflight/preflight.go

@@ -15,6 +15,8 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
+
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/webhook/preflight/skip"
 )
 
 type (
@@ -93,7 +95,16 @@ func (h *WebhookHandler) Handle(ctx context.Context, req admission.Request) admi
 		return admission.Allowed("")
 	}
 
-	resultsOrderedByCheckerAndCheck := run(ctx, h.client, cluster, h.checkers)
+	skipEvaluator := skip.New(cluster)
+
+	if skipEvaluator.ForAll() {
+		// If the cluster has skipped all checks, return allowed.
+		return admission.Allowed("").WithWarnings(
+			"Cluster has skipped all preflight checks",
+		)
+	}
+
+	resultsOrderedByCheckerAndCheck := run(ctx, h.client, cluster, skipEvaluator, h.checkers)
 
 	// Summarize the results.
 	resp := admission.Response{
@@ -145,23 +156,49 @@ func (h *WebhookHandler) Handle(ctx context.Context, req admission.Request) admi
 func run(ctx context.Context,
 	client ctrlclient.Client,
 	cluster *clusterv1.Cluster,
+	skipEvaluator *skip.Evaluator,
 	checkers []Checker,
 ) [][]namedResult {
 	resultsOrderedByCheckerAndCheck := make([][]namedResult, len(checkers))
 
 	checkersWG := sync.WaitGroup{}
 	for i, checker := range checkers {
 		checkersWG.Add(1)
-		go func(ctx context.Context, client ctrlclient.Client, cluster *clusterv1.Cluster, checker Checker, i int) {
+		go func(
+			ctx context.Context,
+			client ctrlclient.Client,
+			cluster *clusterv1.Cluster,
+			skipEvaluator *skip.Evaluator,
+			checker Checker,
+			i int,
+		) {
 			defer checkersWG.Done()
 
 			checks := checker.Init(ctx, client, cluster)
 			resultsOrderedByCheck := make([]namedResult, len(checks))
 
 			checksWG := sync.WaitGroup{}
 			for j, check := range checks {
+				if skipEvaluator.For(check.Name()) {
+					resultsOrderedByCheck[j] = namedResult{
+						Name: check.Name(),
+						CheckResult: CheckResult{
+							Allowed: true,
+							Error:   false,
+							Causes:  nil,
+							Warnings: []string{
+								fmt.Sprintf("Cluster has skipped preflight check %q", check.Name()),
+							},
+						},
+					}
+					continue
+				}
 				checksWG.Add(1)
-				go func(ctx context.Context, check Check, j int) {
+				go func(
+					ctx context.Context,
+					check Check,
+					j int,
+				) {
 					defer checksWG.Done()
 					defer func() {
 						if r := recover(); r != nil {
@@ -200,6 +237,7 @@ func run(ctx context.Context,
 			ctx,
 			client,
 			cluster,
+			skipEvaluator,
 			checker,
 			i,
 		)