fix: copy preflight results job fails with missing k0s binary (#2307)

* fix: copy preflight results job fails with missing k0s binary

* f

* f

* f

* f

* f

* f

Author: emosbaugh

e2e/scripts/common.sh

@@ -321,6 +321,15 @@ ensure_license_in_data_dir() {
     fi
 }
 
+ensure_copy_host_preflight_results_configmap() {
+    if ! kubectl get configmap -n embedded-cluster -l embedded-cluster/host-preflight-result --no-headers 2>/dev/null | grep -q host-preflight-results ; then
+        echo "ConfigMap with label embedded-cluster/host-preflight-result not found in embedded-cluster namespace"
+        kubectl get configmap -n embedded-cluster
+        return 1
+    fi
+    return 0
+}
+
 ensure_node_config() {
     if ! kubectl describe node | grep "controller-label" ; then
         echo "Failed to find controller-label"

e2e/scripts/single-node-airgap-install.sh

@@ -104,6 +104,10 @@ main() {
         if ! ensure_license_in_data_dir; then
             exit 1
         fi
+
+        if ! ensure_copy_host_preflight_results_configmap; then
+            exit 1
+        fi
     fi
 
     echo "kotsadm logs"

e2e/scripts/single-node-install.sh

@@ -162,6 +162,10 @@ main() {
         if ! ensure_license_in_data_dir; then
             exit 1
         fi
+
+        if ! ensure_copy_host_preflight_results_configmap; then
+            exit 1
+        fi
     fi
 
     echo "kotsadm logs"

operator/controllers/installation_controller.go

@@ -27,7 +27,7 @@ import (
 
 	apv1b2 "github.com/k0sproject/k0s/pkg/apis/autopilot/v1beta2"
 	k0shelm "github.com/k0sproject/k0s/pkg/apis/helm/v1beta1"
-	"github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1"
+	ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1"
 	"github.com/replicatedhq/embedded-cluster/operator/pkg/metrics"
 	"github.com/replicatedhq/embedded-cluster/operator/pkg/openebs"
 	"github.com/replicatedhq/embedded-cluster/operator/pkg/util"
@@ -67,8 +67,8 @@ var copyHostPreflightResultsJob = &batchv1.Job{
 		Namespace: ecNamespace,
 	},
 	Spec: batchv1.JobSpec{
-		BackoffLimit:            ptr.To[int32](2),
-		TTLSecondsAfterFinished: ptr.To[int32](0), // we don't want to keep the job around. Delete immediately after it finishes.
+		BackoffLimit:            ptr.To(int32(2)),
+		TTLSecondsAfterFinished: ptr.To(int32(1 * 60)), // we don't want to keep the job around. Delete it shortly after it finishes.
 		Template: corev1.PodTemplateSpec{
 			Spec: corev1.PodSpec{
 				ServiceAccountName: "embedded-cluster-operator",
@@ -77,16 +77,25 @@ var copyHostPreflightResultsJob = &batchv1.Job{
 						Name: "host",
 						VolumeSource: corev1.VolumeSource{
 							HostPath: &corev1.HostPathVolumeSource{
-								Path: v1beta1.DefaultDataDir,
-								Type: ptr.To[corev1.HostPathType]("Directory"),
+								Path: ecv1beta1.DefaultDataDir,
+								Type: ptr.To(corev1.HostPathDirectory),
+							},
+						},
+					},
+					{
+						Name: "k0s",
+						VolumeSource: corev1.VolumeSource{
+							HostPath: &corev1.HostPathVolumeSource{
+								Path: runtimeconfig.K0sBinaryPath,
+								Type: ptr.To(corev1.HostPathFile),
 							},
 						},
 					},
 				},
 				RestartPolicy: corev1.RestartPolicyNever,
 				Containers: []corev1.Container{
 					{
-						Name:  "embedded-cluster-updater",
+						Name:  "copy-host-preflight-results",
 						Image: "busybox:latest",
 						Command: []string{
 							"/bin/sh",
@@ -104,12 +113,23 @@ var copyHostPreflightResultsJob = &batchv1.Job{
 								"echo '/embedded-cluster/support/host-preflight-results.json does not exist'; " +
 								"fi",
 						},
+						Env: []corev1.EnvVar{
+							{
+								Name:  "KUBECONFIG",
+								Value: "", // make k0s kubectl not use admin.conf
+							},
+						},
 						VolumeMounts: []corev1.VolumeMount{
 							{
 								Name:      "host",
 								MountPath: "/embedded-cluster",
 								ReadOnly:  false,
 							},
+							{
+								Name:      "k0s",
+								MountPath: runtimeconfig.K0sBinaryPath,
+								ReadOnly:  true,
+							},
 						},
 					},
 				},
@@ -139,7 +159,7 @@ type InstallationReconciler struct {
 // NodeHasChanged returns true if the node configuration has changed when compared to
 // the node information we keep in the installation status. Returns a bool indicating
 // if a change was detected and a bool indicating if the node is new (not seen yet).
-func (r *InstallationReconciler) NodeHasChanged(in *v1beta1.Installation, ev metrics.NodeEvent) (bool, bool, error) {
+func (r *InstallationReconciler) NodeHasChanged(in *ecv1beta1.Installation, ev metrics.NodeEvent) (bool, bool, error) {
 	for _, nodeStatus := range in.Status.NodesStatus {
 		if nodeStatus.Name != ev.NodeName {
 			continue
@@ -154,7 +174,7 @@ func (r *InstallationReconciler) NodeHasChanged(in *v1beta1.Installation, ev met
 }
 
 // UpdateNodeStatus updates the node status in the Installation object status.
-func (r *InstallationReconciler) UpdateNodeStatus(in *v1beta1.Installation, ev metrics.NodeEvent) error {
+func (r *InstallationReconciler) UpdateNodeStatus(in *ecv1beta1.Installation, ev metrics.NodeEvent) error {
 	hash, err := ev.Hash()
 	if err != nil {
 		return err
@@ -166,15 +186,15 @@ func (r *InstallationReconciler) UpdateNodeStatus(in *v1beta1.Installation, ev m
 		in.Status.NodesStatus[i].Hash = hash
 		return nil
 	}
-	in.Status.NodesStatus = append(in.Status.NodesStatus, v1beta1.NodeStatus{Name: ev.NodeName, Hash: hash})
+	in.Status.NodesStatus = append(in.Status.NodesStatus, ecv1beta1.NodeStatus{Name: ev.NodeName, Hash: hash})
 	return nil
 }
 
 // ReconcileNodeStatuses reconciles the node statuses in the Installation object status. Installation
 // is not updated remotely but only in the memory representation of the object (aka caller must save
 // the object after the call). This function returns a batch of events that need to be sent back to
 // the metrics endpoint, these events represent changes in the node statuses.
-func (r *InstallationReconciler) ReconcileNodeStatuses(ctx context.Context, in *v1beta1.Installation) (*NodeEventsBatch, error) {
+func (r *InstallationReconciler) ReconcileNodeStatuses(ctx context.Context, in *ecv1beta1.Installation) (*NodeEventsBatch, error) {
 	var nodes corev1.NodeList
 	if err := r.List(ctx, &nodes); err != nil {
 		return nil, fmt.Errorf("failed to list nodes: %w", err)
@@ -206,7 +226,7 @@ func (r *InstallationReconciler) ReconcileNodeStatuses(ctx context.Context, in *
 		r.Recorder.Eventf(in, corev1.EventTypeNormal, "NodeUpdated", "Node %s has been updated", node.Name)
 		batch.NodesUpdated = append(batch.NodesUpdated, event)
 	}
-	trimmed := []v1beta1.NodeStatus{}
+	trimmed := []ecv1beta1.NodeStatus{}
 	for _, nodeStatus := range in.Status.NodesStatus {
 		if _, ok := seen[nodeStatus.Name]; ok {
 			trimmed = append(trimmed, nodeStatus)
@@ -224,7 +244,7 @@ func (r *InstallationReconciler) ReconcileNodeStatuses(ctx context.Context, in *
 }
 
 // ReportNodesChanges reports node changes to the metrics endpoint.
-func (r *InstallationReconciler) ReportNodesChanges(ctx context.Context, in *v1beta1.Installation, batch *NodeEventsBatch) {
+func (r *InstallationReconciler) ReportNodesChanges(ctx context.Context, in *ecv1beta1.Installation, batch *NodeEventsBatch) {
 	for _, ev := range batch.NodesAdded {
 		if err := metrics.NotifyNodeAdded(ctx, in.Spec.MetricsBaseURL, ev); err != nil {
 			ctrl.LoggerFrom(ctx).Error(err, "failed to notify node added")
@@ -242,7 +262,7 @@ func (r *InstallationReconciler) ReportNodesChanges(ctx context.Context, in *v1b
 	}
 }
 
-func (r *InstallationReconciler) ReconcileOpenebs(ctx context.Context, in *v1beta1.Installation) error {
+func (r *InstallationReconciler) ReconcileOpenebs(ctx context.Context, in *ecv1beta1.Installation) error {
 	log := ctrl.LoggerFrom(ctx)
 
 	err := openebs.CleanupStatefulPods(ctx, r.Client)
@@ -261,8 +281,8 @@ func (r *InstallationReconciler) ReconcileOpenebs(ctx context.Context, in *v1bet
 // status of the newest one is coherent with whole cluster status. Returns the newest
 // installation object.
 func (r *InstallationReconciler) CoalesceInstallations(
-	ctx context.Context, items []v1beta1.Installation,
-) *v1beta1.Installation {
+	ctx context.Context, items []ecv1beta1.Installation,
+) *ecv1beta1.Installation {
 	sort.SliceStable(items, func(i, j int) bool {
 		return items[j].Name < items[i].Name
 	})
@@ -281,7 +301,7 @@ func (r *InstallationReconciler) CoalesceInstallations(
 
 // ReadClusterConfigSpecFromSecret reads the cluster config from the secret pointed by spec.ConfigSecret
 // if it is set. This overrides the default configuration from spec.Config.
-func (r *InstallationReconciler) ReadClusterConfigSpecFromSecret(ctx context.Context, in *v1beta1.Installation) error {
+func (r *InstallationReconciler) ReadClusterConfigSpecFromSecret(ctx context.Context, in *ecv1beta1.Installation) error {
 	if in.Spec.ConfigSecret == nil {
 		return nil
 	}
@@ -298,7 +318,7 @@ func (r *InstallationReconciler) ReadClusterConfigSpecFromSecret(ctx context.Con
 
 // CopyHostPreflightResultsFromNodes copies the preflight results from any new node that is added to the cluster
 // A job is scheduled on the new node and the results copied from a host path
-func (r *InstallationReconciler) CopyHostPreflightResultsFromNodes(ctx context.Context, in *v1beta1.Installation, events *NodeEventsBatch) error {
+func (r *InstallationReconciler) CopyHostPreflightResultsFromNodes(ctx context.Context, in *ecv1beta1.Installation, events *NodeEventsBatch) error {
 	log := ctrl.LoggerFrom(ctx)
 
 	if len(events.NodesAdded) == 0 {
@@ -328,7 +348,7 @@ func (r *InstallationReconciler) CopyHostPreflightResultsFromNodes(ctx context.C
 	return nil
 }
 
-func constructHostPreflightResultsJob(rc runtimeconfig.RuntimeConfig, in *v1beta1.Installation, nodeName string) *batchv1.Job {
+func constructHostPreflightResultsJob(rc runtimeconfig.RuntimeConfig, in *ecv1beta1.Installation, nodeName string) *batchv1.Job {
 	labels := map[string]string{
 		"embedded-cluster/node-name":    nodeName,
 		"embedded-cluster/installation": in.Name,
@@ -370,9 +390,9 @@ func (r *InstallationReconciler) Reconcile(ctx context.Context, req ctrl.Request
 	if err != nil {
 		return ctrl.Result{}, fmt.Errorf("failed to list installations: %w", err)
 	}
-	var items []v1beta1.Installation
+	var items []ecv1beta1.Installation
 	for _, in := range installs {
-		if in.Status.State == v1beta1.InstallationStateObsolete {
+		if in.Status.State == ecv1beta1.InstallationStateObsolete {
 			continue
 		}
 		items = append(items, in)
@@ -472,7 +492,7 @@ func (r *InstallationReconciler) reconcileHostCABundle(ctx context.Context) erro
 // SetupWithManager sets up the controller with the Manager.
 func (r *InstallationReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewControllerManagedBy(mgr).
-		For(&v1beta1.Installation{}).
+		For(&ecv1beta1.Installation{}).
 		Watches(&corev1.Node{}, &handler.EnqueueRequestForObject{}).
 		Watches(&apv1b2.Plan{}, &handler.EnqueueRequestForObject{}).
 		Watches(&k0shelm.Chart{}, &handler.EnqueueRequestForObject{}).

operator/controllers/installation_controller_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/go-logr/logr"
 	"github.com/go-logr/logr/testr"
 	"github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1"
+	ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1"
 	"github.com/replicatedhq/embedded-cluster/pkg/runtimeconfig"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -29,7 +30,7 @@ import (
 )
 
 func TestInstallationReconciler_constructCreateCMCommand(t *testing.T) {
-	in := &v1beta1.Installation{
+	in := &ecv1beta1.Installation{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: "install-name",
 		},
@@ -39,24 +40,28 @@ func TestInstallationReconciler_constructCreateCMCommand(t *testing.T) {
 			},
 		},
 	}
-	rc := runtimeconfig.New(nil)
+
+	rc := runtimeconfig.New(in.Spec.RuntimeConfig)
+
 	job := constructHostPreflightResultsJob(rc, in, "my-node")
-	require.Len(t, job.Spec.Template.Spec.Volumes, 1)
+
+	require.Len(t, job.Spec.Template.Spec.Volumes, 2)
+	require.Equal(t, "host", job.Spec.Template.Spec.Volumes[0].Name)
 	require.Equal(t, "/var/lib/embedded-cluster", job.Spec.Template.Spec.Volumes[0].VolumeSource.HostPath.Path)
 	require.Len(t, job.Spec.Template.Spec.Containers, 1)
 	require.Len(t, job.Spec.Template.Spec.Containers[0].Command, 4)
 	kctlCmd := job.Spec.Template.Spec.Containers[0].Command[3]
 	expected := "if [ -f /embedded-cluster/support/host-preflight-results.json ]; then /embedded-cluster/bin/kubectl create configmap ${HSPF_CM_NAME} --from-file=results.json=/embedded-cluster/support/host-preflight-results.json -n embedded-cluster --dry-run=client -oyaml | /embedded-cluster/bin/kubectl label -f - embedded-cluster/host-preflight-result=${EC_NODE_NAME} --local -o yaml | /embedded-cluster/bin/kubectl apply -f - && /embedded-cluster/bin/kubectl annotate configmap ${HSPF_CM_NAME} \"update-timestamp=$(date +'%Y-%m-%dT%H:%M:%SZ')\" --overwrite; else echo '/embedded-cluster/support/host-preflight-results.json does not exist'; fi"
 	assert.Equal(t, expected, kctlCmd)
-	require.Len(t, job.Spec.Template.Spec.Containers[0].Env, 2)
+	require.Len(t, job.Spec.Template.Spec.Containers[0].Env, 3)
 	assert.Equal(t, v1.EnvVar{
 		Name:  "EC_NODE_NAME",
 		Value: "my-node",
-	}, job.Spec.Template.Spec.Containers[0].Env[0])
+	}, job.Spec.Template.Spec.Containers[0].Env[1])
 	assert.Equal(t, v1.EnvVar{
 		Name:  "HSPF_CM_NAME",
 		Value: "my-node-host-preflight-results",
-	}, job.Spec.Template.Spec.Containers[0].Env[1])
+	}, job.Spec.Template.Spec.Containers[0].Env[2])
 }
 
 func TestInstallationReconciler_reconcileHostCABundle(t *testing.T) {