Drop default cluster profile, cleanup instance creation profiles and logic (#93)

neoaggelos · web-flow · commit 4e446e9231e9 · 2025-05-26T22:58:17.000+03:00
---------

Signed-off-by: Angelos Kolaitis &lt;neoaggelos@gmail.com&gt;
diff --git a/api/v1alpha2/condition_consts.go b/api/v1alpha2/condition_consts.go
@@ -23,25 +23,6 @@ const (
 	LoadBalancerProvisioningAbortedReason = "LoadBalancerProvisioningAbortedReason"
 )
 
-const (
-	// KubeadmProfileAvailableCondition documents the availability of the default kubeadm LXC profile.
-	KubeadmProfileAvailableCondition = "KubeadmProfileAvailable"
-
-	// KubeadmProfileDisabledReason (Severity=Info) documents a LXCCluster controller detecting that the
-	// LXCCluster spec requests that no default kubeam profile be created.
-	KubeadmProfileDisabledReason = "KubeadmProfileDisabled"
-
-	// KubeadmProfileCreationFailedReason (Severity=Warning) documents a LXCCluster controller detecting
-	// a retriable error while provisioning the default kubeadm LXC profile; those kind of errors are
-	// usually transient and failed provisioning are automatically re-tried by the controller.'
-	KubeadmProfileCreationFailedReason = "KubeadmProfileCreationFailed"
-
-	// KubeadmProfileCreationAbortedReason (Severity=Error) documents a LXCCluster controller detecting
-	// an unrecoverable error while provisioning the default kubeadm LXC profile. This is usually because
-	// of permission issues on the server, therefore requires user intervention.
-	KubeadmProfileCreationAbortedReason = "KubeadmProfileCreationAborted"
-)
-
 // Conditions and condition Reasons for the LXCMachine object.
 
 const (
diff --git a/api/v1alpha2/lxccluster_types.go b/api/v1alpha2/lxccluster_types.go
@@ -51,15 +51,14 @@ type LXCClusterSpec struct {
 	// +optional
 	Unprivileged bool `json:"unprivileged"`
 
-	// Skip creation of the default kubeadm profile "cluster-api-$namespace-$name"
-	// for LXCClusters.
+	// Do not apply the default kubeadm profile on container instances.
 	//
 	// In this case, the cluster administrator is responsible to create the
 	// profile manually and set the `.spec.template.spec.profiles` field of all
 	// LXCMachineTemplate objects.
 	//
-	// This is useful in cases where a restricted project is used, which does not
-	// allow privileged containers.
+	// For more details on the default kubeadm profile that is applied, see
+	// https://lxc.github.io/cluster-api-provider-incus/reference/profile/kubeadm.html
 	//
 	// +optional
 	SkipDefaultKubeadmProfile bool `json:"skipDefaultKubeadmProfile"`
@@ -262,11 +261,6 @@ func (c *LXCCluster) GetLoadBalancerInstanceName() string {
 	return fmt.Sprintf("%s-%s-lb", c.Name, hex.EncodeToString(hash[:3])[:5])
 }
 
-// GetProfileName returns the profile name for the cluster LXC machines.
-func (c *LXCCluster) GetProfileName() string {
-	return fmt.Sprintf("cluster-api-%s-%s", c.Namespace, c.Name)
-}
-
 // +kubebuilder:object:root=true
 
 // LXCClusterList contains a list of LXCCluster.
diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_lxcclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_lxcclusters.yaml
@@ -241,15 +241,14 @@ spec:
                 type: object
               skipDefaultKubeadmProfile:
                 description: |-
-                  Skip creation of the default kubeadm profile "cluster-api-$namespace-$name"
-                  for LXCClusters.
+                  Do not apply the default kubeadm profile on container instances.
 
                   In this case, the cluster administrator is responsible to create the
                   profile manually and set the `.spec.template.spec.profiles` field of all
                   LXCMachineTemplate objects.
 
-                  This is useful in cases where a restricted project is used, which does not
-                  allow privileged containers.
+                  For more details on the default kubeadm profile that is applied, see
+                  https://lxc.github.io/cluster-api-provider-incus/reference/profile/kubeadm.html
                 type: boolean
               unprivileged:
                 description: |-
diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_lxcclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_lxcclustertemplates.yaml
@@ -264,15 +264,14 @@ spec:
                         type: object
                       skipDefaultKubeadmProfile:
                         description: |-
-                          Skip creation of the default kubeadm profile "cluster-api-$namespace-$name"
-                          for LXCClusters.
+                          Do not apply the default kubeadm profile on container instances.
 
                           In this case, the cluster administrator is responsible to create the
                           profile manually and set the `.spec.template.spec.profiles` field of all
                           LXCMachineTemplate objects.
 
-                          This is useful in cases where a restricted project is used, which does not
-                          allow privileged containers.
+                          For more details on the default kubeadm profile that is applied, see
+                          https://lxc.github.io/cluster-api-provider-incus/reference/profile/kubeadm.html
                         type: boolean
                       unprivileged:
                         description: |-
diff --git a/docs/book/src/reference/api/v1alpha2/api.md b/docs/book/src/reference/api/v1alpha2/api.md
@@ -105,13 +105,12 @@ bool
 </td>
 <td>
 <em>(Optional)</em>
-<p>Skip creation of the default kubeadm profile &ldquo;cluster-api-$namespace-$name&rdquo;
-for LXCClusters.</p>
+<p>Do not apply the default kubeadm profile on container instances.</p>
 <p>In this case, the cluster administrator is responsible to create the
 profile manually and set the <code>.spec.template.spec.profiles</code> field of all
 LXCMachineTemplate objects.</p>
-<p>This is useful in cases where a restricted project is used, which does not
-allow privileged containers.</p>
+<p>For more details on the default kubeadm profile that is applied, see
+<a href="https://lxc.github.io/cluster-api-provider-incus/reference/profile/kubeadm.html">https://lxc.github.io/cluster-api-provider-incus/reference/profile/kubeadm.html</a></p>
 </td>
 </tr>
 </table>
@@ -297,13 +296,12 @@ bool
 </td>
 <td>
 <em>(Optional)</em>
-<p>Skip creation of the default kubeadm profile &ldquo;cluster-api-$namespace-$name&rdquo;
-for LXCClusters.</p>
+<p>Do not apply the default kubeadm profile on container instances.</p>
 <p>In this case, the cluster administrator is responsible to create the
 profile manually and set the <code>.spec.template.spec.profiles</code> field of all
 LXCMachineTemplate objects.</p>
-<p>This is useful in cases where a restricted project is used, which does not
-allow privileged containers.</p>
+<p>For more details on the default kubeadm profile that is applied, see
+<a href="https://lxc.github.io/cluster-api-provider-incus/reference/profile/kubeadm.html">https://lxc.github.io/cluster-api-provider-incus/reference/profile/kubeadm.html</a></p>
 </td>
 </tr>
 </tbody>
@@ -532,13 +530,12 @@ bool
 </td>
 <td>
 <em>(Optional)</em>
-<p>Skip creation of the default kubeadm profile &ldquo;cluster-api-$namespace-$name&rdquo;
-for LXCClusters.</p>
+<p>Do not apply the default kubeadm profile on container instances.</p>
 <p>In this case, the cluster administrator is responsible to create the
 profile manually and set the <code>.spec.template.spec.profiles</code> field of all
 LXCMachineTemplate objects.</p>
-<p>This is useful in cases where a restricted project is used, which does not
-allow privileged containers.</p>
+<p>For more details on the default kubeadm profile that is applied, see
+<a href="https://lxc.github.io/cluster-api-provider-incus/reference/profile/kubeadm.html">https://lxc.github.io/cluster-api-provider-incus/reference/profile/kubeadm.html</a></p>
 </td>
 </tr>
 </table>
diff --git a/docs/book/src/reference/profile/kubeadm.md b/docs/book/src/reference/profile/kubeadm.md
@@ -17,7 +17,18 @@ When using unprivileged containers, the following profile is applied instead:
 
 ```yaml
 # incus profile create kubeadm-unprivileged
-# curl https://lxc.github.io/cluster-api-provider-incus/static/v0.1/profile.yaml | incus profile edit kubeadm-unprivileged
+# curl https://lxc.github.io/cluster-api-provider-incus/static/v0.1/unprivileged.yaml | incus profile edit kubeadm-unprivileged
 
 {{#include ../../static/v0.1/unprivileged.yaml }}
 ```
+
+## Unprivileged containers (Canonical LXD)
+
+When using unprivileged containers with Canonical LXD, it is also required to enable `security.nesting` and disable apparmor:
+
+```bash
+# lxc profile create kubeadm-unprivileged
+# curl https://lxc.github.io/cluster-api-provider-incus/static/v0.1/unprivileged-lxd.yaml | lxc profile edit kubeadm-unprivileged
+
+{{#include ../../static/v0.1/unprivileged-lxd.yaml }}
+```
diff --git a/docs/book/src/static/v0.1/unprivileged-lxd.yaml b/docs/book/src/static/v0.1/unprivileged-lxd.yaml
@@ -0,0 +1 @@
+../../../../../internal/static/embed/unprivileged-lxd.yaml
diff --git a/internal/controller/lxccluster/controller_delete.go b/internal/controller/lxccluster/controller_delete.go
@@ -29,7 +29,6 @@ func (r *LXCClusterReconciler) reconcileDelete(ctx context.Context, cluster *clu
 		return ctrl.Result{}, err
 	}
 	conditions.MarkFalse(lxcCluster, infrav1.LoadBalancerAvailableCondition, clusterv1.DeletingReason, clusterv1.ConditionSeverityInfo, "")
-	conditions.MarkFalse(lxcCluster, infrav1.KubeadmProfileAvailableCondition, clusterv1.DeletingReason, clusterv1.ConditionSeverityInfo, "")
 	if err := patchLXCCluster(ctx, patchHelper, lxcCluster); err != nil {
 		return ctrl.Result{}, fmt.Errorf("failed to patch LXCCluster: %w", err)
 	}
@@ -48,11 +47,6 @@ func (r *LXCClusterReconciler) reconcileDelete(ctx context.Context, cluster *clu
 		return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
 	}
 
-	log.FromContext(ctx).Info("Deleting default kubeadm profile")
-	if err := lxcClient.DeleteProfile(lxcCluster.GetProfileName()); err != nil {
-		return ctrl.Result{}, fmt.Errorf("failed to delete the default kubeadm profile: %w", err)
-	}
-
 	// Cluster is deleted so remove the finalizer.
 	controllerutil.RemoveFinalizer(lxcCluster, infrav1.ClusterFinalizer)
 
diff --git a/internal/controller/lxccluster/controller_normal.go b/internal/controller/lxccluster/controller_normal.go
@@ -2,54 +2,18 @@ package lxccluster
 
 import (
 	"context"
-	"strings"
 
-	"github.com/lxc/incus/v6/shared/api"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	"sigs.k8s.io/cluster-api/util/conditions"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 
 	infrav1 "github.com/lxc/cluster-api-provider-incus/api/v1alpha2"
 	"github.com/lxc/cluster-api-provider-incus/internal/loadbalancer"
 	"github.com/lxc/cluster-api-provider-incus/internal/lxc"
-	"github.com/lxc/cluster-api-provider-incus/internal/static"
 	"github.com/lxc/cluster-api-provider-incus/internal/utils"
 )
 
 func (r *LXCClusterReconciler) reconcileNormal(ctx context.Context, cluster *clusterv1.Cluster, lxcCluster *infrav1.LXCCluster, lxcClient *lxc.Client) error {
-	// Create the default kubeadm profile for LXC containers
-	profileName := lxcCluster.GetProfileName()
-	if lxcCluster.Spec.SkipDefaultKubeadmProfile {
-		// only log the message once, before the condition is set.
-		if !conditions.Has(lxcCluster, infrav1.KubeadmProfileAvailableCondition) {
-			log.FromContext(ctx).Info("Skipping kubeadm profile creation")
-		}
-		conditions.MarkTrue(lxcCluster, infrav1.KubeadmProfileAvailableCondition)
-	} else {
-		log := log.FromContext(ctx).WithValues("profileName", profileName)
-
-		if _, _, err := lxcClient.GetProfile(profileName); err != nil {
-			if !strings.Contains(err.Error(), "Profile not found") {
-				conditions.MarkFalse(lxcCluster, infrav1.KubeadmProfileAvailableCondition, infrav1.KubeadmProfileCreationFailedReason, clusterv1.ConditionSeverityWarning, "failed to check profile %q status: %s", profileName, err)
-				return err
-			}
-
-			log.Info("Creating default kubeadm profile for cluster")
-			if err := lxcClient.CreateProfile(api.ProfilesPost{Name: profileName, ProfilePut: static.DefaultKubeadmProfile(!lxcCluster.Spec.Unprivileged)}); err != nil {
-				log.Error(err, "Failed to create default kubeadm profile")
-
-				if strings.Contains(err.Error(), "Privileged containers are forbidden") {
-					conditions.MarkFalse(lxcCluster, infrav1.KubeadmProfileAvailableCondition, infrav1.KubeadmProfileCreationAbortedReason, clusterv1.ConditionSeverityError, "The default kubeadm LXC profile could not be created, most likely because of a permissions issue. Either enable privileged containers on the project, or specify .spec.skipDefaultKubeadmProfile=true on the LXCCluster object. The error was: %s", err)
-					return nil
-				}
-				conditions.MarkFalse(lxcCluster, infrav1.KubeadmProfileAvailableCondition, infrav1.KubeadmProfileCreationFailedReason, clusterv1.ConditionSeverityWarning, "%s", err)
-				return err
-			}
-		}
-
-		conditions.MarkTrue(lxcCluster, infrav1.KubeadmProfileAvailableCondition)
-	}
-
 	// Create the container hosting the load balancer.
 	log.FromContext(ctx).Info("Creating load balancer")
 	lbIPs, err := loadbalancer.ManagerForCluster(cluster, lxcCluster, lxcClient).Create(ctx)
diff --git a/internal/controller/lxccluster/controller_util.go b/internal/controller/lxccluster/controller_util.go
@@ -13,7 +13,6 @@ import (
 
 func patchLXCCluster(ctx context.Context, patchHelper *patch.Helper, lxcCluster *infrav1.LXCCluster) error {
 	infraConditions := []clusterv1.ConditionType{
-		infrav1.KubeadmProfileAvailableCondition,
 		infrav1.LoadBalancerAvailableCondition,
 	}
 	hasInfraConditionError := false
diff --git a/internal/controller/lxcmachine/controller_normal.go b/internal/controller/lxcmachine/controller_normal.go
@@ -3,7 +3,7 @@ package lxcmachine
 import (
 	"context"
 	"fmt"
-	"slices"
+	"maps"
 	"strings"
 	"time"
 
@@ -117,26 +117,14 @@ func launchInstance(ctx context.Context, cluster *clusterv1.Cluster, lxcCluster
 		instanceType = lxcMachine.Spec.InstanceType
 	}
 
-	ctx = log.IntoContext(ctx, log.FromContext(ctx).WithValues("instance", name, "role", role))
-
-	profiles := lxcMachine.Spec.Profiles
-	if lxcMachine.Spec.InstanceType == lxc.Container && !lxcCluster.Spec.SkipDefaultKubeadmProfile && !slices.Contains(lxcMachine.Spec.Profiles, lxcCluster.GetProfileName()) {
-		// for containers, include the default kubeadm profile
-		profiles = append(lxcMachine.Spec.Profiles, lxcCluster.GetProfileName())
-	}
-
 	// Parse device configurations
-	var devices map[string]map[string]string
+	devices := map[string]map[string]string{}
 	for _, deviceSpec := range lxcMachine.Spec.Devices {
 		deviceName, deviceArgs, hasSeparator := strings.Cut(deviceSpec, ",")
 		if !hasSeparator {
 			return nil, utils.TerminalError(fmt.Errorf("device spec %q is not using the expected %q format", deviceSpec, "<device>,<key>=<value>,<key2>=<value2>"))
 		}
 
-		if devices == nil {
-			devices = map[string]map[string]string{}
-		}
-
 		if _, ok := devices[deviceName]; !ok {
 			devices[deviceName] = map[string]string{}
 		}
@@ -190,42 +178,33 @@ func launchInstance(ctx context.Context, cluster *clusterv1.Cluster, lxcCluster
 		}
 	}
 
-	config := map[string]string{
-		"user.cluster-name":      cluster.Name,
-		"user.cluster-namespace": cluster.Namespace,
-		"user.machine-name":      machine.Name,
-		"user.cluster-role":      role,
-		"cloud-init.user-data":   cloudInit,
-	}
-
-	if lxcClient.GetServerName(ctx) == "lxd" && lxcCluster.Spec.Unprivileged && instanceType == lxc.Container {
-		config["security.nesting"] = "true"
-		if devices == nil {
-			devices = make(map[string]map[string]string, 2)
-		}
-		devices["00-disable-snapd"] = map[string]string{
-			"type":   "disk",
-			"source": "/dev/null",
-			"path":   "/usr/lib/systemd/system/snapd.service",
-		}
-		devices["00-disable-apparmor"] = map[string]string{
-			"type":   "disk",
-			"source": "/dev/null",
-			"path":   "/usr/lib/systemd/system/apparmor.service",
-		}
-	}
-
-	return lxcClient.WaitForLaunchInstance(ctx, api.InstancesPost{
+	instance := api.InstancesPost{
 		Name:         name,
 		Type:         api.InstanceType(instanceType),
 		Source:       image,
 		InstanceType: lxcMachine.Spec.Flavor,
 		InstancePut: api.InstancePut{
-			Profiles: profiles,
+			Profiles: lxcMachine.Spec.Profiles,
 			Devices:  devices,
-			Config:   util.MergeMap(config, lxcMachine.Spec.Config),
+			Config: util.MergeMap(map[string]string{
+				"user.cluster-name":      cluster.Name,
+				"user.cluster-namespace": cluster.Namespace,
+				"user.machine-name":      machine.Name,
+				"user.cluster-role":      role,
+				"cloud-init.user-data":   cloudInit,
+			}, lxcMachine.Spec.Config),
 		},
-	}, defaultTemplateFiles)
+	}
+
+	// apply profile for Kubernetes to run in LXC containers
+	if instanceType == lxc.Container && !lxcCluster.Spec.SkipDefaultKubeadmProfile {
+		profile := static.DefaultKubeadmProfile(!lxcCluster.Spec.Unprivileged, lxcClient.GetServerName(ctx))
+
+		maps.Copy(instance.Devices, profile.Devices)
+		maps.Copy(instance.Config, profile.Config)
+	}
+
+	return lxcClient.WaitForLaunchInstance(ctx, instance, defaultTemplateFiles)
 }
 
 // defaultTemplateFiles that are injected to LXCMachine instances.
diff --git a/internal/lxc/client_instances.go b/internal/lxc/client_instances.go
@@ -72,7 +72,7 @@ func (c *Client) WaitForStartInstance(ctx context.Context, name string) ([]strin
 
 	if state.Status == "Running" {
 		log.V(2).Info("Instance is already running")
-	} else if err := c.WaitForOperation(ctx, "UpdateInstanceState(Start)", func() (incus.Operation, error) {
+	} else if err := c.WaitForOperation(ctx, "StartInstance", func() (incus.Operation, error) {
 		log.V(2).Info("Starting instance")
 		return c.UpdateInstanceState(name, api.InstanceStatePut{Action: "start"}, "")
 	}); err != nil {
@@ -99,7 +99,7 @@ func (c *Client) WaitForStopInstance(ctx context.Context, name string) error {
 		return nil
 	}
 	log.V(2).Info("Stopping instance")
-	return c.WaitForOperation(ctx, "UpdateInstanceState(Stop)", func() (incus.Operation, error) {
+	return c.WaitForOperation(ctx, "StopInstance", func() (incus.Operation, error) {
 		return c.UpdateInstanceState(name, api.InstanceStatePut{Action: "stop", Force: true}, "")
 	})
 }
diff --git a/internal/static/embed/unprivileged-lxd.yaml b/internal/static/embed/unprivileged-lxd.yaml
@@ -0,0 +1,18 @@
+description: Profile for cluster-api-provider-incus unprivileged nodes (LXD)
+config:
+  linux.kernel_modules: ip_vs,ip_vs_rr,ip_vs_wrr,ip_vs_sh,ip_tables,ip6_tables,iptable_raw,netlink_diag,nf_nat,overlay,br_netfilter,xt_socket
+  security.nesting: "true"
+devices:
+  kubeadm-host-boot:
+    path: /usr/lib/ostree-boot
+    readonly: "true"
+    source: /boot
+    type: disk
+  00-disable-snapd:
+    type: disk
+    source: /dev/null
+    path: /usr/lib/systemd/system/snapd.service
+  00-disable-apparmor:
+    type: disk
+    source: /dev/null
+    path: /usr/lib/systemd/system/apparmor.service
diff --git a/internal/static/profiles.go b/internal/static/profiles.go

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+../../../../../internal/static/embed/unprivileged-lxd.yaml`
Original file line number	Diff line number	Diff line change
`@@ -29,7 +29,6 @@ func (r LXCClusterReconciler) reconcileDelete(ctx context.Context, cluster clu`
`29`	`29`	`return ctrl.Result{}, err`
`30`	`30`	`}`
`31`	`31`	`conditions.MarkFalse(lxcCluster, infrav1.LoadBalancerAvailableCondition, clusterv1.DeletingReason, clusterv1.ConditionSeverityInfo, "")`
`32`		`- conditions.MarkFalse(lxcCluster, infrav1.KubeadmProfileAvailableCondition, clusterv1.DeletingReason, clusterv1.ConditionSeverityInfo, "")`
`33`	`32`	`if err := patchLXCCluster(ctx, patchHelper, lxcCluster); err != nil {`
`34`	`33`	`return ctrl.Result{}, fmt.Errorf("failed to patch LXCCluster: %w", err)`
`35`	`34`	`}`
`@@ -48,11 +47,6 @@ func (r LXCClusterReconciler) reconcileDelete(ctx context.Context, cluster clu`
`48`	`47`	`return ctrl.Result{RequeueAfter: 5 * time.Second}, nil`
`49`	`48`	`}`
`50`	`49`
`51`		`- log.FromContext(ctx).Info("Deleting default kubeadm profile")`
`52`		`- if err := lxcClient.DeleteProfile(lxcCluster.GetProfileName()); err != nil {`
`53`		`- return ctrl.Result{}, fmt.Errorf("failed to delete the default kubeadm profile: %w", err)`
`54`		`- }`
`55`		`-`
`56`	`50`	`// Cluster is deleted so remove the finalizer.`
`57`	`51`	`controllerutil.RemoveFinalizer(lxcCluster, infrav1.ClusterFinalizer)`
`58`	`52`
Original file line number	Diff line number	Diff line change
`@@ -13,7 +13,6 @@ import (`
`13`	`13`
`14`	`14`	`func patchLXCCluster(ctx context.Context, patchHelper patch.Helper, lxcCluster infrav1.LXCCluster) error {`
`15`	`15`	`infraConditions := []clusterv1.ConditionType{`
`16`		`- infrav1.KubeadmProfileAvailableCondition,`
`17`	`16`	`infrav1.LoadBalancerAvailableCondition,`
`18`	`17`	`}`
`19`	`18`	`hasInfraConditionError := false`