runtime/kubernetes: Support specifying additional capabilities for containers.

Also make sure that when a container is marked privileged it is allowed to run as root.

Author: nichtverstehen

internal/frontend/cuefrontendopaque/container.go

@@ -23,6 +23,7 @@ type cueContainer struct {
 
 	Requests *schema.Container_ResourceLimits `json:"resourceRequests"`
 	Limits   *schema.Container_ResourceLimits `json:"resourceLimits"`
+	Security *cueServerSecurity               `json:"security,omitempty"`
 }
 
 type parsedCueContainer struct {
@@ -53,6 +54,18 @@ func parseCueContainer(ctx context.Context, env *schema.Environment, pl parsing.
 		},
 	}
 
+	if bits.Security != nil {
+		if err := parsing.RequireFeature(loc.Module, "experimental/container/security"); err != nil {
+			return nil, fnerrors.AttachLocation(loc, err)
+		}
+
+		out.container.Security = &schema.Container_Security{
+			Privileged:   bits.Security.Privileged,
+			HostNetwork:  bits.Security.HostNetwork,
+			Capabilities: bits.Security.Capabilities,
+		}
+	}
+
 	if mounts := v.LookupPath("mounts"); mounts.Exists() {
 		var err error
 		out.container.Mount, out.volumes, err = cuefrontend.ParseMounts(ctx, pl, loc, mounts)

internal/frontend/cuefrontendopaque/phase0.go

@@ -26,7 +26,7 @@ var packageFields = []string{
 }
 
 var sidecarFields = []string{
-	"args", "env", "mounts", "image", "imageFrom", "init",
+	"args", "env", "mounts", "image", "imageFrom", "init", "security",
 }
 
 type Frontend struct {

internal/frontend/cuefrontendopaque/server.go

@@ -79,8 +79,9 @@ type cuePermissions struct {
 }
 
 type cueServerSecurity struct {
-	Privileged  bool `json:"privileged"`
-	HostNetwork bool `json:"hostNetwork"`
+	Privileged   bool     `json:"privileged"`
+	HostNetwork  bool     `json:"hostNetwork"`
+	Capabilities []string `json:"capabilities"`
 }
 
 // TODO: converge the relevant parts with parseCueContainer.
@@ -347,8 +348,9 @@ func parseServerExtension(ctx context.Context, env *schema.Environment, pl parsi
 		}
 
 		out.MainContainer.Security = &schema.Container_Security{
-			Privileged:  bits.Security.Privileged,
-			HostNetwork: bits.Security.HostNetwork,
+			Privileged:   bits.Security.Privileged,
+			HostNetwork:  bits.Security.HostNetwork,
+			Capabilities: bits.Security.Capabilities,
 		}
 	}
 

internal/planning/deploy/deploy.go

@@ -935,6 +935,7 @@ func PrepareRunOpts(ctx context.Context, stack *planning.Stack, srv planning.Pla
 	out.MainContainer.Args = append(out.MainContainer.Args, main.Args...)
 	out.MainContainer.Privileged = main.GetSecurity().GetPrivileged()
 	out.MainContainer.HostNetwork = main.GetSecurity().GetHostNetwork()
+	out.MainContainer.Capabilities = main.GetSecurity().GetCapabilities()
 	out.MainContainer.ResourceLimits = main.Limits
 	out.MainContainer.ResourceRequests = main.Requests
 	out.MainContainer.ContainerPorts = append(out.MainContainer.ContainerPorts, main.ContainerPort...)
@@ -984,6 +985,7 @@ func prepareContainerRunOpts(containers []*schema.Container, resolved ResolvedSe
 				ResourceLimits:   container.Limits,
 				ResourceRequests: container.Requests,
 				ContainerPorts:   container.ContainerPort,
+				Capabilities:     container.GetSecurity().GetCapabilities(),
 			},
 		}
 

internal/runtime/interface.go

@@ -318,6 +318,7 @@ type ContainerRunOpts struct {
 	ReadOnlyFilesystem            bool
 	Privileged                    bool
 	HostNetwork                   bool
+	Capabilities                  []string
 	Mounts                        []*schema.Mount
 	ContainerPorts                []*schema.Endpoint_Port
 	ResourceLimits                *schema.Container_ResourceLimits

internal/runtime/kubernetes/deployment.go

@@ -30,6 +30,7 @@ import (
 	"namespacelabs.dev/foundation/framework/kubernetes/kubedef"
 	"namespacelabs.dev/foundation/framework/kubernetes/kubenaming"
 	"namespacelabs.dev/foundation/framework/secrets"
+	"namespacelabs.dev/foundation/internal/console"
 	"namespacelabs.dev/foundation/internal/fnerrors"
 	"namespacelabs.dev/foundation/internal/protos"
 	"namespacelabs.dev/foundation/internal/runtime"
@@ -98,7 +99,7 @@ func prepareDeployment(ctx context.Context, target BoundNamespace, deployable ru
 		return fnerrors.InternalError("kubernetes: no repository defined in image: %v", deployable.MainContainer.Image)
 	}
 
-	secCtx, err := makeSecurityContext(deployable.MainContainer)
+	secCtx, err := makeSecurityContext(deployable.MainContainer, deployable.Name, console.Debug(ctx))
 	if err != nil {
 		return err
 	}
@@ -760,7 +761,7 @@ func prepareDeployment(ctx context.Context, target BoundNamespace, deployable ru
 
 		containers = append(containers, name)
 
-		sidecarSecCtx, err := makeSecurityContext(sidecar.ContainerRunOpts)
+		sidecarSecCtx, err := makeSecurityContext(sidecar.ContainerRunOpts, sidecar.Name, console.Debug(ctx))
 		if err != nil {
 			return fnerrors.AttachLocation(deployable.ErrorLocation, err)
 		}
@@ -865,10 +866,6 @@ func prepareDeployment(ctx context.Context, target BoundNamespace, deployable ru
 		return fnerrors.AttachLocation(deployable.ErrorLocation, err)
 	}
 
-	if deployable.MainContainer.Privileged {
-		podSecCtx = podSecCtx.WithRunAsUser(0).WithRunAsGroup(0)
-	}
-
 	if deployable.MainContainer.HostNetwork {
 		spec = spec.WithHostNetwork(true).WithDNSPolicy(corev1.DNSClusterFirstWithHostNet)
 	}

internal/runtime/kubernetes/securitycontext.go

@@ -5,16 +5,33 @@
 package kubernetes
 
 import (
+	"fmt"
+	"io"
 	"io/fs"
 	"strconv"
 
+	k8sv1 "k8s.io/api/core/v1"
 	applycorev1 "k8s.io/client-go/applyconfigurations/core/v1"
 	"namespacelabs.dev/foundation/internal/fnerrors"
 	"namespacelabs.dev/foundation/internal/runtime"
 	"sigs.k8s.io/yaml"
 )
 
-func makeSecurityContext(opts runtime.ContainerRunOpts) (*applycorev1.SecurityContextApplyConfiguration, error) {
+// Effective rules:
+//
+// PodSecurityContext
+//   * defaults/pod.podsecuritycontext.yaml
+//   * kubedef.SpecExtension
+//   * MainContainer.RunAs
+//
+// SecurityContext (main or sidecar):
+//   * defaults/container.securitycontext.yaml
+//   * Container.Privileged -> RunAsUser=RunAsGroup=0, RunAsNonRoot=false
+//   * Container.Capabilities
+//
+// Sidecar.RunAs ignored.
+
+func makeSecurityContext(opts runtime.ContainerRunOpts, containerName string, debugLog io.Writer) (*applycorev1.SecurityContextApplyConfiguration, error) {
 	secCtx := applycorev1.SecurityContext()
 
 	const path = "defaults/container.securitycontext.yaml"
@@ -28,9 +45,23 @@ func makeSecurityContext(opts runtime.ContainerRunOpts) (*applycorev1.SecurityCo
 	}
 
 	if opts.Privileged {
-		secCtx = secCtx.WithPrivileged(true).WithAllowPrivilegeEscalation(true)
+		fmt.Fprintf(debugLog, "privileged container %q will run as root\n", containerName)
+		secCtx = secCtx.
+			WithPrivileged(true).
+			WithAllowPrivilegeEscalation(true).
+			WithRunAsUser(0).
+			WithRunAsGroup(0).
+			WithRunAsNonRoot(false)
 	}
 
+	var caps []k8sv1.Capability
+	for _, cap := range opts.Capabilities {
+		caps = append(caps, k8sv1.Capability(cap))
+	}
+	secCtx = secCtx.WithCapabilities(&applycorev1.CapabilitiesApplyConfiguration{
+		Add: caps,
+	})
+
 	if opts.ReadOnlyFilesystem {
 		secCtx = secCtx.WithReadOnlyRootFilesystem(true)
 	}

internal/versions/versions.json

@@ -1,5 +1,5 @@
 {
-	"api_version": 100,
+	"api_version": 101,
 	"minimum_api_version": 40,
 	"cache_version": 1
-}
+}

schema/container.pb.go

@@ -38,6 +38,7 @@ message Container {
     message Security {
         bool privileged   = 1;
         bool host_network = 2;
+        repeated string capabilities = 3;
     }
 
     message ResourceLimits {