feat: auto-migrate k3s to k8s (#2654)

Author: FabianKramm

config/config.go

@@ -325,7 +325,7 @@ func ValidateChanges(oldCfg, newCfg *Config) error {
 
 // ValidateStoreAndDistroChanges checks whether migrating from one store to the other is allowed.
 func ValidateStoreAndDistroChanges(currentStoreType, previousStoreType StoreType, currentDistro, previousDistro string) error {
-	if currentDistro != previousDistro && !(previousDistro == "eks" && currentDistro == K8SDistro) {
+	if currentDistro != previousDistro && !(previousDistro == "eks" && currentDistro == K8SDistro) && !(previousDistro == K3SDistro && currentDistro == K8SDistro) {
 		return fmt.Errorf("seems like you were using %s as a distro before and now have switched to %s, please make sure to not switch between vCluster distros", previousDistro, currentDistro)
 	}
 

pkg/certs/constants.go

@@ -29,6 +29,16 @@ const (
 	// CAKeyName defines certificate name
 	CAKeyName = "ca.key"
 
+	// ServerCAKeyName defines server ca key name
+	ServerCAKeyName = "server-ca.key"
+	// ServerCACertName defines server ca cert name
+	ServerCACertName = "server-ca.crt"
+
+	// ClientCACertName defines client ca cert name
+	ClientCACertName = "client-ca.crt"
+	// ClientCAKeyName defines client ca key name
+	ClientCAKeyName = "client-ca.key"
+
 	// APIServerCertAndKeyBaseName defines API's server certificate and key base name
 	APIServerCertAndKeyBaseName = "apiserver"
 	// APIServerCertName defines API's server certificate name
@@ -148,6 +158,12 @@ var certMap = map[string]string{
 	CACertName: CACertName,
 	CAKeyName:  CAKeyName,
 
+	ServerCACertName: ServerCACertName,
+	ServerCAKeyName:  ServerCAKeyName,
+
+	ClientCACertName: ClientCACertName,
+	ClientCAKeyName:  ClientCAKeyName,
+
 	FrontProxyCACertName: FrontProxyCACertName,
 	FrontProxyCAKeyName:  FrontProxyCAKeyName,
 

pkg/certs/ensure.go

@@ -42,10 +42,10 @@ func EnsureCerts(
 		return errors.New("nil currentNamespaceClient")
 	}
 
-	// we create a certificate for up to 20 etcd replicas, this should be sufficient for most use cases. Eventually we probably
+	// we create a certificate for up to 5 etcd replicas, this should be sufficient for most use cases. Eventually we probably
 	// want to update this to the actual etcd number, but for now this is the easiest way to allow up and downscaling without
 	// regenerating certificates.
-	secretName := vClusterName + "-certs"
+	secretName := CertSecretName(vClusterName)
 	secret, err := currentNamespaceClient.CoreV1().Secrets(currentNamespace).Get(ctx, secretName, metav1.GetOptions{})
 	if err == nil {
 		// download certs from secret
@@ -181,6 +181,10 @@ func EnsureCerts(
 	return downloadCertsFromSecret(secret, certificateDir)
 }
 
+func CertSecretName(vClusterName string) string {
+	return vClusterName + "-certs"
+}
+
 func createConfig(serviceCIDR, vClusterName, certificateDir, clusterDomain string, etcdSans []string) (*InitConfiguration, error) {
 	// init config
 	cfg, err := SetInitDynamicDefaults()
@@ -228,6 +232,11 @@ func generateCertificates(
 		return fmt.Errorf("create kube configs: %w", err)
 	}
 
+	err = splitCACert(certificateDir)
+	if err != nil {
+		return fmt.Errorf("split ca cert: %w", err)
+	}
+
 	return nil
 }
 
@@ -267,6 +276,46 @@ func downloadCertsFromSecret(
 		}
 	}
 
+	err := splitCACert(certificateDir)
+	if err != nil {
+		return fmt.Errorf("split ca cert: %w", err)
+	}
+
+	return nil
+}
+
+func splitCACert(certificateDir string) error {
+	// make sure to write server-ca and client-ca to file system
+	err := copyFileIfNotExists(filepath.Join(certificateDir, CACertName), filepath.Join(certificateDir, ServerCACertName))
+	if err != nil {
+		return fmt.Errorf("copy %s: %w", ServerCACertName, err)
+	}
+	err = copyFileIfNotExists(filepath.Join(certificateDir, CAKeyName), filepath.Join(certificateDir, ServerCAKeyName))
+	if err != nil {
+		return fmt.Errorf("copy %s: %w", ServerCAKeyName, err)
+	}
+	err = copyFileIfNotExists(filepath.Join(certificateDir, CACertName), filepath.Join(certificateDir, ClientCACertName))
+	if err != nil {
+		return fmt.Errorf("copy %s: %w", ClientCACertName, err)
+	}
+	err = copyFileIfNotExists(filepath.Join(certificateDir, CAKeyName), filepath.Join(certificateDir, ClientCAKeyName))
+	if err != nil {
+		return fmt.Errorf("copy %s: %w", ClientCAKeyName, err)
+	}
+
+	return nil
+}
+
+func copyFileIfNotExists(src, dst string) error {
+	_, err := os.Stat(dst)
+	if os.IsNotExist(err) {
+		srcBytes, err := os.ReadFile(src)
+		if err != nil {
+			return fmt.Errorf("read %s: %w", src, err)
+		}
+
+		return os.WriteFile(dst, srcBytes, 0666)
+	}
 	return nil
 }
 

pkg/config/config.go

@@ -75,9 +75,9 @@ func (v VirtualClusterConfig) VirtualClusterKubeConfig() config.VirtualClusterKu
 	case config.K8SDistro:
 		distroConfig = config.VirtualClusterKubeConfig{
 			KubeConfig:          "/data/pki/admin.conf",
-			ServerCAKey:         "/data/pki/ca.key",
-			ServerCACert:        "/data/pki/ca.crt",
-			ClientCACert:        "/data/pki/ca.crt",
+			ServerCAKey:         "/data/pki/server-ca.key",
+			ServerCACert:        "/data/pki/server-ca.crt",
+			ClientCACert:        "/data/pki/client-ca.crt",
 			RequestHeaderCACert: "/data/pki/front-proxy-ca.crt",
 		}
 	}

pkg/k8s/migrate.go

@@ -0,0 +1,205 @@
+package k8s
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	vclusterconfig "github.com/loft-sh/vcluster/config"
+	"github.com/loft-sh/vcluster/pkg/certs"
+	"github.com/loft-sh/vcluster/pkg/config"
+	"github.com/loft-sh/vcluster/pkg/constants"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/klog/v2"
+)
+
+var (
+	migratedFromK3sAnnotation = "vcluster.loft.sh/migrated-from-k3s"
+
+	k3sKubeConfig = map[string]string{
+		certs.AdminKubeConfigFileName:             "/data/server/cred/admin.kubeconfig",
+		certs.ControllerManagerKubeConfigFileName: "/data/server/cred/controller.kubeconfig",
+		certs.SchedulerKubeConfigFileName:         "/data/server/cred/scheduler.kubeconfig",
+	}
+
+	k3sTLS = map[string]string{
+		certs.APIServerCertName: "/data/server/tls/serving-kube-apiserver.crt",
+		certs.APIServerKeyName:  "/data/server/tls/serving-kube-apiserver.key",
+
+		certs.ServerCACertName: "/data/server/tls/server-ca.crt",
+		certs.ServerCAKeyName:  "/data/server/tls/server-ca.key",
+
+		certs.ClientCACertName: "/data/server/tls/client-ca.crt",
+		certs.ClientCAKeyName:  "/data/server/tls/client-ca.key",
+
+		certs.FrontProxyCACertName: "/data/server/tls/request-header-ca.crt",
+		certs.FrontProxyCAKeyName:  "/data/server/tls/request-header-ca.key",
+
+		certs.FrontProxyClientCertName: "/data/server/tls/client-auth-proxy.crt",
+		certs.FrontProxyClientKeyName:  "/data/server/tls/client-auth-proxy.key",
+
+		certs.ServiceAccountPrivateKeyName: "/data/server/tls/service.current.key",
+		certs.ServiceAccountPublicKeyName:  "/data/server/tls/service.key",
+	}
+)
+
+func MigrateK3sToK8s(ctx context.Context, currentNamespaceClient kubernetes.Interface, currentNamespace string, options *config.VirtualClusterConfig) error {
+	// only migrate if we are migrating from k3s to k8s
+	if options.Distro() != vclusterconfig.K8SDistro {
+		return nil
+	} else if _, err := os.Stat("/data/server/tls"); err != nil { // fast path
+		return nil
+	}
+
+	// migrate data first
+	if options.BackingStoreType() == vclusterconfig.StoreTypeEmbeddedDatabase || options.BackingStoreType() == vclusterconfig.StoreTypeEmbeddedEtcd {
+		// copy over the data
+		err := renameIfExists(constants.K3sSqliteDatabase, constants.K8sSqliteDatabase)
+		if err != nil {
+			return fmt.Errorf("failed to rename sqlite database: %w", err)
+		}
+		err = renameIfExists(constants.K3sSqliteDatabase+"-wal", constants.K8sSqliteDatabase+"-wal")
+		if err != nil {
+			return fmt.Errorf("failed to rename sqlite database: %w", err)
+		}
+		err = renameIfExists(constants.K3sSqliteDatabase+"-shm", constants.K8sSqliteDatabase+"-shm")
+		if err != nil {
+			return fmt.Errorf("failed to rename sqlite database: %w", err)
+		}
+	}
+
+	// get the secret
+	secret, err := currentNamespaceClient.CoreV1().Secrets(currentNamespace).Get(ctx, certs.CertSecretName(options.Name), metav1.GetOptions{})
+	if err != nil {
+		if kerrors.IsNotFound(err) {
+			// this is fine, we can just skip this
+			return nil
+		}
+
+		return fmt.Errorf("failed to get certificate secret %s: %w", certs.CertSecretName(options.Name), err)
+	} else if secret.Annotations[migratedFromK3sAnnotation] == "true" { // already migrated
+		return nil
+	}
+
+	// convert tls secrets
+	for inSecretName, fileName := range k3sKubeConfig {
+		if _, err := os.Stat(fileName); os.IsNotExist(err) {
+			return nil
+		}
+
+		secret.Data[inSecretName], err = fillKubeConfig(fileName)
+		if err != nil {
+			klog.Errorf("failed to fill k3s kube config %s: %s", fileName, err)
+			return err
+		}
+	}
+
+	// convert kube configs
+	for inSecretName, fileName := range k3sTLS {
+		if _, err := os.Stat(fileName); os.IsNotExist(err) {
+			return nil
+		}
+
+		secret.Data[inSecretName], err = os.ReadFile(fileName)
+		if err != nil {
+			klog.Errorf("failed to read k3s tls secret %s: %s", fileName, err)
+			return err
+		}
+	}
+
+	// update secret
+	klog.Infof("Migrating k3s distro certificates to k8s...")
+	if secret.Annotations == nil {
+		secret.Annotations = make(map[string]string)
+	}
+	secret.Annotations[migratedFromK3sAnnotation] = "true"
+	_, err = currentNamespaceClient.CoreV1().Secrets(secret.Namespace).Update(ctx, secret, metav1.UpdateOptions{})
+	if err != nil {
+		if kerrors.IsConflict(err) {
+			klog.Infof("failed to migrate k3s tls secret %s: %s, retrying", secret.Name, err)
+
+			// get the secret again
+			secret, err = currentNamespaceClient.CoreV1().Secrets(secret.Namespace).Get(ctx, secret.Name, metav1.GetOptions{})
+			if err != nil {
+				klog.Errorf("failed to get k3s tls secret %s: %s", secret.Name, err)
+				return err
+			}
+
+			return MigrateK3sToK8s(ctx, currentNamespaceClient, currentNamespace, options)
+		}
+
+		klog.Errorf("failed to migrate k3s tls secret %s: %s", secret.Name, err)
+		return err
+	}
+
+	// remove old data
+	_ = os.RemoveAll("/data/server")
+	return nil
+}
+
+func renameIfExists(old, new string) error {
+	if _, err := os.Stat(old); os.IsNotExist(err) {
+		return nil
+	}
+
+	return os.Rename(old, new)
+}
+
+func fillKubeConfig(kubeConfigPath string) ([]byte, error) {
+	config, err := clientcmd.LoadFromFile(kubeConfigPath)
+	if err != nil {
+		return nil, err
+	}
+
+	// exchange kube config server & resolve certificate
+	for _, cluster := range config.Clusters {
+		if cluster == nil {
+			continue
+		}
+
+		// fill in data
+		if cluster.CertificateAuthorityData == nil && cluster.CertificateAuthority != "" {
+			o, err := os.ReadFile(cluster.CertificateAuthority)
+			if err != nil {
+				return nil, err
+			}
+
+			cluster.CertificateAuthority = ""
+			cluster.CertificateAuthorityData = o
+		}
+
+		cluster.Server = "https://127.0.0.1:6443"
+	}
+
+	// resolve auth info cert & key
+	for _, authInfo := range config.AuthInfos {
+		if authInfo == nil {
+			continue
+		}
+
+		// fill in data
+		if authInfo.ClientCertificateData == nil && authInfo.ClientCertificate != "" {
+			o, err := os.ReadFile(authInfo.ClientCertificate)
+			if err != nil {
+				return nil, err
+			}
+
+			authInfo.ClientCertificate = ""
+			authInfo.ClientCertificateData = o
+		}
+		if authInfo.ClientKeyData == nil && authInfo.ClientKey != "" {
+			o, err := os.ReadFile(authInfo.ClientKey)
+			if err != nil {
+				return nil, err
+			}
+
+			authInfo.ClientKey = ""
+			authInfo.ClientKeyData = o
+		}
+	}
+
+	return clientcmd.Write(*config)
+}

pkg/setup/config.go

@@ -122,7 +122,7 @@ func EnsureBackingStoreChanges(ctx context.Context, client kubernetes.Interface,
 // It checks for the existence of the default K3s token path or the K0s data directory.
 func CheckUsingHeuristic(distro string) (bool, error) {
 	// check if previously we were using k3s as a default and now have switched to a different distro
-	if distro != vclusterconfig.K3SDistro {
+	if distro != vclusterconfig.K3SDistro && distro != vclusterconfig.K8SDistro {
 		_, err := os.Stat(k3s.TokenPath)
 		if err == nil {
 			return false, fmt.Errorf("seems like you were using k3s as a distro before and now have switched to %s, please make sure to not switch between vCluster distros", distro)

pkg/setup/initialize.go

@@ -31,8 +31,14 @@ import (
 
 // Initialize creates the required secrets and configmaps for the control plane to start
 func Initialize(ctx context.Context, options *config.VirtualClusterConfig) error {
+	// migrate k3s to k8s if needed
+	err := k8s.MigrateK3sToK8s(ctx, options.ControlPlaneClient, options.ControlPlaneNamespace, options)
+	if err != nil {
+		return fmt.Errorf("migrate k3s to k8s: %w", err)
+	}
+
 	// Ensure that service CIDR range is written into the expected location
-	err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 2*time.Minute, true, func(waitCtx context.Context) (bool, error) {
+	err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 2*time.Minute, true, func(waitCtx context.Context) (bool, error) {
 		err := initialize(waitCtx, options)
 		if err != nil {
 			klog.Errorf("error initializing service cidr, certs and token: %v", err)
@@ -253,9 +259,8 @@ func GenerateCerts(ctx context.Context, currentNamespaceClient kubernetes.Interf
 		)
 	}
 
-	// expect up to 20 etcd members, number could be lower since more
-	// than 5 is generally a bad idea
-	for i := range 20 {
+	// expect up to 5 etcd members
+	for i := range 5 {
 		// this is for embedded etcd
 		hostname := vClusterName + "-" + strconv.Itoa(i)
 		etcdSans = append(etcdSans, hostname, hostname+"."+vClusterName+"-headless", hostname+"."+vClusterName+"-headless"+"."+currentNamespace)