From 2925398364e0eda28a6ebd4f4dc4d8edcfd24d7c Mon Sep 17 00:00:00 2001 From: smiletan Date: Wed, 28 May 2025 15:46:19 +0800 Subject: [PATCH 1/2] support kerberos on ddc --- api/disaggregated/v1/types.go | 18 ++++++++++ pkg/common/utils/resource/pod.go | 22 ++++++++++-- .../computegroups/statefulset.go | 4 ++- .../disaggregated_fe/statefulset.go | 5 ++- .../disaggregated_subcontroller.go | 36 +++++++++++++++++++ 5 files changed, 81 insertions(+), 4 deletions(-) diff --git a/api/disaggregated/v1/types.go b/api/disaggregated/v1/types.go index 6f20e648..9970a5ef 100644 --- a/api/disaggregated/v1/types.go +++ b/api/disaggregated/v1/types.go @@ -48,6 +48,24 @@ type DorisDisaggregatedClusterSpec struct { // if true, will decommission be node when scale down compute group. // if false, will drop be node when scale down compute group. EnableDecommission bool `json:"enableDecommission,omitempty"` + + // KerberosInfo contains a series of access key files, Provides access to kerberos. + KerberosInfo *KerberosInfo `json:"kerberosInfo,omitempty"` +} + +type KerberosInfo struct { + // Krb5ConfigMap is the name of configmap within 'krb5.conf' + Krb5ConfigMap string `json:"krb5ConfigMap,omitempty"` + + // SecretName is the name of sercet within '*.keytab' files, + // refer to the following command to create a Secret : + // 'kubectl create secret generic {secret-name} --from-file=. ' + KeytabSecretName string `json:"keytabSecretName,omitempty"` + + // KeytabPath is the path where the Secret is finally stored inside the pod. default '/etc/keytab/'. + // It is not recommended to modify it unless necessary. + // This path is the path filled in when configuring "hadoop.kerberos.keytab". + KeytabPath string `json:"keytabPath,omitempty"` } // AdminUser describe administrator for manage components in specified cluster. diff --git a/pkg/common/utils/resource/pod.go b/pkg/common/utils/resource/pod.go index 84721a98..d1be37b0 100644 --- a/pkg/common/utils/resource/pod.go +++ b/pkg/common/utils/resource/pod.go @@ -900,6 +900,24 @@ func GetMultiSecretVolumeAndVolumeMountWithCommonSpec(cSpec *dv1.CommonSpec) ([] } func getKerberosVolumeAndVolumeMount(kerberosInfo *v1.KerberosInfo) ([]corev1.Volume, []corev1.VolumeMount) { + if kerberosInfo == nil { + return []corev1.Volume{}, []corev1.VolumeMount{} + } + + return getKerberosConfigAndSecretVolumeAndVolumeMount(kerberosInfo.Krb5ConfigMap, kerberosInfo.KeytabSecretName) +} + +//get the kerberos volume and mounts to ddc. +func GetDv1KerberosVolumeAndVolumeMount(kerberosInfo *dv1.KerberosInfo)([]corev1.Volume, []corev1.VolumeMount) { + if kerberosInfo == nil { + return []corev1.Volume{}, []corev1.VolumeMount{} + } + + return getKerberosConfigAndSecretVolumeAndVolumeMount(kerberosInfo.Krb5ConfigMap, kerberosInfo.KeytabSecretName) +} + +//abstract a base function for dcr and ddc used. +func getKerberosConfigAndSecretVolumeAndVolumeMount(configMapName, secretName string) ([]corev1.Volume, []corev1.VolumeMount) { var volumes []corev1.Volume var volumeMounts []corev1.VolumeMount @@ -909,7 +927,7 @@ func getKerberosVolumeAndVolumeMount(kerberosInfo *v1.KerberosInfo) ([]corev1.Vo VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: kerberosInfo.Krb5ConfigMap, + Name: configMapName, }, }, }, @@ -925,7 +943,7 @@ func getKerberosVolumeAndVolumeMount(kerberosInfo *v1.KerberosInfo) ([]corev1.Vo Name: keytab_volume_name, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: kerberosInfo.KeytabSecretName, + SecretName: secretName, }, }, }) diff --git a/pkg/controller/sub_controller/disaggregated_cluster/computegroups/statefulset.go b/pkg/controller/sub_controller/disaggregated_cluster/computegroups/statefulset.go index 7d4aeeea..f0de9c4f 100644 --- a/pkg/controller/sub_controller/disaggregated_cluster/computegroups/statefulset.go +++ b/pkg/controller/sub_controller/disaggregated_cluster/computegroups/statefulset.go @@ -115,6 +115,8 @@ func (dcgs *DisaggregatedComputeGroupsController) NewPodTemplateSpec(ddc *dv1.Do pts.Spec.Volumes = append(pts.Spec.Volumes, secretVolumes...) } + //add last supplementary spec. if add new config in ddc spec and the config need add in pod, use the follow function to add. + dcgs.DisaggregatedSubDefaultController.AddClusterSpecForPodTemplate(dv1.DisaggregatedBE, &ddc.Spec, &pts) cgUniqueId := selector[dv1.DorisDisaggregatedComputeGroupUniqueId] pts.Spec.Affinity = dcgs.ConstructDefaultAffinity(dv1.DorisDisaggregatedComputeGroupUniqueId, cgUniqueId, pts.Spec.Affinity) @@ -135,7 +137,7 @@ func (dcgs *DisaggregatedComputeGroupsController) NewCGContainer(ddc *dv1.DorisD cmd, args := sub.GetDisaggregatedCommand(dv1.DisaggregatedBE) c.Command = cmd c.Args = args - c.Name = "compute" + c.Name = sub.BEMainContainerName c.Ports = resource.GetDisaggregatedContainerPorts(cvs, dv1.DisaggregatedBE) c.Env = cg.CommonSpec.EnvVars diff --git a/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/statefulset.go b/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/statefulset.go index fb3ee588..4b0813ac 100644 --- a/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/statefulset.go +++ b/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/statefulset.go @@ -38,6 +38,7 @@ const ( //DefaultStorageSize int64 = 107374182400 basic_auth_path = "/etc/basic_auth" auth_volume_name = "basic-auth" + ) func (dfc *DisaggregatedFEController) newFEPodsSelector(ddcName string) map[string]string { @@ -59,6 +60,8 @@ func (dfc *DisaggregatedFEController) NewStatefulset(ddc *v1.DorisDisaggregatedC spec := ddc.Spec.FeSpec _, _, vcts := dfc.BuildVolumesVolumeMountsAndPVCs(confMap, v1.DisaggregatedFE, &spec.CommonSpec) pts := dfc.NewPodTemplateSpec(ddc, confMap) + //add last supplementary spec. if add new config in ddc spec and the config need add in pod, use the follow function to add. + dfc.DisaggregatedSubDefaultController.AddClusterSpecForPodTemplate(v1.DisaggregatedFE, &ddc.Spec, &pts) st := dfc.NewDefaultStatefulset(ddc) //metadata func() { @@ -124,7 +127,7 @@ func (dfc *DisaggregatedFEController) NewFEContainer(ddc *v1.DorisDisaggregatedC cmd, args := sub.GetDisaggregatedCommand(v1.DisaggregatedFE) c.Command = cmd c.Args = args - c.Name = "fe" + c.Name = sub.FEMainContainerName c.Ports = resource.GetDisaggregatedContainerPorts(cvs, v1.DisaggregatedFE) c.Env = ddc.Spec.FeSpec.CommonSpec.EnvVars diff --git a/pkg/controller/sub_controller/disaggregated_subcontroller.go b/pkg/controller/sub_controller/disaggregated_subcontroller.go index 584581f6..c8b55151 100644 --- a/pkg/controller/sub_controller/disaggregated_subcontroller.go +++ b/pkg/controller/sub_controller/disaggregated_subcontroller.go @@ -57,6 +57,8 @@ const ( FileCachePathKey = "file_cache_path" FileCacheSubConfigPathKey = "path" FileCacheSubConfigTotalSizeKey = "total_size" + FEMainContainerName = "fe" + BEMainContainerName = "compute" ) type DisaggregatedSubController interface { @@ -292,7 +294,41 @@ func (d *DisaggregatedSubDefaultController) GetManagementAdminUserAndPWD(ctx con } +// add cluster specification on container spec. this is useful to add common spec on different type pods, example: kerberos volume for fe and be. +func(d *DisaggregatedSubDefaultController) AddClusterSpecForPodTemplate(componentType v1.DisaggregatedComponentType, spec *v1.DorisDisaggregatedClusterSpec, pts *corev1.PodTemplateSpec){ + //TODO: realize the kerberos volumeMounts added. + var c *corev1.Container + switch componentType { + case v1.DisaggregatedFE: + for i, _ := range pts.Spec.Containers { + if pts.Spec.Containers[i].Name == FEMainContainerName { + c = &pts.Spec.Containers[i] + break + } + } + case v1.DisaggregatedBE: + for i, _ := range pts.Spec.Containers { + if pts.Spec.Containers[i].Name == BEMainContainerName { + c = &pts.Spec.Containers[i] + break + } + } + + default: + klog.Errorf("DisaggregatedSubDefaultController AddClusterSpecForPodTemplate componentType %s not supported.", componentType) + return + } + //add kerberos volumeMounts and volumes + volumes, volumeMounts := resource.GetDv1KerberosVolumeAndVolumeMount(spec.KerberosInfo) + if len(volumeMounts) != 0 { + c.VolumeMounts = append(c.VolumeMounts, volumeMounts...) + } + if len(volumes) != 0 { + pts.Spec.Volumes = append(pts.Spec.Volumes, volumes...) + } + +} func (d *DisaggregatedSubDefaultController) BuildVolumesVolumeMountsAndPVCs(confMap map[string]interface{}, componentType v1.DisaggregatedComponentType, commonSpec *v1.CommonSpec) ([]corev1.Volume, []corev1.VolumeMount, []corev1.PersistentVolumeClaim) { if commonSpec.PersistentVolume == nil && len(commonSpec.PersistentVolumes) == 0 { From 344881f923c0ecc94c9d4d09b96b59a7c25ffd00 Mon Sep 17 00:00:00 2001 From: smiletan Date: Thu, 29 May 2025 15:30:55 +0800 Subject: [PATCH 2/2] add kerberos envs --- pkg/common/utils/resource/pod.go | 22 +++++++++++++++---- .../computegroups/statefulset.go | 2 +- .../disaggregated_fe/statefulset.go | 2 +- .../disaggregated_subcontroller.go | 9 ++++++-- 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/pkg/common/utils/resource/pod.go b/pkg/common/utils/resource/pod.go index d1be37b0..c054557a 100644 --- a/pkg/common/utils/resource/pod.go +++ b/pkg/common/utils/resource/pod.go @@ -562,11 +562,23 @@ func buildKerberosEnv(info *v1.KerberosInfo, config map[string]interface{}, comp return nil } + return buildKerberosEnvUseSecretMountPath(info.KeytabPath, config, string(componentType)) +} + +func BuildKerberosEnvForDDC(info *dv1.KerberosInfo, config map[string]interface{}, componentType dv1.DisaggregatedComponentType)[]corev1.EnvVar { + if info == nil { + return nil + } + + return buildKerberosEnvUseSecretMountPath(info.KeytabPath, config, string(componentType)) +} + +func buildKerberosEnvUseSecretMountPath(keytabPath string, config map[string]interface{}, componentType string) []corev1.EnvVar { var krb5ConfPath string switch componentType { - case v1.Component_FE: + case string(v1.Component_FE), string(dv1.DisaggregatedFE): krb5ConfPath = kerberos.GetKrb5ConfFromJavaOpts(config) - case v1.Component_BE, v1.Component_CN: + case string(v1.Component_BE), string(v1.Component_CN), string(dv1.DisaggregatedBE): // be config krb5.conf file must set 'kerberos_krb5_conf_path' in be.conf // https://doris.apache.org/docs/3.0/lakehouse/datalake-analytics/hive?_highlight=kerberos_krb5_conf_path#connect-to-kerberos-enabled-hive if value, exists := config["kerberos_krb5_conf_path"]; exists { @@ -574,11 +586,13 @@ func buildKerberosEnv(info *v1.KerberosInfo, config map[string]interface{}, comp } else { krb5ConfPath = kerberos.KRB5_DEFAULT_CONFIG } + default: + klog.Errorf("BuildKerberosEnvUseSecretMountPath, componentType %s not supported.", componentType) } keytabFinalUsedPath := keytab_default_mount_path - if info.KeytabPath != "" { - keytabFinalUsedPath = info.KeytabPath + if keytabPath != "" { + keytabFinalUsedPath = keytabPath } return []corev1.EnvVar{ diff --git a/pkg/controller/sub_controller/disaggregated_cluster/computegroups/statefulset.go b/pkg/controller/sub_controller/disaggregated_cluster/computegroups/statefulset.go index f0de9c4f..6edc3df2 100644 --- a/pkg/controller/sub_controller/disaggregated_cluster/computegroups/statefulset.go +++ b/pkg/controller/sub_controller/disaggregated_cluster/computegroups/statefulset.go @@ -116,7 +116,7 @@ func (dcgs *DisaggregatedComputeGroupsController) NewPodTemplateSpec(ddc *dv1.Do } //add last supplementary spec. if add new config in ddc spec and the config need add in pod, use the follow function to add. - dcgs.DisaggregatedSubDefaultController.AddClusterSpecForPodTemplate(dv1.DisaggregatedBE, &ddc.Spec, &pts) + dcgs.DisaggregatedSubDefaultController.AddClusterSpecForPodTemplate(dv1.DisaggregatedBE, cvs, &ddc.Spec, &pts) cgUniqueId := selector[dv1.DorisDisaggregatedComputeGroupUniqueId] pts.Spec.Affinity = dcgs.ConstructDefaultAffinity(dv1.DorisDisaggregatedComputeGroupUniqueId, cgUniqueId, pts.Spec.Affinity) diff --git a/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/statefulset.go b/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/statefulset.go index 4b0813ac..8a63a5ed 100644 --- a/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/statefulset.go +++ b/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/statefulset.go @@ -61,7 +61,7 @@ func (dfc *DisaggregatedFEController) NewStatefulset(ddc *v1.DorisDisaggregatedC _, _, vcts := dfc.BuildVolumesVolumeMountsAndPVCs(confMap, v1.DisaggregatedFE, &spec.CommonSpec) pts := dfc.NewPodTemplateSpec(ddc, confMap) //add last supplementary spec. if add new config in ddc spec and the config need add in pod, use the follow function to add. - dfc.DisaggregatedSubDefaultController.AddClusterSpecForPodTemplate(v1.DisaggregatedFE, &ddc.Spec, &pts) + dfc.DisaggregatedSubDefaultController.AddClusterSpecForPodTemplate(v1.DisaggregatedFE,confMap, &ddc.Spec, &pts) st := dfc.NewDefaultStatefulset(ddc) //metadata func() { diff --git a/pkg/controller/sub_controller/disaggregated_subcontroller.go b/pkg/controller/sub_controller/disaggregated_subcontroller.go index c8b55151..b9aacece 100644 --- a/pkg/controller/sub_controller/disaggregated_subcontroller.go +++ b/pkg/controller/sub_controller/disaggregated_subcontroller.go @@ -295,8 +295,7 @@ func (d *DisaggregatedSubDefaultController) GetManagementAdminUserAndPWD(ctx con } // add cluster specification on container spec. this is useful to add common spec on different type pods, example: kerberos volume for fe and be. -func(d *DisaggregatedSubDefaultController) AddClusterSpecForPodTemplate(componentType v1.DisaggregatedComponentType, spec *v1.DorisDisaggregatedClusterSpec, pts *corev1.PodTemplateSpec){ - //TODO: realize the kerberos volumeMounts added. +func(d *DisaggregatedSubDefaultController) AddClusterSpecForPodTemplate(componentType v1.DisaggregatedComponentType, configMap map[string]interface{}, spec *v1.DorisDisaggregatedClusterSpec, pts *corev1.PodTemplateSpec){ var c *corev1.Container switch componentType { case v1.DisaggregatedFE: @@ -319,6 +318,12 @@ func(d *DisaggregatedSubDefaultController) AddClusterSpecForPodTemplate(componen return } + //add pod envs + envs := resource.BuildKerberosEnvForDDC(spec.KerberosInfo, configMap, componentType) + if len(envs) != 0 { + c.Env = append(c.Env, envs...) + } + //add kerberos volumeMounts and volumes volumes, volumeMounts := resource.GetDv1KerberosVolumeAndVolumeMount(spec.KerberosInfo) if len(volumeMounts) != 0 {