From 61c0147aa6cde6f7decf6e7e031bd9aaab7e11ae Mon Sep 17 00:00:00 2001 From: catpineapple Date: Wed, 7 May 2025 19:08:01 +0800 Subject: [PATCH 1/2] ddc-skip-default-systeminit --- api/disaggregated/v1/types.go | 6 ++++++ pkg/common/utils/resource/pod.go | 8 ++++---- pkg/common/utils/resource/pod_test.go | 2 +- .../disaggregated_cluster/computegroups/statefulset.go | 8 +++++++- .../disaggregated_cluster/disaggregated_fe/statefulset.go | 2 +- .../disaggregated_cluster/metaservice/statefulset.go | 2 +- 6 files changed, 20 insertions(+), 8 deletions(-) diff --git a/api/disaggregated/v1/types.go b/api/disaggregated/v1/types.go index 7cc8a7fe..97995e8d 100644 --- a/api/disaggregated/v1/types.go +++ b/api/disaggregated/v1/types.go @@ -90,6 +90,12 @@ type ComputeGroup struct { UniqueId string `json:"uniqueId"` CommonSpec `json:",inline"` + + // SkipDefaultSystemInit is a switch that skips the default initialization and is used to set the default environment configuration required by the doris BE node. + // Default value is 'false'. + // Default System Init means that the container must be started in privileged mode. + // Default System Init configuration is implemented through the initContainers of the pod, so changes to this configuration may be ignored by k8s when it is not the first deployment. + SkipDefaultSystemInit bool `json:"skipDefaultSystemInit,omitempty"` } type CommonSpec struct { diff --git a/pkg/common/utils/resource/pod.go b/pkg/common/utils/resource/pod.go index c3a1893d..cfa13a08 100644 --- a/pkg/common/utils/resource/pod.go +++ b/pkg/common/utils/resource/pod.go @@ -190,7 +190,7 @@ func NewPodTemplateSpec(dcr *v1.DorisCluster, config map[string]interface{}, com } // for disaggregated cluster. -func NewPodTemplateSpecWithCommonSpec(cs *dv1.CommonSpec, componentType dv1.DisaggregatedComponentType) corev1.PodTemplateSpec { +func NewPodTemplateSpecWithCommonSpec(skipInit bool, cs *dv1.CommonSpec, componentType dv1.DisaggregatedComponentType) corev1.PodTemplateSpec { var vs []corev1.Volume si := cs.SystemInitialization var defaultInitContainers []corev1.Container @@ -214,7 +214,7 @@ func NewPodTemplateSpecWithCommonSpec(cs *dv1.CommonSpec, componentType dv1.Disa Volumes: vs, }, } - constructDisaggregatedInitContainers(componentType, &pts.Spec, si) + constructDisaggregatedInitContainers(skipInit, componentType, &pts.Spec, si) return pts } @@ -264,7 +264,7 @@ func constructInitContainers(skipInit bool, componentType v1.ComponentType, podS podSpec.InitContainers = append(podSpec.InitContainers, defaultInitContains...) } -func constructDisaggregatedInitContainers(componentType dv1.DisaggregatedComponentType, podSpec *corev1.PodSpec, si *dv1.SystemInitialization) { +func constructDisaggregatedInitContainers(skipInit bool, componentType dv1.DisaggregatedComponentType, podSpec *corev1.PodSpec, si *dv1.SystemInitialization) { initImage := DEFAULT_INIT_IMAGE var defaultInitContains []corev1.Container if si != nil { @@ -287,7 +287,7 @@ func constructDisaggregatedInitContainers(componentType dv1.DisaggregatedCompone } // the init containers have sequence，should confirm use initial is always in the first priority. - if componentType == dv1.DisaggregatedBE { + if !skipInit && componentType == dv1.DisaggregatedBE { podSpec.InitContainers = append(podSpec.InitContainers, constructBeDefaultInitContainer(initImage)) } podSpec.InitContainers = append(podSpec.InitContainers, defaultInitContains...) diff --git a/pkg/common/utils/resource/pod_test.go b/pkg/common/utils/resource/pod_test.go index a9dc5cba..41c4ce14 100644 --- a/pkg/common/utils/resource/pod_test.go +++ b/pkg/common/utils/resource/pod_test.go @@ -122,7 +122,7 @@ func Test_NewPodTemplateSpecWithCommonSpec(t *testing.T) { tm[dv1.DisaggregatedMS] = mcs for dct, cs := range tm { - pts := NewPodTemplateSpecWithCommonSpec(cs, dct) + pts := NewPodTemplateSpecWithCommonSpec(false, cs, dct) t.Log(pts) } } diff --git a/pkg/controller/sub_controller/disaggregated_cluster/computegroups/statefulset.go b/pkg/controller/sub_controller/disaggregated_cluster/computegroups/statefulset.go index c37a2800..9cde10c4 100644 --- a/pkg/controller/sub_controller/disaggregated_cluster/computegroups/statefulset.go +++ b/pkg/controller/sub_controller/disaggregated_cluster/computegroups/statefulset.go @@ -81,7 +81,7 @@ func (dcgs *DisaggregatedComputeGroupsController) NewStatefulset(ddc *dv1.DorisD } func (dcgs *DisaggregatedComputeGroupsController) NewPodTemplateSpec(ddc *dv1.DorisDisaggregatedCluster, selector map[string]string, cvs map[string]interface{}, cg *dv1.ComputeGroup) corev1.PodTemplateSpec { - pts := resource.NewPodTemplateSpecWithCommonSpec(&cg.CommonSpec, dv1.DisaggregatedBE) + pts := resource.NewPodTemplateSpecWithCommonSpec(cg.SkipDefaultSystemInit, &cg.CommonSpec, dv1.DisaggregatedBE) //pod template metadata. func() { l := (resource.Labels)(selector) @@ -132,6 +132,12 @@ func (dcgs *DisaggregatedComputeGroupsController) NewCGContainer(ddc *dv1.DorisD c.Env = append(c.Env, resource.GetPodDefaultEnv()...) c.Env = append(c.Env, dcgs.newSpecificEnvs(ddc, cg)...) + if cg.SkipDefaultSystemInit { + // Only works when the doris version is higher than 2.1.8 or 3.0.4 + // When the environment variable SKIP_CHECK_ULIMIT=true is passed in, the start_be.sh will not check system parameters like ulimit and vm.max_map_count etc. + c.Env = append(c.Env, corev1.EnvVar{Name: "SKIP_CHECK_ULIMIT", Value: "true"}) + } + resource.BuildDisaggregatedProbe(&c, &cg.CommonSpec, dv1.DisaggregatedBE) _, vms, _ := dcgs.BuildVolumesVolumeMountsAndPVCs(cvs, dv1.DisaggregatedBE, &cg.CommonSpec) _, cmvms := dcgs.BuildDefaultConfigMapVolumesVolumeMounts(cg.ConfigMaps) diff --git a/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/statefulset.go b/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/statefulset.go index b38d6eb3..fb3ee588 100644 --- a/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/statefulset.go +++ b/pkg/controller/sub_controller/disaggregated_cluster/disaggregated_fe/statefulset.go @@ -86,7 +86,7 @@ func (dfc *DisaggregatedFEController) getFEPodLabels(ddc *v1.DorisDisaggregatedC } func (dfc *DisaggregatedFEController) NewPodTemplateSpec(ddc *v1.DorisDisaggregatedCluster, confMap map[string]interface{}) corev1.PodTemplateSpec { - pts := resource.NewPodTemplateSpecWithCommonSpec(&ddc.Spec.FeSpec.CommonSpec, v1.DisaggregatedFE) + pts := resource.NewPodTemplateSpecWithCommonSpec(false, &ddc.Spec.FeSpec.CommonSpec, v1.DisaggregatedFE) //pod template metadata. labels := dfc.getFEPodLabels(ddc) pts.Labels = labels diff --git a/pkg/controller/sub_controller/disaggregated_cluster/metaservice/statefulset.go b/pkg/controller/sub_controller/disaggregated_cluster/metaservice/statefulset.go index e688daf2..4f37e38c 100644 --- a/pkg/controller/sub_controller/disaggregated_cluster/metaservice/statefulset.go +++ b/pkg/controller/sub_controller/disaggregated_cluster/metaservice/statefulset.go @@ -88,7 +88,7 @@ func (dms *DisaggregatedMSController) newStatefulset(ddc *v1.DorisDisaggregatedC } func (dms *DisaggregatedMSController) NewPodTemplateSpec(ddc *v1.DorisDisaggregatedCluster, selector map[string]string, confMap map[string]interface{}) corev1.PodTemplateSpec { - pts := resource.NewPodTemplateSpecWithCommonSpec(&ddc.Spec.MetaService.CommonSpec, v1.DisaggregatedMS) + pts := resource.NewPodTemplateSpecWithCommonSpec(false, &ddc.Spec.MetaService.CommonSpec, v1.DisaggregatedMS) //pod template metadata. func() { l := (resource.Labels)(selector) From 1400525dcedc0fd9c6d604c30b49eec320ca4309 Mon Sep 17 00:00:00 2001 From: catpineapple Date: Thu, 8 May 2025 16:32:54 +0800 Subject: [PATCH 2/2] ddc-skip-default-systeminit --- config/crd/bases/crds.yaml | 7 +++++++ ...ated.cluster.doris.com_dorisdisaggregatedclusters.yaml | 7 +++++++ pkg/common/utils/resource/pod.go | 8 ++++---- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/config/crd/bases/crds.yaml b/config/crd/bases/crds.yaml index b06e8c2e..d4fb67fd 100644 --- a/config/crd/bases/crds.yaml +++ b/config/crd/bases/crds.yaml @@ -11361,6 +11361,13 @@ spec: serviceAccount: description: serviceAccount for compute node access cloud service. type: string + skipDefaultSystemInit: + description: |- + SkipDefaultSystemInit is a switch that skips the default initialization and is used to set the default environment configuration required by the doris BE node. + Default value is 'false'. + Default System Init means that the container must be started in privileged mode. + Default System Init configuration is implemented through the initContainers of the pod, so changes to this configuration may be ignored by k8s when it is not the first deployment. + type: boolean startTimeout: description: pod start timeout, unit is second format: int32 diff --git a/config/crd/bases/disaggregated.cluster.doris.com_dorisdisaggregatedclusters.yaml b/config/crd/bases/disaggregated.cluster.doris.com_dorisdisaggregatedclusters.yaml index edac79b0..89081f27 100644 --- a/config/crd/bases/disaggregated.cluster.doris.com_dorisdisaggregatedclusters.yaml +++ b/config/crd/bases/disaggregated.cluster.doris.com_dorisdisaggregatedclusters.yaml @@ -2234,6 +2234,13 @@ spec: serviceAccount: description: serviceAccount for compute node access cloud service. type: string + skipDefaultSystemInit: + description: |- + SkipDefaultSystemInit is a switch that skips the default initialization and is used to set the default environment configuration required by the doris BE node. + Default value is 'false'. + Default System Init means that the container must be started in privileged mode. + Default System Init configuration is implemented through the initContainers of the pod, so changes to this configuration may be ignored by k8s when it is not the first deployment. + type: boolean startTimeout: description: pod start timeout, unit is second format: int32 diff --git a/pkg/common/utils/resource/pod.go b/pkg/common/utils/resource/pod.go index cfa13a08..84721a98 100644 --- a/pkg/common/utils/resource/pod.go +++ b/pkg/common/utils/resource/pod.go @@ -190,7 +190,7 @@ func NewPodTemplateSpec(dcr *v1.DorisCluster, config map[string]interface{}, com } // for disaggregated cluster. -func NewPodTemplateSpecWithCommonSpec(skipInit bool, cs *dv1.CommonSpec, componentType dv1.DisaggregatedComponentType) corev1.PodTemplateSpec { +func NewPodTemplateSpecWithCommonSpec(skipDefaultInit bool, cs *dv1.CommonSpec, componentType dv1.DisaggregatedComponentType) corev1.PodTemplateSpec { var vs []corev1.Volume si := cs.SystemInitialization var defaultInitContainers []corev1.Container @@ -214,7 +214,7 @@ func NewPodTemplateSpecWithCommonSpec(skipInit bool, cs *dv1.CommonSpec, compone Volumes: vs, }, } - constructDisaggregatedInitContainers(skipInit, componentType, &pts.Spec, si) + constructDisaggregatedInitContainers(skipDefaultInit, componentType, &pts.Spec, si) return pts } @@ -264,7 +264,7 @@ func constructInitContainers(skipInit bool, componentType v1.ComponentType, podS podSpec.InitContainers = append(podSpec.InitContainers, defaultInitContains...) } -func constructDisaggregatedInitContainers(skipInit bool, componentType dv1.DisaggregatedComponentType, podSpec *corev1.PodSpec, si *dv1.SystemInitialization) { +func constructDisaggregatedInitContainers(skipDefaultInit bool, componentType dv1.DisaggregatedComponentType, podSpec *corev1.PodSpec, si *dv1.SystemInitialization) { initImage := DEFAULT_INIT_IMAGE var defaultInitContains []corev1.Container if si != nil { @@ -287,7 +287,7 @@ func constructDisaggregatedInitContainers(skipInit bool, componentType dv1.Disag } // the init containers have sequence，should confirm use initial is always in the first priority. - if !skipInit && componentType == dv1.DisaggregatedBE { + if !skipDefaultInit && componentType == dv1.DisaggregatedBE { podSpec.InitContainers = append(podSpec.InitContainers, constructBeDefaultInitContainer(initImage)) } podSpec.InitContainers = append(podSpec.InitContainers, defaultInitContains...)