Merge pull request #633 from WangZzzhe/dev/npd-controller

feat: add npd controller

Author: xu282934741

cmd/katalyst-controller/app/controller/npd.go

@@ -0,0 +1,57 @@
+/*
+Copyright 2022 The Katalyst Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"context"
+	"fmt"
+
+	"k8s.io/klog/v2"
+
+	katalyst "github.com/kubewharf/katalyst-core/cmd/base"
+	"github.com/kubewharf/katalyst-core/pkg/config"
+	"github.com/kubewharf/katalyst-core/pkg/controller/npd"
+)
+
+const (
+	NPDControllerName = "npd"
+)
+
+func StartNPDController(ctx context.Context, controlCtx *katalyst.GenericContext,
+	conf *config.Configuration, extraConf interface{}, _ string,
+) (bool, error) {
+	if controlCtx == nil || conf == nil {
+		err := fmt.Errorf("controlCtx and controllerConf can't be nil")
+		klog.Error(err)
+		return false, err
+	}
+
+	npdController, err := npd.NewNPDController(
+		ctx,
+		controlCtx, conf.GenericConfiguration,
+		conf.GenericControllerConfiguration,
+		conf.NPDConfig,
+		extraConf,
+	)
+	if err != nil {
+		klog.Errorf("failed to new npd controller")
+		return false, err
+	}
+
+	go npdController.Run()
+	return true, nil
+}

cmd/katalyst-controller/app/enablecontrollers.go

@@ -51,6 +51,7 @@ func init() {
 	controllerInitializers.Store(controller.VPAControllerName, ControllerStarter{Starter: controller.StartVPAController})
 	controllerInitializers.Store(controller.KCCControllerName, ControllerStarter{Starter: controller.StartKCCController})
 	controllerInitializers.Store(controller.SPDControllerName, ControllerStarter{Starter: controller.StartSPDController})
+	controllerInitializers.Store(controller.NPDControllerName, ControllerStarter{Starter: controller.StartNPDController})
 	controllerInitializers.Store(controller.LifeCycleControllerName, ControllerStarter{Starter: controller.StartLifeCycleController})
 	controllerInitializers.Store(controller.MonitorControllerName, ControllerStarter{Starter: controller.StartMonitorController})
 	controllerInitializers.Store(controller.OvercommitControllerName, ControllerStarter{Starter: controller.StartOvercommitController})

cmd/katalyst-controller/app/options/controller.go

@@ -28,6 +28,7 @@ type ControllersOptions struct {
 	*VPAOptions
 	*KCCOptions
 	*SPDOptions
+	*NPDOptions
 	*LifeCycleOptions
 	*MonitorOptions
 	*OvercommitOptions
@@ -40,6 +41,7 @@ func NewControllersOptions() *ControllersOptions {
 		VPAOptions:                 NewVPAOptions(),
 		KCCOptions:                 NewKCCOptions(),
 		SPDOptions:                 NewSPDOptions(),
+		NPDOptions:                 NewNPDOptions(),
 		LifeCycleOptions:           NewLifeCycleOptions(),
 		MonitorOptions:             NewMonitorOptions(),
 		OvercommitOptions:          NewOvercommitOptions(),
@@ -52,6 +54,7 @@ func (o *ControllersOptions) AddFlags(fss *cliflag.NamedFlagSets) {
 	o.VPAOptions.AddFlags(fss)
 	o.KCCOptions.AddFlags(fss)
 	o.SPDOptions.AddFlags(fss)
+	o.NPDOptions.AddFlags(fss)
 	o.LifeCycleOptions.AddFlags(fss)
 	o.MonitorOptions.AddFlags(fss)
 	o.OvercommitOptions.AddFlags(fss)
@@ -66,6 +69,7 @@ func (o *ControllersOptions) ApplyTo(c *controllerconfig.ControllersConfiguratio
 	errList = append(errList, o.VPAOptions.ApplyTo(c.VPAConfig))
 	errList = append(errList, o.KCCOptions.ApplyTo(c.KCCConfig))
 	errList = append(errList, o.SPDOptions.ApplyTo(c.SPDConfig))
+	errList = append(errList, o.NPDOptions.ApplyTo(c.NPDConfig))
 	errList = append(errList, o.LifeCycleOptions.ApplyTo(c.LifeCycleConfig))
 	errList = append(errList, o.MonitorOptions.ApplyTo(c.MonitorConfig))
 	errList = append(errList, o.OvercommitOptions.ApplyTo(c.OvercommitConfig))

cmd/katalyst-controller/app/options/npd.go

@@ -0,0 +1,64 @@
+/*
+Copyright 2022 The Katalyst Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	cliflag "k8s.io/component-base/cli/flag"
+
+	"github.com/kubewharf/katalyst-core/pkg/config/controller"
+)
+
+type NPDOptions struct {
+	NPDMetricsPlugins     []string
+	EnableScopeDuplicated bool
+	SyncWorkers           int
+}
+
+func NewNPDOptions() *NPDOptions {
+	return &NPDOptions{
+		NPDMetricsPlugins:     []string{},
+		EnableScopeDuplicated: false,
+		SyncWorkers:           1,
+	}
+}
+
+func (o *NPDOptions) AddFlags(fss *cliflag.NamedFlagSets) {
+	fs := fss.FlagSet("npd")
+
+	fs.StringSliceVar(&o.NPDMetricsPlugins, "npd-metrics-plugins", o.NPDMetricsPlugins,
+		"A list  of metrics plugins to be used")
+	fs.BoolVar(&o.EnableScopeDuplicated, "npd-enable-scope-duplicated", o.EnableScopeDuplicated,
+		"Whether metrics with the same scope can be updated by multiple plugins")
+	fs.IntVar(&o.SyncWorkers, "npd-sync-workers", o.SyncWorkers,
+		"Number of workers to sync npd status")
+}
+
+func (o *NPDOptions) ApplyTo(c *controller.NPDConfig) error {
+	c.NPDMetricsPlugins = o.NPDMetricsPlugins
+	c.EnableScopeDuplicated = o.EnableScopeDuplicated
+	c.SyncWorkers = o.SyncWorkers
+	return nil
+}
+
+func (o *NPDOptions) Config() (*controller.NPDConfig, error) {
+	c := &controller.NPDConfig{}
+	if err := o.ApplyTo(c); err != nil {
+		return nil, err
+	}
+
+	return c, nil
+}

pkg/client/control/npd.go

@@ -0,0 +1,77 @@
+/*
+Copyright 2022 The Katalyst Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package control
+
+import (
+	"context"
+	"fmt"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/kubewharf/katalyst-api/pkg/apis/node/v1alpha1"
+	clientset "github.com/kubewharf/katalyst-api/pkg/client/clientset/versioned"
+)
+
+type NodeProfileControl interface {
+	CreateNPD(ctx context.Context, npd *v1alpha1.NodeProfileDescriptor, opts metav1.CreateOptions) (*v1alpha1.NodeProfileDescriptor, error)
+	UpdateNPDStatus(ctx context.Context, npd *v1alpha1.NodeProfileDescriptor, opts metav1.UpdateOptions) (*v1alpha1.NodeProfileDescriptor, error)
+	DeleteNPD(ctx context.Context, npdName string, opts metav1.DeleteOptions) error
+}
+
+type DummyNPDControl struct{}
+
+func (d *DummyNPDControl) CreateNPD(ctx context.Context, npd *v1alpha1.NodeProfileDescriptor, opts metav1.CreateOptions) (*v1alpha1.NodeProfileDescriptor, error) {
+	return nil, nil
+}
+
+func (d *DummyNPDControl) UpdateNPDStatus(ctx context.Context, npd *v1alpha1.NodeProfileDescriptor, opts metav1.UpdateOptions) (*v1alpha1.NodeProfileDescriptor, error) {
+	return nil, nil
+}
+
+func (d *DummyNPDControl) DeleteNPD(ctx context.Context, npdName string, opts metav1.DeleteOptions) error {
+	return nil
+}
+
+type NPDControlImp struct {
+	client clientset.Interface
+}
+
+func NewNPDControlImp(client clientset.Interface) *NPDControlImp {
+	return &NPDControlImp{
+		client: client,
+	}
+}
+
+func (n *NPDControlImp) CreateNPD(ctx context.Context, npd *v1alpha1.NodeProfileDescriptor, opts metav1.CreateOptions) (*v1alpha1.NodeProfileDescriptor, error) {
+	if npd == nil {
+		return nil, fmt.Errorf("npd is nil")
+	}
+
+	return n.client.NodeV1alpha1().NodeProfileDescriptors().Create(ctx, npd, opts)
+}
+
+func (n *NPDControlImp) UpdateNPDStatus(ctx context.Context, npd *v1alpha1.NodeProfileDescriptor, opts metav1.UpdateOptions) (*v1alpha1.NodeProfileDescriptor, error) {
+	if npd == nil {
+		return nil, fmt.Errorf("npd is nil")
+	}
+
+	return n.client.NodeV1alpha1().NodeProfileDescriptors().UpdateStatus(ctx, npd, opts)
+}
+
+func (n *NPDControlImp) DeleteNPD(ctx context.Context, npdName string, opts metav1.DeleteOptions) error {
+	return n.client.NodeV1alpha1().NodeProfileDescriptors().Delete(ctx, npdName, opts)
+}

pkg/config/controller/controller_base.go

@@ -46,6 +46,7 @@ type ControllersConfiguration struct {
 	*VPAConfig
 	*KCCConfig
 	*SPDConfig
+	*NPDConfig
 	*LifeCycleConfig
 	*MonitorConfig
 	*OvercommitConfig
@@ -63,6 +64,7 @@ func NewControllersConfiguration() *ControllersConfiguration {
 		VPAConfig:                 NewVPAConfig(),
 		KCCConfig:                 NewKCCConfig(),
 		SPDConfig:                 NewSPDConfig(),
+		NPDConfig:                 NewNPDConfig(),
 		LifeCycleConfig:           NewLifeCycleConfig(),
 		MonitorConfig:             NewMonitorConfig(),
 		OvercommitConfig:          NewOvercommitConfig(),

pkg/config/controller/npd.go

@@ -0,0 +1,33 @@
+/*
+Copyright 2022 The Katalyst Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+type NPDConfig struct {
+	NPDMetricsPlugins []string
+
+	EnableScopeDuplicated bool
+
+	SyncWorkers int
+}
+
+func NewNPDConfig() *NPDConfig {
+	return &NPDConfig{
+		NPDMetricsPlugins:     []string{},
+		EnableScopeDuplicated: false,
+		SyncWorkers:           1,
+	}
+}

pkg/controller/npd/handler.go

@@ -0,0 +1,73 @@
+/*
+Copyright 2022 The Katalyst Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package npd
+
+import (
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/klog/v2"
+)
+
+func (nc *NPDController) onNodeAdd(obj interface{}) {
+	node, ok := obj.(*v1.Node)
+	if !ok {
+		klog.Errorf("[npd] cannot convert obj to *v1.node")
+		return
+	}
+
+	nc.enqueueNode(node)
+}
+
+func (nc *NPDController) onNodeUpdate(_, newObj interface{}) {
+	node, ok := newObj.(*v1.Node)
+	if !ok {
+		klog.Errorf("[npd] cannot convert obj to *v1.node")
+		return
+	}
+
+	nc.enqueueNode(node)
+}
+
+func (nc *NPDController) onNodeDelete(obj interface{}) {
+	node, ok := obj.(*v1.Node)
+	if !ok {
+		klog.Errorf("[npd] cannot convert obj to *v1.node")
+		return
+	}
+
+	err := nc.deleteNPD(node.Name)
+	if err != nil {
+		klog.Errorf("delete node %v fail: %v", node.Name, err)
+		return
+	}
+	nc.metricsManager.DeleteNodeProfileStatus(node.Name)
+}
+
+func (nc *NPDController) enqueueNode(node *v1.Node) {
+	if node == nil {
+		klog.Warningf("[npd] enqueue a nil node")
+		return
+	}
+
+	key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(node)
+	if err != nil {
+		klog.Errorf("[npd] couldn't get key for node: %v, err: %v", node.Name, err)
+		return
+	}
+
+	nc.nodeQueue.Add(key)
+}