Add new Enforceable interface for subset of strategies

A new enforceable interface divides out some of the behavior of the
strategies allowing those strategies that are assessed and actioned
in a timed loop - like deschedule - to be handled under a new
interface that covers their enforcement and their cleanup when
the underlying policy is deleted.

Co-authored-by:      togashidm <denisio.togashi@intel.com>
Co-authored-by: killianmuldoon <killian.muldoon@intel.com>

Author: 2 people

telemetry-aware-scheduling/pkg/strategies/core/enforcer.go

@@ -71,6 +71,13 @@ func (e *MetricEnforcer) RemoveStrategy(str Interface, strategyType string) {
 			klog.V(2).InfoS(msg, "component", "controller")
 		}
 	}
+	if enf, ok := str.(Enforceable); ok {
+		err := enf.Cleanup(e, str.GetPolicyName())
+		if err != nil {
+			msg := fmt.Sprintf("cleaning up the strategy failed: %v", err)
+			klog.V(2).InfoS(msg, "component", "controller")
+		}
+	}
 }
 
 //AddStrategy includes the specific strategy under its type in the strategy registry.
@@ -87,8 +94,10 @@ func (e *MetricEnforcer) AddStrategy(str Interface, strategyType string) {
 	msg := fmt.Sprintf("Adding strategies: %v %v", str.StrategyType(), str.GetPolicyName())
 	klog.V(2).InfoS(msg, "component", "controller")
 	if _, ok := e.RegisteredStrategies[strategyType]; ok {
-		e.RegisteredStrategies[strategyType][str] = nil
-		return
+		if _, ok := str.(Enforceable); ok {
+			e.RegisteredStrategies[strategyType][str] = nil
+			return
+		}
 	}
 }
 
@@ -109,10 +118,12 @@ func (e *MetricEnforcer) enforceStrategy(strategyType string, cache cache.Reader
 	strList, ok := e.RegisteredStrategies[strategyType]
 	if ok {
 		for str := range strList {
-			_, err := str.Enforce(e, cache)
-			if err != nil {
-				msg := fmt.Sprintf("Enforce the strategy failed: %v", err)
-				klog.V(2).InfoS(msg, "component", "controller")
+			if enf, ok := str.(Enforceable); ok {
+				_, err := enf.Enforce(e, cache)
+				if err != nil {
+					msg := fmt.Sprintf("Enforce the strategy failed: %v", err)
+					klog.V(2).InfoS(msg, "component", "controller")
+				}
 			}
 		}
 	}

telemetry-aware-scheduling/pkg/strategies/core/types.go

@@ -11,12 +11,16 @@ import (
 //Interface describes expected behavior of a specific strategy.
 type Interface interface {
 	Violated(cache cache.Reader) map[string]interface{}
-	Enforce(enforcer *MetricEnforcer, cache cache.Reader) (int, error)
 	StrategyType() string
 	Equals(Interface) bool
 	GetPolicyName() string
 	SetPolicyName(string)
 }
+//Enforceable enforce strategies and clean up after strategies are removed
+type Enforceable interface {
+	Enforce(enforcer *MetricEnforcer, cache cache.Reader) (int, error)
+	Cleanup(enforcer *MetricEnforcer, policyName string) error
+}
 
 //Enforcer registers strategies by type, adds specific strategies to a registry, and Enforces those strategies.
 type Enforcer interface {

telemetry-aware-scheduling/pkg/strategies/deschedule/enforce.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"k8s.io/apimachinery/pkg/labels"
 
 	"github.com/intel/platform-aware-scheduling/telemetry-aware-scheduling/pkg/cache"
 	strategy "github.com/intel/platform-aware-scheduling/telemetry-aware-scheduling/pkg/strategies/core"
@@ -21,6 +22,32 @@ type patchValue struct {
 	Path  string `json:"path"`
 	Value string `json:"value"`
 }
+//Cleanup remove node labels for violating when policy is deleted
+func (d *Strategy) Cleanup(enforcer *strategy.MetricEnforcer, policyName string) error {
+	lbls := metav1.LabelSelector{MatchLabels: map[string]string{policyName: "violating"}}
+	nodes, err := enforcer.KubeClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{LabelSelector: labels.Set(lbls.MatchLabels).String()})
+	if err != nil {
+		msg := fmt.Sprintf("cannot list nodes: %v", err)
+		klog.V(2).InfoS(msg, "component", "controller")
+		return err
+	}
+	for _, node := range nodes.Items {
+		var payload []patchValue
+		if _, ok := node.Labels[policyName]; ok {
+			payload = append(payload,
+				patchValue{
+					Op:   "remove",
+					Path: "/metadata/labels/" + policyName,
+				})
+		}
+		err := d.patchNode(node.Name, enforcer, payload)
+			if err != nil {
+				klog.V(2).InfoS(err.Error(), "component", "controller")
+			}
+	}
+	klog.V(2).InfoS(fmt.Sprintf("Unlabel node that was violating the policy %v", policyName),"component", "controller")
+	return nil
+}
 
 //Enforce describes the behavior followed by this strategy to return associated pods to non-violating status.
 //For descheduling enforcement is done by labelling the nodes as violators. This label can then be used externally, for example by descheduler, to remedy the situation.