Merge pull request #772 from funnydreamwinz/dev/fix-util-based-headroom

fix(sysadvisor): get util based headroom per-numa

Author: luomingmeng

pkg/agent/sysadvisor/plugin/qosaware/resource/cpu/assembler/headroomassembler/assembler_common.go

@@ -162,6 +162,12 @@ func (ha *HeadroomAssemblerCommon) getHeadroomByUtil() (resource.Quantity, map[i
 		MaxCapacity:       dynamicConfig.MaxHeadroomCapacityRate * float64(ha.metaServer.MachineInfo.NumCores/ha.metaServer.NumNUMANodes),
 	}
 
+	reclaimedCPUs, err := ha.getLastReclaimedCPUPerNUMA()
+	if err != nil {
+		general.Errorf("getLastReclaimedCPUPerNUMA failed: %v", err)
+		return resource.Quantity{}, nil, err
+	}
+
 	// get headroom per NUMA
 	for _, numaID := range bindingNUMAs {
 		cpuSet, ok := reclaimPoolInfo.TopologyAwareAssignments[numaID]
@@ -174,7 +180,9 @@ func (ha *HeadroomAssemblerCommon) getHeadroomByUtil() (resource.Quantity, map[i
 			return resource.Quantity{}, nil, fmt.Errorf("get reclaim Metrics failed with numa %d: %v", numaID, err)
 		}
 
-		headroom, err := ha.getUtilBasedHeadroom(options, reclaimMetrics)
+		lastReclaimedCPUPerNumaForCalculate := make(map[int]float64)
+		lastReclaimedCPUPerNumaForCalculate[numaID] = reclaimedCPUs[numaID]
+		headroom, err := ha.getUtilBasedHeadroom(options, reclaimMetrics, lastReclaimedCPUPerNumaForCalculate)
 		if err != nil {
 			return resource.Quantity{}, nil, fmt.Errorf("get util-based headroom failed with numa %d: %v", numaID, err)
 		}
@@ -186,13 +194,15 @@ func (ha *HeadroomAssemblerCommon) getHeadroomByUtil() (resource.Quantity, map[i
 	// get global reclaim headroom
 	if len(nonBindingNumas) > 0 {
 		cpusets := machine.NewCPUSet()
+		lastReclaimedCPUPerNumaForCalculate := make(map[int]float64)
 		for _, numaID := range nonBindingNumas {
 			cpuSet, ok := reclaimPoolInfo.TopologyAwareAssignments[numaID]
 			if !ok {
 				return resource.Quantity{}, nil, fmt.Errorf("reclaim pool NOT found TopologyAwareAssignments with numaID: %v", numaID)
 			}
 
 			cpusets = cpusets.Union(cpuSet)
+			lastReclaimedCPUPerNumaForCalculate[numaID] = reclaimedCPUs[numaID]
 		}
 
 		reclaimMetrics, err := metricHelper.GetReclaimMetrics(cpusets, ha.getReclaimCgroupPath(), ha.metaServer.MetricsFetcher)
@@ -201,7 +211,7 @@ func (ha *HeadroomAssemblerCommon) getHeadroomByUtil() (resource.Quantity, map[i
 		}
 
 		options.MaxCapacity *= float64(len(nonBindingNumas))
-		headroom, err := ha.getUtilBasedHeadroom(options, reclaimMetrics)
+		headroom, err := ha.getUtilBasedHeadroom(options, reclaimMetrics, lastReclaimedCPUPerNumaForCalculate)
 		if err != nil {
 			return resource.Quantity{}, nil, fmt.Errorf("get util-based headroom failed: %v", err)
 		}

pkg/agent/sysadvisor/plugin/qosaware/resource/cpu/assembler/headroomassembler/assembler_common_util.go

@@ -30,6 +30,7 @@ import (
 	"github.com/kubewharf/katalyst-core/pkg/agent/sysadvisor/plugin/qosaware/resource/helper"
 	"github.com/kubewharf/katalyst-core/pkg/agent/sysadvisor/types"
 	metaserverHelper "github.com/kubewharf/katalyst-core/pkg/metaserver/agent/metric/helper"
+	"github.com/kubewharf/katalyst-core/pkg/util"
 	"github.com/kubewharf/katalyst-core/pkg/util/general"
 	"github.com/kubewharf/katalyst-core/pkg/util/native"
 )
@@ -40,11 +41,8 @@ const (
 
 func (ha *HeadroomAssemblerCommon) getUtilBasedHeadroom(options helper.UtilBasedCapacityOptions,
 	reclaimMetrics *metaserverHelper.ReclaimMetrics,
+	lastReclaimedCPUPerNumaForCalculate map[int]float64,
 ) (resource.Quantity, error) {
-	lastReclaimedCPU, err := ha.getLastReclaimedCPU()
-	if err != nil {
-		return resource.Quantity{}, err
-	}
 	if reclaimMetrics == nil {
 		return resource.Quantity{}, fmt.Errorf("reclaimMetrics is nil")
 	}
@@ -54,10 +52,14 @@ func (ha *HeadroomAssemblerCommon) getUtilBasedHeadroom(options helper.UtilBased
 	}
 
 	util := reclaimMetrics.CgroupCPUUsage / reclaimMetrics.ReclaimedCoresSupply
+	lastReclaimedCPU := 0.0
+	for _, cpu := range lastReclaimedCPUPerNumaForCalculate {
+		lastReclaimedCPU += cpu
+	}
 
 	general.InfoS("getUtilBasedHeadroom", "reclaimedCoresSupply", reclaimMetrics.ReclaimedCoresSupply,
 		"util", util, "reclaim PoolCPUUsage", reclaimMetrics.PoolCPUUsage, "reclaim CgroupCPUUsage", reclaimMetrics.CgroupCPUUsage,
-		"lastReclaimedCPU", lastReclaimedCPU)
+		"lastReclaimedCPUPerNUMA", lastReclaimedCPUPerNumaForCalculate)
 
 	headroom, err := helper.EstimateUtilBasedCapacity(
 		options,
@@ -72,20 +74,13 @@ func (ha *HeadroomAssemblerCommon) getUtilBasedHeadroom(options helper.UtilBased
 	return *resource.NewQuantity(int64(math.Ceil(headroom)), resource.DecimalSI), nil
 }
 
-func (ha *HeadroomAssemblerCommon) getLastReclaimedCPU() (float64, error) {
+func (ha *HeadroomAssemblerCommon) getLastReclaimedCPUPerNUMA() (map[int]float64, error) {
 	cnr, err := ha.metaServer.CNRFetcher.GetCNR(context.Background())
 	if err != nil {
-		return 0, err
-	}
-
-	if cnr.Status.Resources.Allocatable != nil {
-		if reclaimedMilliCPU, ok := (*cnr.Status.Resources.Allocatable)[consts.ReclaimedResourceMilliCPU]; ok {
-			return float64(reclaimedMilliCPU.Value()) / 1000, nil
-		}
+		return nil, err
 	}
 
-	klog.Errorf("cnr status resource allocatable reclaimed milli cpu not found")
-	return 0, nil
+	return util.GetReclaimedCPUPerNUMA(cnr.Status.TopologyZone), nil
 }
 
 func (ha *HeadroomAssemblerCommon) getReclaimNUMABindingTopo(reclaimPool *types.PoolInfo) (bindingNUMAs, nonBindingNumas []int, err error) {

pkg/util/cnr_topology.go

@@ -19,10 +19,13 @@ package util
 import (
 	"fmt"
 	"sort"
+	"strconv"
 
 	v1 "k8s.io/api/core/v1"
+	"k8s.io/klog/v2"
 
 	nodev1alpha1 "github.com/kubewharf/katalyst-api/pkg/apis/node/v1alpha1"
+	"github.com/kubewharf/katalyst-api/pkg/consts"
 	"github.com/kubewharf/katalyst-core/pkg/config/generic"
 	"github.com/kubewharf/katalyst-core/pkg/util/qos"
 )
@@ -212,3 +215,35 @@ func ValidateSharedCoresWithNumaBindingPod(qosConf *generic.QoSConfiguration, po
 
 	return true, nil
 }
+
+func GetReclaimedCPUPerNUMA(topologyZones []*nodev1alpha1.TopologyZone) map[int]float64 {
+	numaMap := make(map[int]float64)
+	for _, topologyZone := range topologyZones {
+		if topologyZone.Type != nodev1alpha1.TopologyTypeSocket {
+			continue
+		}
+
+		for _, child := range topologyZone.Children {
+			if child.Type != nodev1alpha1.TopologyTypeNuma {
+				continue
+			}
+
+			numaID, err := strconv.Atoi(child.Name)
+			if err != nil {
+				klog.Errorf("invalid numa name: %v, %v", child.Name, err)
+				continue
+			}
+
+			if child.Resources.Allocatable == nil {
+				klog.Errorf("numa zone without allocatable resource: %d", numaID)
+				continue
+			}
+
+			if reclaimedMilliCPU, ok := (*child.Resources.Allocatable)[consts.ReclaimedResourceMilliCPU]; ok {
+				numaMap[numaID] = float64(reclaimedMilliCPU.Value()) / 1000
+			}
+		}
+	}
+
+	return numaMap
+}