CP/DP Split: Collect telemetry for cp dp split (#3352)

Collect telemetry for number of data plane pods, control plane pods, and nginx proxy resources attached to a gateway.

Problem: With the control data plane split, there are a few telemetry metrics which need to be updated or added.

Solution: Update/Add telemetry metrics.

Author: bjee19

internal/mode/static/telemetry/collector.go

@@ -60,8 +60,10 @@ type Data struct {
 	// then lastly by directive string.
 	SnippetsFiltersDirectivesCount []int64
 	NGFResourceCounts              // embedding is required by the generator.
-	// NGFReplicaCount is the number of replicas of the NGF Pod.
-	NGFReplicaCount int64
+	// NginxPodCount is the total number of Nginx data plane Pods.
+	NginxPodCount int64
+	// ControlPlanePodCount is the total number of NGF control plane Pods.
+	ControlPlanePodCount int64
 }
 
 // NGFResourceCounts stores the counts of all relevant resources that NGF processes and generates configuration from.
@@ -99,6 +101,8 @@ type NGFResourceCounts struct {
 	SnippetsFilterCount int64
 	// UpstreamSettingsPolicyCount is the number of UpstreamSettingsPolicies.
 	UpstreamSettingsPolicyCount int64
+	// GatewayAttachedNpCount is the total number of NginxProxy resources that are attached to a Gateway.
+	GatewayAttachedNpCount int64
 }
 
 // DataCollectorConfig holds configuration parameters for DataCollectorImpl.
@@ -164,6 +168,8 @@ func (c DataCollectorImpl) Collect(ctx context.Context) (Data, error) {
 
 	snippetsFiltersDirectives, snippetsFiltersDirectivesCount := collectSnippetsFilterDirectives(g)
 
+	nginxPodCount := getNginxPodCount(g)
+
 	data := Data{
 		Data: tel.Data{
 			ProjectName:         "NGF",
@@ -179,9 +185,10 @@ func (c DataCollectorImpl) Collect(ctx context.Context) (Data, error) {
 		ImageSource:                    c.cfg.ImageSource,
 		FlagNames:                      c.cfg.Flags.Names,
 		FlagValues:                     c.cfg.Flags.Values,
-		NGFReplicaCount:                int64(replicaCount),
 		SnippetsFiltersDirectives:      snippetsFiltersDirectives,
 		SnippetsFiltersDirectivesCount: snippetsFiltersDirectivesCount,
+		NginxPodCount:                  nginxPodCount,
+		ControlPlanePodCount:           int64(replicaCount),
 	}
 
 	return data, nil
@@ -241,6 +248,18 @@ func collectGraphResourceCount(
 	ngfResourceCounts.NginxProxyCount = int64(len(g.ReferencedNginxProxies))
 	ngfResourceCounts.SnippetsFilterCount = int64(len(g.SnippetsFilters))
 
+	var gatewayAttachedNPCount int64
+	if g.GatewayClass != nil && g.GatewayClass.NginxProxy != nil {
+		gatewayClassNP := g.GatewayClass.NginxProxy
+		for _, np := range g.ReferencedNginxProxies {
+			if np != gatewayClassNP {
+				gatewayAttachedNPCount++
+			}
+		}
+	}
+
+	ngfResourceCounts.GatewayAttachedNpCount = gatewayAttachedNPCount
+
 	return ngfResourceCounts
 }
 
@@ -495,3 +514,22 @@ func parseDirectiveContextMapIntoLists(directiveContextMap map[sfDirectiveContex
 
 	return directiveContextList, countList
 }
+
+func getNginxPodCount(g *graph.Graph) int64 {
+	var count int64
+	for _, gateway := range g.Gateways {
+		replicas := int64(1)
+
+		np := gateway.EffectiveNginxProxy
+		if np != nil &&
+			np.Kubernetes != nil &&
+			np.Kubernetes.Deployment != nil &&
+			np.Kubernetes.Deployment.Replicas != nil {
+			replicas = int64(*np.Kubernetes.Deployment.Replicas)
+		}
+
+		count += replicas
+	}
+
+	return count
+}

internal/mode/static/telemetry/collector_test.go

@@ -18,6 +18,8 @@ import (
 	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
 
 	ngfAPI "github.com/nginx/nginx-gateway-fabric/apis/v1alpha1"
+	"github.com/nginx/nginx-gateway-fabric/apis/v1alpha2"
+	"github.com/nginx/nginx-gateway-fabric/internal/framework/helpers"
 	"github.com/nginx/nginx-gateway-fabric/internal/framework/kinds"
 	"github.com/nginx/nginx-gateway-fabric/internal/framework/kubernetes/kubernetesfakes"
 	"github.com/nginx/nginx-gateway-fabric/internal/mode/static/config"
@@ -170,7 +172,7 @@ var _ = Describe("Collector", Ordered, func() {
 				ClusterNodeCount:    1,
 			},
 			NGFResourceCounts:              telemetry.NGFResourceCounts{},
-			NGFReplicaCount:                1,
+			ControlPlanePodCount:           1,
 			ImageSource:                    "local",
 			FlagNames:                      flags.Names,
 			FlagValues:                     flags.Values,
@@ -262,6 +264,24 @@ var _ = Describe("Collector", Ordered, func() {
 
 				k8sClientReader.ListCalls(createListCallsFunc(nodes))
 
+				k8sClientReader.GetCalls(mergeGetCallsWithBase(createGetCallsFunc(
+					&appsv1.ReplicaSet{
+						Spec: appsv1.ReplicaSetSpec{
+							Replicas: helpers.GetPointer(int32(2)),
+						},
+						ObjectMeta: metav1.ObjectMeta{
+							Name: "replica",
+							OwnerReferences: []metav1.OwnerReference{
+								{
+									Kind: "Deployment",
+									Name: "Deployment1",
+									UID:  "test-uid-replicaSet",
+								},
+							},
+						},
+					},
+				)))
+
 				secret1 := &v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "secret1"}}
 				secret2 := &v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "secret2"}}
 				nilsecret := &v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "nilsecret"}}
@@ -270,11 +290,33 @@ var _ = Describe("Collector", Ordered, func() {
 				svc2 := &v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "svc2"}}
 				nilsvc := &v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "nilsvc"}}
 
+				gcNP := graph.NginxProxy{
+					Source:  nil,
+					ErrMsgs: nil,
+					Valid:   false,
+				}
+
 				graph := &graph.Graph{
-					GatewayClass: &graph.GatewayClass{},
+					GatewayClass: &graph.GatewayClass{NginxProxy: &gcNP},
 					Gateways: map[types.NamespacedName]*graph.Gateway{
-						{Name: "gateway1"}: {},
-						{Name: "gateway2"}: {},
+						{Name: "gateway1"}: {
+							EffectiveNginxProxy: &graph.EffectiveNginxProxy{
+								Kubernetes: &v1alpha2.KubernetesSpec{
+									Deployment: &v1alpha2.DeploymentSpec{
+										Replicas: helpers.GetPointer(int32(1)),
+									},
+								},
+							},
+						},
+						{Name: "gateway2"}: {
+							EffectiveNginxProxy: &graph.EffectiveNginxProxy{
+								Kubernetes: &v1alpha2.KubernetesSpec{
+									Deployment: &v1alpha2.DeploymentSpec{
+										Replicas: helpers.GetPointer(int32(3)),
+									},
+								},
+							},
+						},
 						{Name: "gateway3"}: {},
 					},
 					IgnoredGatewayClasses: map[types.NamespacedName]*gatewayv1.GatewayClass{
@@ -335,9 +377,11 @@ var _ = Describe("Collector", Ordered, func() {
 						}: {},
 					},
 					ReferencedNginxProxies: map[types.NamespacedName]*graph.NginxProxy{
-						{Namespace: "test", Name: "NginxProxy-1"}: {},
-						{Namespace: "test", Name: "NginxProxy-2"}: {},
-					}, SnippetsFilters: map[types.NamespacedName]*graph.SnippetsFilter{
+						{Namespace: "test", Name: "NginxProxy-1"}: &gcNP,
+						{Namespace: "test", Name: "NginxProxy-2"}: {Valid: true},
+						{Namespace: "test", Name: "NginxProxy-3"}: {Valid: true},
+					},
+					SnippetsFilters: map[types.NamespacedName]*graph.SnippetsFilter{
 						{Namespace: "test", Name: "sf-1"}: {
 							Snippets: map[ngfAPI.NginxContext]string{
 								ngfAPI.NginxContextMain:               "worker_priority 0;",
@@ -432,9 +476,10 @@ var _ = Describe("Collector", Ordered, func() {
 					GatewayAttachedClientSettingsPolicyCount: 1,
 					RouteAttachedClientSettingsPolicyCount:   2,
 					ObservabilityPolicyCount:                 1,
-					NginxProxyCount:                          2,
+					NginxProxyCount:                          3,
 					SnippetsFilterCount:                      3,
 					UpstreamSettingsPolicyCount:              1,
+					GatewayAttachedNpCount:                   2,
 				}
 				expData.ClusterVersion = "1.29.2"
 				expData.ClusterPlatform = "kind"
@@ -462,6 +507,11 @@ var _ = Describe("Collector", Ordered, func() {
 					1,
 				}
 
+				// one gateway with one replica + one gateway with three replicas + one gateway with replica field
+				// empty
+				expData.NginxPodCount = int64(5)
+				expData.ControlPlanePodCount = int64(2)
+
 				data, err := dataCollector.Collect(ctx)
 				Expect(err).ToNot(HaveOccurred())
 
@@ -593,7 +643,7 @@ var _ = Describe("Collector", Ordered, func() {
 			svc := &v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "svc1"}}
 
 			graph1 = &graph.Graph{
-				GatewayClass: &graph.GatewayClass{},
+				GatewayClass: &graph.GatewayClass{NginxProxy: &graph.NginxProxy{Valid: true}},
 				Gateways: map[types.NamespacedName]*graph.Gateway{
 					{Name: "gateway1"}: {},
 				},
@@ -634,12 +684,14 @@ var _ = Describe("Collector", Ordered, func() {
 					}: {},
 				},
 				ReferencedNginxProxies: map[types.NamespacedName]*graph.NginxProxy{
-					{Namespace: "test", Name: "NginxProxy-1"}: {},
-					{Namespace: "test", Name: "NginxProxy-2"}: {},
+					{Namespace: "test", Name: "NginxProxy-1"}: {Valid: true},
 				},
 				SnippetsFilters: map[types.NamespacedName]*graph.SnippetsFilter{
 					{Namespace: "test", Name: "sf-1"}: {},
 				},
+				BackendTLSPolicies: map[types.NamespacedName]*graph.BackendTLSPolicy{
+					{Namespace: "test", Name: "BackendTLSPolicy-1"}: {},
+				},
 			}
 
 			config1 = []*dataplane.Configuration{
@@ -716,10 +768,13 @@ var _ = Describe("Collector", Ordered, func() {
 					GatewayAttachedClientSettingsPolicyCount: 1,
 					RouteAttachedClientSettingsPolicyCount:   1,
 					ObservabilityPolicyCount:                 1,
-					NginxProxyCount:                          2,
+					NginxProxyCount:                          1,
 					SnippetsFilterCount:                      1,
 					UpstreamSettingsPolicyCount:              1,
+					GatewayAttachedNpCount:                   1,
+					BackendTLSPolicyCount:                    1,
 				}
+				expData.NginxPodCount = 1
 
 				data, err := dataCollector.Collect(ctx)
 

internal/mode/static/telemetry/data.avdl

@@ -102,8 +102,14 @@ attached at the Gateway level. */
 		/** UpstreamSettingsPolicyCount is the number of UpstreamSettingsPolicies. */
 		long? UpstreamSettingsPolicyCount = null;
 
-		/** NGFReplicaCount is the number of replicas of the NGF Pod. */
-		long? NGFReplicaCount = null;
+		/** GatewayAttachedNpCount is the total number of NginxProxy resources that are attached to a Gateway. */
+		long? GatewayAttachedNpCount = null;
+		
+		/** NginxPodCount is the total number of Nginx data plane Pods. */
+		long? NginxPodCount = null;
+		
+		/** ControlPlanePodCount is the total number of NGF control plane Pods. */
+		long? ControlPlanePodCount = null;
 
 	}
 }

internal/mode/static/telemetry/data_attributes_generated.go

@@ -20,7 +20,8 @@ func (d *Data) Attributes() []attribute.KeyValue {
 	attrs = append(attrs, attribute.StringSlice("SnippetsFiltersDirectives", d.SnippetsFiltersDirectives))
 	attrs = append(attrs, attribute.Int64Slice("SnippetsFiltersDirectivesCount", d.SnippetsFiltersDirectivesCount))
 	attrs = append(attrs, d.NGFResourceCounts.Attributes()...)
-	attrs = append(attrs, attribute.Int64("NGFReplicaCount", d.NGFReplicaCount))
+	attrs = append(attrs, attribute.Int64("NginxPodCount", d.NginxPodCount))
+	attrs = append(attrs, attribute.Int64("ControlPlanePodCount", d.ControlPlanePodCount))
 
 	return attrs
 }

internal/mode/static/telemetry/data_test.go

@@ -40,10 +40,12 @@ func TestDataAttributes(t *testing.T) {
 			NginxProxyCount:                          12,
 			SnippetsFilterCount:                      13,
 			UpstreamSettingsPolicyCount:              14,
+			GatewayAttachedNpCount:                   15,
 		},
-		NGFReplicaCount:                3,
 		SnippetsFiltersDirectives:      []string{"main-three-count", "http-two-count", "server-one-count"},
 		SnippetsFiltersDirectivesCount: []int64{3, 2, 1},
+		NginxPodCount:                  3,
+		ControlPlanePodCount:           3,
 	}
 
 	expected := []attribute.KeyValue{
@@ -79,7 +81,9 @@ func TestDataAttributes(t *testing.T) {
 		attribute.Int64("NginxProxyCount", 12),
 		attribute.Int64("SnippetsFilterCount", 13),
 		attribute.Int64("UpstreamSettingsPolicyCount", 14),
-		attribute.Int64("NGFReplicaCount", 3),
+		attribute.Int64("GatewayAttachedNpCount", 15),
+		attribute.Int64("NginxPodCount", 3),
+		attribute.Int64("ControlPlanePodCount", 3),
 	}
 
 	result := data.Attributes()
@@ -122,7 +126,9 @@ func TestDataAttributesWithEmptyData(t *testing.T) {
 		attribute.Int64("NginxProxyCount", 0),
 		attribute.Int64("SnippetsFilterCount", 0),
 		attribute.Int64("UpstreamSettingsPolicyCount", 0),
-		attribute.Int64("NGFReplicaCount", 0),
+		attribute.Int64("GatewayAttachedNpCount", 0),
+		attribute.Int64("NginxPodCount", 0),
+		attribute.Int64("ControlPlanePodCount", 0),
 	}
 
 	result := data.Attributes()

internal/mode/static/telemetry/ngfresourcecounts_attributes_generated.go

@@ -27,6 +27,7 @@ func (d *NGFResourceCounts) Attributes() []attribute.KeyValue {
 	attrs = append(attrs, attribute.Int64("NginxProxyCount", d.NginxProxyCount))
 	attrs = append(attrs, attribute.Int64("SnippetsFilterCount", d.SnippetsFilterCount))
 	attrs = append(attrs, attribute.Int64("UpstreamSettingsPolicyCount", d.UpstreamSettingsPolicyCount))
+	attrs = append(attrs, attribute.Int64("GatewayAttachedNpCount", d.GatewayAttachedNpCount))
 
 	return attrs
 }

tests/suite/telemetry_test.go

@@ -92,7 +92,9 @@ var _ = Describe("Telemetry test with OTel collector", Label("telemetry"), func(
 				"NginxProxyCount: Int(1)",
 				"SnippetsFilterCount: Int(0)",
 				"UpstreamSettingsPolicyCount: Int(0)",
-				"NGFReplicaCount: Int(1)",
+				"GatewayAttachedNpCount: Int(0)",
+				"NginxPodCount: Int(0)",
+				"ControlPlanePodCount: Int(1)",
 			},
 		)
 	})