Support IPv6 for egress metrics

Signed-off-by: terashima <tomoya-terashima@cybozu.co.jp>

Author: terassyi

v2/cmd/coil-egress/sub/run.go

@@ -63,18 +63,26 @@ func subMain() error {
 		return errors.New(constants.EnvAddresses + " environment variable must be set")
 	}
 	var ipv4, ipv6 net.IP
+	protocolMap := make(map[string]struct{})
 	for _, addr := range myAddresses {
 		n := net.ParseIP(addr)
 		if n == nil {
 			return errors.New(constants.EnvAddresses + "contains invalid address: " + addr)
 		}
 		if n4 := n.To4(); n4 != nil {
 			ipv4 = n4
+			protocolMap["ipv4"] = struct{}{}
 		} else {
 			ipv6 = n
+			protocolMap["ipv6"] = struct{}{}
 		}
 	}
 
+	protocols := make([]string, 0, 0)
+	for protocol, _ := range protocolMap {
+		protocols = append(protocols, protocol)
+	}
+
 	setupLog.Info("detected local IP addresses", "ipv4", ipv4.String(), "ipv6", ipv6.String())
 
 	timeout := gracefulTimeout
@@ -117,7 +125,7 @@ func subMain() error {
 
 	setupLog.Info("setup egress metrics collector")
 	runner := egressMetrics.NewRunner()
-	egressCollector, err := egressMetrics.NewEgressCollector(myNS, os.Getenv("HOSTNAME"), myName)
+	egressCollector, err := egressMetrics.NewEgressCollector(myNS, os.Getenv("HOSTNAME"), myName, protocols)
 	if err != nil {
 		return err
 	}

v2/pkg/metrics/egress.go

@@ -3,6 +3,7 @@ package metrics
 import (
 	"context"
 	"errors"
+	"fmt"
 	"os"
 	"strconv"
 	"strings"
@@ -24,55 +25,68 @@ var (
 		Subsystem: "egress",
 		Name:      "nf_conntrack_entries_count",
 		Help:      "the number of entries in the nf_conntrack table",
-	}, []string{"namespace", "pod", "egress"})
+	}, []string{"namespace", "pod", "egress", "protocol"})
 
 	NfConnctackLimit = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: constants.MetricsNS,
 		Subsystem: "egress",
 		Name:      "nf_conntrack_entries_limit",
 		Help:      "the limit of the nf_conntrack table",
-	}, []string{"namespace", "pod", "egress"})
+	}, []string{"namespace", "pod", "egress", "protocol"})
 
 	NfTableMasqueradePackets = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: constants.MetricsNS,
 		Subsystem: "egress",
 		Name:      "nftables_masqueraded_packets_total",
 		Help:      "the number of packets that are masqueraded by nftables",
-	}, []string{"namespace", "pod", "egress"})
+	}, []string{"namespace", "pod", "egress", "protocol"})
 
 	NfTableMasqueradeBytes = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: constants.MetricsNS,
 		Subsystem: "egress",
 		Name:      "nftables_masqueraded_bytes_total",
 		Help:      "the number of bytes that are masqueraded by nftables",
-	}, []string{"namespace", "pod", "egress"})
+	}, []string{"namespace", "pod", "egress", "protocol"})
 
 	NfTableInvalidPackets = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: constants.MetricsNS,
 		Subsystem: "egress",
 		Name:      "nftables_invalid_packets_total",
 		Help:      "the number of packets that are dropped as invalid packets by nftables",
-	}, []string{"namespace", "pod", "egress"})
+	}, []string{"namespace", "pod", "egress", "protocol"})
 
 	NfTableInvalidBytes = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: constants.MetricsNS,
 		Subsystem: "egress",
 		Name:      "nftables_invalid_bytes_total",
 		Help:      "the number of bytes that are dropped as invalid packets by nftables",
-	}, []string{"namespace", "pod", "egress"})
+	}, []string{"namespace", "pod", "egress", "protocol"})
 )
 
 type egressCollector struct {
-	conn                   *nftables.Conn
-	nfConnctackCount       prometheus.Gauge
-	nfConnctackLimit       prometheus.Gauge
+	conn             *nftables.Conn
+	nfConnctackCount prometheus.Gauge
+	nfConnctackLimit prometheus.Gauge
+	perProtocol      map[string]*nfTablesPerProtocolMetrics
+}
+
+type nfTablesPerProtocolMetrics struct {
 	nfTablesNATPackets     prometheus.Gauge
 	nfTablesNATBytes       prometheus.Gauge
 	nfTablesInvalidPackets prometheus.Gauge
 	nfTablesInvalidBytes   prometheus.Gauge
 }
 
-func NewEgressCollector(ns, pod, egress string) (Collector, error) {
+func newNfTablesPerProtocolMetrics(ns, pod, egress, protocol string) *nfTablesPerProtocolMetrics {
+	return &nfTablesPerProtocolMetrics{
+		nfTablesNATPackets:     NfTableMasqueradePackets.WithLabelValues(ns, pod, egress, protocol),
+		nfTablesNATBytes:       NfTableMasqueradeBytes.WithLabelValues(ns, pod, egress, protocol),
+		nfTablesInvalidPackets: NfTableInvalidPackets.WithLabelValues(ns, pod, egress, protocol),
+		nfTablesInvalidBytes:   NfTableInvalidBytes.WithLabelValues(ns, pod, egress, protocol),
+	}
+}
+
+func NewEgressCollector(ns, pod, egress string, protocols []string) (Collector, error) {
 	NfConnctackCount.Reset()
 	NfConnctackLimit.Reset()
 	NfTableMasqueradeBytes.Reset()
@@ -85,14 +99,16 @@ func NewEgressCollector(ns, pod, egress string) (Collector, error) {
 		return nil, err
 	}
 
+	perProtocols := make(map[string]*nfTablesPerProtocolMetrics)
+	for _, protocol := range protocols {
+		perProtocols[protocol] = newNfTablesPerProtocolMetrics(ns, pod, egress, protocol)
+	}
+
 	return &egressCollector{
-		conn:                   c,
-		nfConnctackCount:       NfConnctackCount.WithLabelValues(ns, pod, egress),
-		nfConnctackLimit:       NfConnctackLimit.WithLabelValues(ns, pod, egress),
-		nfTablesNATPackets:     NfTableMasqueradePackets.WithLabelValues(ns, pod, egress),
-		nfTablesNATBytes:       NfTableMasqueradeBytes.WithLabelValues(ns, pod, egress),
-		nfTablesInvalidPackets: NfTableInvalidPackets.WithLabelValues(ns, pod, egress),
-		nfTablesInvalidBytes:   NfTableInvalidBytes.WithLabelValues(ns, pod, egress),
+		conn:             c,
+		nfConnctackCount: NfConnctackCount.WithLabelValues(ns, pod, egress),
+		nfConnctackLimit: NfConnctackLimit.WithLabelValues(ns, pod, egress),
+		perProtocol:      perProtocols,
 	}, nil
 }
 
@@ -114,25 +130,32 @@ func (c *egressCollector) Update(ctx context.Context) error {
 	}
 	c.nfConnctackLimit.Set(float64(val))
 
-	natPackets, natBytes, err := c.getNfTablesNATCounter()
-	if err != nil {
-		return err
-	}
-	c.nfTablesNATPackets.Set(float64(natPackets))
-	c.nfTablesNATBytes.Set(float64(natBytes))
+	for protocol, nfTablesMetrics := range c.perProtocol {
+		natPackets, natBytes, err := c.getNfTablesNATCounter(protocol)
+		if err != nil {
+			return err
+		}
+		nfTablesMetrics.nfTablesNATPackets.Set(float64(natPackets))
+		nfTablesMetrics.nfTablesNATBytes.Set(float64(natBytes))
+
+		invalidPackets, invalidBytes, err := c.getNfTablesInvalidCounter(protocol)
+		if err != nil {
+			return err
+		}
+		nfTablesMetrics.nfTablesInvalidPackets.Set(float64(invalidPackets))
+		nfTablesMetrics.nfTablesInvalidBytes.Set(float64(invalidBytes))
 
-	invalidPackets, invalidBytes, err := c.getNfTablesInvalidCounter()
-	if err != nil {
-		return err
 	}
-	c.nfTablesInvalidPackets.Set(float64(invalidPackets))
-	c.nfTablesInvalidBytes.Set(float64(invalidBytes))
 
 	return nil
 }
 
-func (c *egressCollector) getNfTablesNATCounter() (uint64, uint64, error) {
-	table := &nftables.Table{Family: nftables.TableFamilyIPv4, Name: "nat"}
+func (c *egressCollector) getNfTablesNATCounter(protocol string) (uint64, uint64, error) {
+	family, err := stringToTableFamily(protocol)
+	if err != nil {
+		return 0, 0, err
+	}
+	table := &nftables.Table{Family: family, Name: "nat"}
 	rules, err := c.conn.GetRules(table, &nftables.Chain{
 		Name:    "POSTROUTING",
 		Type:    nftables.ChainTypeNAT,
@@ -153,8 +176,12 @@ func (c *egressCollector) getNfTablesNATCounter() (uint64, uint64, error) {
 	return 0, 0, errors.New("a masquerade rule is not found")
 }
 
-func (c *egressCollector) getNfTablesInvalidCounter() (uint64, uint64, error) {
-	table := &nftables.Table{Family: nftables.TableFamilyIPv4, Name: "filter"}
+func (c *egressCollector) getNfTablesInvalidCounter(protocol string) (uint64, uint64, error) {
+	family, err := stringToTableFamily(protocol)
+	if err != nil {
+		return 0, 0, err
+	}
+	table := &nftables.Table{Family: family, Name: "filter"}
 	rules, err := c.conn.GetRules(table, &nftables.Chain{
 		Name:    "FORWARD",
 		Type:    nftables.ChainTypeFilter,
@@ -186,3 +213,14 @@ func readUintFromFile(path string) (uint64, error) {
 	}
 	return val, nil
 }
+
+func stringToTableFamily(protocol string) (nftables.TableFamily, error) {
+	switch strings.ToLower(protocol) {
+	case "ipv4":
+		return nftables.TableFamilyIPv4, nil
+	case "ipv6":
+		return nftables.TableFamilyIPv6, nil
+	default:
+		return nftables.TableFamilyUnspecified, fmt.Errorf("unsupported family type: %s", protocol)
+	}
+}