Skip to content

Commit 7d0307a

Browse files
terassyiterassyi
authored
Introduce egress metrics to count invalid packets and bytes (#331)
* Introduce egress metrics to count invalid packets and bytes Signed-off-by: terashima <tomoya-terashima@cybozu.co.jp> * register metrics at initialization Signed-off-by: Hiroki Hanada <hiroki-hanada@cybozu.co.jp> * Add documents for added metrics Signed-off-by: terashima <tomoya-terashima@cybozu.co.jp> --------- Signed-off-by: terashima <tomoya-terashima@cybozu.co.jp> Signed-off-by: Hiroki Hanada <hiroki-hanada@cybozu.co.jp>
1 parent 314fc3a commit 7d0307a

File tree

4 files changed

+93
-18
lines changed

4 files changed

+93
-18
lines changed

‎docs/cmd-coil-egress.md‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,3 +95,26 @@ This value is from the result of `iptables -t nat -L POSTROUTING -vn`.
9595
| `namespace` | The egress resource namespace |
9696
| `egress` | The egress resource name |
9797
| `pod` | The pod name |
98+
99+
### `coil_egress_nftables_invalid_packets_total`
100+
101+
This is the total number of packets dropped as invalid packets by iptables in a egress NAT pod.
102+
This value is from the result of `iptables -t filter -L -vn`.
103+
104+
| Label | Description |
105+
| ----------- | ----------------------------- |
106+
| `namespace` | The egress resource namespace |
107+
| `egress` | The egress resource name |
108+
| `pod` | The pod name |
109+
110+
### `coil_egress_nftables_invalid_bytes_total`
111+
112+
This is the total bytes of packets dropped as invalid packets by iptables in a egress NAT pod.
113+
This value is from the result of `iptables -t filter -L -vn`.
114+
115+
| Label | Description |
116+
| ----------- | ----------------------------- |
117+
| `namespace` | The egress resource namespace |
118+
| `egress` | The egress resource name |
119+
| `pod` | The pod name |
120+

‎v2/cmd/coil-egress/sub/run.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@ func init() {
4141
metrics.Registry.MustRegister(egressMetrics.NfConnctackLimit)
4242
metrics.Registry.MustRegister(egressMetrics.NfTableMasqueradeBytes)
4343
metrics.Registry.MustRegister(egressMetrics.NfTableMasqueradePackets)
44+
metrics.Registry.MustRegister(egressMetrics.NfTableInvalidBytes)
45+
metrics.Registry.MustRegister(egressMetrics.NfTableInvalidPackets)
4446
}
4547

4648
func subMain() error {

‎v2/pkg/metrics/collector.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import (
99
)
1010

1111
type Collector interface {
12-
Update() error
12+
Update(ctx context.Context) error
1313
Name() string
1414
}
1515

@@ -44,7 +44,7 @@ func (r *Runner) collect(ctx context.Context) {
4444
wg.Add(len(r.collectors))
4545
for _, c := range r.collectors {
4646
go func(c Collector) {
47-
if err := c.Update(); err != nil {
47+
if err := c.Update(ctx); err != nil {
4848
logger.Error(err, "failed to collect metrics", "name", c.Name())
4949
}
5050
wg.Done()

‎v2/pkg/metrics/egress.go‎

Lines changed: 66 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package metrics
22

33
import (
4+
"context"
45
"errors"
56
"os"
67
"strconv"
@@ -45,41 +46,61 @@ var (
4546
Name: "nftables_masqueraded_bytes_total",
4647
Help: "the number of bytes that are masqueraded by nftables",
4748
}, []string{"namespace", "pod", "egress"})
49+
50+
NfTableInvalidPackets = prometheus.NewGaugeVec(prometheus.GaugeOpts{
51+
Namespace: constants.MetricsNS,
52+
Subsystem: "egress",
53+
Name: "nftables_invalid_packets_total",
54+
Help: "the number of packets that are dropped as invalid packets by nftables",
55+
}, []string{"namespace", "pod", "egress"})
56+
57+
NfTableInvalidBytes = prometheus.NewGaugeVec(prometheus.GaugeOpts{
58+
Namespace: constants.MetricsNS,
59+
Subsystem: "egress",
60+
Name: "nftables_invalid_bytes_total",
61+
Help: "the number of bytes that are dropped as invalid packets by nftables",
62+
}, []string{"namespace", "pod", "egress"})
4863
)
4964

5065
type egressCollector struct {
51-
conn *nftables.Conn
52-
nfConnctackCount prometheus.Gauge
53-
nfConnctackLimit prometheus.Gauge
54-
nfTablesPackets prometheus.Gauge
55-
nfTablesBytes prometheus.Gauge
66+
conn *nftables.Conn
67+
nfConnctackCount prometheus.Gauge
68+
nfConnctackLimit prometheus.Gauge
69+
nfTablesNATPackets prometheus.Gauge
70+
nfTablesNATBytes prometheus.Gauge
71+
nfTablesInvalidPackets prometheus.Gauge
72+
nfTablesInvalidBytes prometheus.Gauge
5673
}
5774

5875
func NewEgressCollector(ns, pod, egress string) (Collector, error) {
5976
NfConnctackCount.Reset()
6077
NfConnctackLimit.Reset()
6178
NfTableMasqueradeBytes.Reset()
6279
NfTableMasqueradePackets.Reset()
80+
NfTableInvalidPackets.Reset()
81+
NfTableInvalidBytes.Reset()
6382

6483
c, err := nftables.New()
6584
if err != nil {
6685
return nil, err
6786
}
6887

6988
return &egressCollector{
70-
conn: c,
71-
nfConnctackCount: NfConnctackCount.WithLabelValues(ns, pod, egress),
72-
nfConnctackLimit: NfConnctackLimit.WithLabelValues(ns, pod, egress),
73-
nfTablesPackets: NfTableMasqueradePackets.WithLabelValues(ns, pod, egress),
74-
nfTablesBytes: NfTableMasqueradeBytes.WithLabelValues(ns, pod, egress),
89+
conn: c,
90+
nfConnctackCount: NfConnctackCount.WithLabelValues(ns, pod, egress),
91+
nfConnctackLimit: NfConnctackLimit.WithLabelValues(ns, pod, egress),
92+
nfTablesNATPackets: NfTableMasqueradePackets.WithLabelValues(ns, pod, egress),
93+
nfTablesNATBytes: NfTableMasqueradeBytes.WithLabelValues(ns, pod, egress),
94+
nfTablesInvalidPackets: NfTableInvalidPackets.WithLabelValues(ns, pod, egress),
95+
nfTablesInvalidBytes: NfTableInvalidBytes.WithLabelValues(ns, pod, egress),
7596
}, nil
7697
}
7798

7899
func (c *egressCollector) Name() string {
79100
return "egress-collector"
80101
}
81102

82-
func (c *egressCollector) Update() error {
103+
func (c *egressCollector) Update(ctx context.Context) error {
83104

84105
val, err := readUintFromFile(NF_CONNTRACK_COUNT_PATH)
85106
if err != nil {
@@ -93,17 +114,24 @@ func (c *egressCollector) Update() error {
93114
}
94115
c.nfConnctackLimit.Set(float64(val))
95116

96-
packets, bytes, err := c.getNfTablesCounter()
117+
natPackets, natBytes, err := c.getNfTablesNATCounter()
97118
if err != nil {
98-
return nil
119+
return err
99120
}
100-
c.nfTablesPackets.Set(float64(packets))
101-
c.nfTablesBytes.Set(float64(bytes))
121+
c.nfTablesNATPackets.Set(float64(natPackets))
122+
c.nfTablesNATBytes.Set(float64(natBytes))
123+
124+
invalidPackets, invalidBytes, err := c.getNfTablesInvalidCounter()
125+
if err != nil {
126+
return err
127+
}
128+
c.nfTablesInvalidPackets.Set(float64(invalidPackets))
129+
c.nfTablesInvalidBytes.Set(float64(invalidBytes))
102130

103131
return nil
104132
}
105133

106-
func (c *egressCollector) getNfTablesCounter() (uint64, uint64, error) {
134+
func (c *egressCollector) getNfTablesNATCounter() (uint64, uint64, error) {
107135
table := &nftables.Table{Family: nftables.TableFamilyIPv4, Name: "nat"}
108136
rules, err := c.conn.GetRules(table, &nftables.Chain{
109137
Name: "POSTROUTING",
@@ -125,6 +153,28 @@ func (c *egressCollector) getNfTablesCounter() (uint64, uint64, error) {
125153
return 0, 0, errors.New("a masquerade rule is not found")
126154
}
127155

156+
func (c *egressCollector) getNfTablesInvalidCounter() (uint64, uint64, error) {
157+
table := &nftables.Table{Family: nftables.TableFamilyIPv4, Name: "filter"}
158+
rules, err := c.conn.GetRules(table, &nftables.Chain{
159+
Name: "FORWARD",
160+
Type: nftables.ChainTypeFilter,
161+
Table: table,
162+
Hooknum: nftables.ChainHookForward,
163+
})
164+
if err != nil {
165+
return 0, 0, err
166+
}
167+
for _, rule := range rules {
168+
for _, e := range rule.Exprs {
169+
if counter, ok := e.(*expr.Counter); ok {
170+
// A rule in the egress pod must be only one, so we can return by finding a first one.
171+
return counter.Packets, counter.Bytes, nil
172+
}
173+
}
174+
}
175+
return 0, 0, errors.New("a rule for invalid packets is not found")
176+
}
177+
128178
func readUintFromFile(path string) (uint64, error) {
129179
data, err := os.ReadFile(path)
130180
if err != nil {

0 commit comments

Comments
 (0)