From a14aba517b5df5a55eefb023f0fb3f7368c4b0fe Mon Sep 17 00:00:00 2001 From: etgraylog Date: Sun, 11 Feb 2024 20:09:43 -0500 Subject: [PATCH 01/12] Add struct for fileCacheMetric Signed-off-by: etgraylog --- collector/nodes.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/collector/nodes.go b/collector/nodes.go index 364c541d..1aae13b0 100644 --- a/collector/nodes.go +++ b/collector/nodes.go @@ -171,6 +171,13 @@ type filesystemIODeviceMetric struct { Labels func(cluster string, node NodeStatsNodeResponse, device string) []string } +type fileCacheMetric struct { + Type prometheus.ValueType + Desc *prometheus.Desc + Value func(fileCacheStats NodeStatsFileCacheResponse) float64 + Labels func(cluster string, node NodeStatsNodeResponse) []string +} + // Nodes information struct type Nodes struct { logger log.Logger From 3c035971f088daced455fce326e388a04eb7f96e Mon Sep 17 00:00:00 2001 From: etgraylog Date: Sun, 11 Feb 2024 20:10:09 -0500 Subject: [PATCH 02/12] Add search role to map of roles within getRoles method Signed-off-by: etgraylog --- collector/nodes.go | 1 + 1 file changed, 1 insertion(+) diff --git a/collector/nodes.go b/collector/nodes.go index 1aae13b0..a3e0f1a2 100644 --- a/collector/nodes.go +++ b/collector/nodes.go @@ -38,6 +38,7 @@ func getRoles(node NodeStatsNodeResponse) map[string]bool { "data_content": false, "ml": false, "remote_cluster_client": false, + "search": false, "transform": false, "ingest": false, "client": true, From 9676eea7bdfd9abf38d76de0e183a5561f2ee33f Mon Sep 17 00:00:00 2001 From: etgraylog Date: Sun, 11 Feb 2024 20:10:54 -0500 Subject: [PATCH 03/12] Add field for slice of fileCacheMetrics to Nodes struct Signed-off-by: etgraylog --- collector/nodes.go | 1 + 1 file changed, 1 insertion(+) diff --git a/collector/nodes.go b/collector/nodes.go index a3e0f1a2..c7f185e8 100644 --- a/collector/nodes.go +++ b/collector/nodes.go @@ -196,6 +196,7 @@ type Nodes struct { threadPoolMetrics []*threadPoolMetric filesystemDataMetrics []*filesystemDataMetric filesystemIODeviceMetrics []*filesystemIODeviceMetric + fileCacheMetrics []*fileCacheMetric } // NewNodes defines Nodes Prometheus metrics From f032cfe897bb17b20bfe3a449172ebb96dad6367 Mon Sep 17 00:00:00 2001 From: etgraylog Date: Sun, 11 Feb 2024 20:12:33 -0500 Subject: [PATCH 04/12] Add Gauge & Counter values of fileCacheStats to slices of FileCacheMetrics Signed-off-by: etgraylog --- collector/nodes.go | 98 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/collector/nodes.go b/collector/nodes.go index c7f185e8..b28f2812 100644 --- a/collector/nodes.go +++ b/collector/nodes.go @@ -1790,6 +1790,104 @@ func NewNodes(logger log.Logger, client *http.Client, url *url.URL, all bool, no Labels: defaultFilesystemIODeviceLabelValues, }, }, + fileCacheMetrics: []*fileCacheMetric{ + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "filecache", "active_in_bytes"), + "file_cache active memory in bytes", + defaultNodeLabels, nil, + ), + Value: func(fileCacheStats NodeStatsFileCacheResponse) float64 { + return float64(fileCacheStats.ActiveInBytes) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "filecache", "total_in_bytes"), + "file_cache total memory in bytes", + defaultNodeLabels, nil, + ), + Value: func(fileCacheStats NodeStatsFileCacheResponse) float64 { + return float64(fileCacheStats.TotalInBytes) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "filecache", "used_in_bytes"), + "file_cache used memory in bytes", + defaultNodeLabels, nil, + ), + Value: func(fileCacheStats NodeStatsFileCacheResponse) float64 { + return float64(fileCacheStats.UsedInBytes) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "filecache", "evictions_in_bytes"), + "file_cache evicted memory in bytes", + defaultNodeLabels, nil, + ), + Value: func(fileCacheStats NodeStatsFileCacheResponse) float64 { + return float64(fileCacheStats.EvictionsInBytes) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "filecache", "active_percent"), + "file_cache active memory as percent", + defaultNodeLabels, nil, + ), + Value: func(fileCacheStats NodeStatsFileCacheResponse) float64 { + return float64(fileCacheStats.ActivePercent) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "filecache", "used_percent"), + "file_cache used memory as percent", + defaultNodeLabels, nil, + ), + Value: func(fileCacheStats NodeStatsFileCacheResponse) float64 { + return float64(fileCacheStats.UsedPercent) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "filecache", "hit_count"), + "file_cache hit count", + defaultNodeLabels, nil, + ), + Value: func(fileCacheStats NodeStatsFileCacheResponse) float64 { + return float64(fileCacheStats.HitCount) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "filecache", "miss_count"), + "file_cache hit count", + defaultNodeLabels, nil, + ), + Value: func(fileCacheStats NodeStatsFileCacheResponse) float64 { + return float64(fileCacheStats.MissCount) + }, + Labels: defaultNodeLabelValues, + }, + }, } } From 26bc7e9b228d90257aeab4a3f901eab64b03fff8 Mon Sep 17 00:00:00 2001 From: etgraylog Date: Sun, 11 Feb 2024 20:12:52 -0500 Subject: [PATCH 05/12] Add description for fileCacheMetrics Signed-off-by: etgraylog --- collector/nodes.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/collector/nodes.go b/collector/nodes.go index b28f2812..436b26a2 100644 --- a/collector/nodes.go +++ b/collector/nodes.go @@ -1908,6 +1908,9 @@ func (c *Nodes) Describe(ch chan<- *prometheus.Desc) { for _, metric := range c.filesystemIODeviceMetrics { ch <- metric.Desc } + for _, metric := range c.fileCacheMetrics { + ch <- metric.Desc + } ch <- c.up.Desc() ch <- c.totalScrapes.Desc() ch <- c.jsonParseFailures.Desc() From 2f890e3356c0bf7dbd584270ce6c997b2043be59 Mon Sep 17 00:00:00 2001 From: etgraylog Date: Sun, 11 Feb 2024 20:13:46 -0500 Subject: [PATCH 06/12] Add values & labels of fileCache to the Collect method Signed-off-by: etgraylog --- collector/nodes.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/collector/nodes.go b/collector/nodes.go index 436b26a2..2158107d 100644 --- a/collector/nodes.go +++ b/collector/nodes.go @@ -2065,5 +2065,16 @@ func (c *Nodes) Collect(ch chan<- prometheus.Metric) { } } + // File cache Stats + for _, metric := range c.fileCacheMetrics { + ch <- prometheus.MustNewConstMetric( + metric.Desc, + metric.Type, + metric.Value(node.FileCache), + metric.Labels(nodeStatsResp.ClusterName, node)..., + ) + } + } + } From 1b0cfdeec80e0570d7181bfb9c9ecd9049c99332 Mon Sep 17 00:00:00 2001 From: etgraylog Date: Sun, 11 Feb 2024 20:14:13 -0500 Subject: [PATCH 07/12] Add field for file_cache json to NodesStatsNodeResponse struct Signed-off-by: etgraylog --- collector/nodes_response.go | 1 + 1 file changed, 1 insertion(+) diff --git a/collector/nodes_response.go b/collector/nodes_response.go index 4985add0..74991253 100644 --- a/collector/nodes_response.go +++ b/collector/nodes_response.go @@ -40,6 +40,7 @@ type NodeStatsNodeResponse struct { HTTP map[string]interface{} `json:"http"` Transport NodeStatsTransportResponse `json:"transport"` Process NodeStatsProcessResponse `json:"process"` + FileCache NodeStatsFileCacheResponse `json:"file_cache"` } // NodeStatsBreakersResponse is a representation of a statistics about the field data circuit breaker From d7671d5993576a5247280c2cef4ae38f167ef4e2 Mon Sep 17 00:00:00 2001 From: etgraylog Date: Sun, 11 Feb 2024 20:14:35 -0500 Subject: [PATCH 08/12] Add struct for NodeStatsFileCacheResponse Signed-off-by: etgraylog --- collector/nodes_response.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/collector/nodes_response.go b/collector/nodes_response.go index 74991253..fb14977a 100644 --- a/collector/nodes_response.go +++ b/collector/nodes_response.go @@ -385,3 +385,15 @@ type ClusterHealthResponse struct { TimedOut bool `json:"timed_out"` UnassignedShards int64 `json:"unassigned_shards"` } + +// NodeStatsFileCacheResponse is a representation of OpenSearch Searchable Snapshots File_cache +type NodeStatsFileCacheResponse struct { + ActiveInBytes int64 `json:"active_in_bytes"` + TotalInBytes int64 `json:"total_in_bytes"` + UsedInBytes int64 `json:"used_in_bytes"` + EvictionsInBytes int64 `json:"evictions_in_bytes"` + ActivePercent int64 `json:"active_percent"` + UsedPercent int64 `json:"used_percent"` + HitCount int64 `json:"hit_count"` + MissCount int64 `json:"miss_count"` +} From c2afed3df83981c3bdc71a352fc661861799a474 Mon Sep 17 00:00:00 2001 From: etgraylog Date: Mon, 12 Feb 2024 00:47:20 -0500 Subject: [PATCH 09/12] Added .idea Signed-off-by: etgraylog --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e470da31..4321fd02 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ elasticsearch_exporter *-stamp .tarballs /vendor +.idea \ No newline at end of file From e88bc2690dfee8c32d88c8126bdbae8efcf11a9c Mon Sep 17 00:00:00 2001 From: etgraylog Date: Fri, 23 Feb 2024 17:33:21 -0500 Subject: [PATCH 10/12] Added es_search_node to defaultNodeLabels Signed-off-by: etgraylog --- collector/nodes.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collector/nodes.go b/collector/nodes.go index 2158107d..e1fadb13 100644 --- a/collector/nodes.go +++ b/collector/nodes.go @@ -93,7 +93,7 @@ func createRoleMetric(role string) *nodeMetric { } var ( - defaultNodeLabels = []string{"cluster", "host", "name", "es_master_node", "es_data_node", "es_ingest_node", "es_client_node"} + defaultNodeLabels = []string{"cluster", "host", "name", "es_master_node", "es_data_node", "es_ingest_node", "es_client_node", "es_search_node"} defaultRoleLabels = []string{"cluster", "host", "name"} defaultThreadPoolLabels = append(defaultNodeLabels, "type") defaultBreakerLabels = append(defaultNodeLabels, "breaker") From 7b559f9ee52eaef5983f12ca46e80cbd17444619 Mon Sep 17 00:00:00 2001 From: etgraylog Date: Fri, 23 Feb 2024 17:33:45 -0500 Subject: [PATCH 11/12] Added search to defaultNodeLabelValues Signed-off-by: etgraylog --- collector/nodes.go | 1 + 1 file changed, 1 insertion(+) diff --git a/collector/nodes.go b/collector/nodes.go index e1fadb13..755dfad4 100644 --- a/collector/nodes.go +++ b/collector/nodes.go @@ -111,6 +111,7 @@ var ( fmt.Sprintf("%t", roles["data"]), fmt.Sprintf("%t", roles["ingest"]), fmt.Sprintf("%t", roles["client"]), + fmt.Sprintf("%t", roles["search"]), } } defaultThreadPoolLabelValues = func(cluster string, node NodeStatsNodeResponse, pool string) []string { From 10c519a401b42861ade7da6dce412b66b10bb091 Mon Sep 17 00:00:00 2001 From: etgraylog Date: Fri, 23 Feb 2024 17:34:28 -0500 Subject: [PATCH 12/12] Fixing incorrect description for fileCacheStats.MissCount Signed-off-by: etgraylog --- collector/nodes.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collector/nodes.go b/collector/nodes.go index 755dfad4..a6c2a63f 100644 --- a/collector/nodes.go +++ b/collector/nodes.go @@ -1880,7 +1880,7 @@ func NewNodes(logger log.Logger, client *http.Client, url *url.URL, all bool, no Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "filecache", "miss_count"), - "file_cache hit count", + "file_cache miss count", defaultNodeLabels, nil, ), Value: func(fileCacheStats NodeStatsFileCacheResponse) float64 {