Skip to content

process-exporter: collect mmap statistics if enabled #313

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
+142 −33
Open

process-exporter: collect mmap statistics if enabled #313

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,11 @@ Number of context switches based on /proc/[pid]/status fields voluntary_ctxt_swi
and nonvoluntary_ctxt_switches. The extra label `ctxswitchtype` can have two values:
`voluntary` and `nonvoluntary`.

### mmap_count gauge

If gathering mmaps file is enabled, this indicates the maximum number of mmap entries
of any process in the group.

### memory_bytes gauge

Number of bytes of memory used. The extra label `memtype` can have three values:
Expand Down Expand Up @@ -355,6 +360,12 @@ Same as minor_page_faults_total, but broken down per-thread subgroup.

Same as context_switches_total, but broken down per-thread subgroup.

## Global metrics

### max_map_count gauge

If gathering mmaps file is enabled, this metric indicates the global maximum per process.

## Instrumentation cost

process-exporter will consume CPU in proportion to the number of processes in
Expand Down
3 changes: 3 additions & 0 deletions cmd/process-exporter/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,8 @@ func main() {
"if a proc is tracked, track with it any children that aren't part of their own group")
threads = flag.Bool("threads", true,
"report on per-threadname metrics as well")
mmaps = flag.Bool("gather-mmaps", true,
"gather metrics from maps file, which contains mmap info")
smaps = flag.Bool("gather-smaps", true,
"gather metrics from smaps file, which contains proportional resident memory size")
man = flag.Bool("man", false,
Expand Down Expand Up @@ -244,6 +246,7 @@ func main() {
ProcFSPath: *procfsPath,
Children: *children,
Threads: *threads,
GatherMMaps: *mmaps,
GatherSMaps: *smaps,
Namer: matchnamer,
Recheck: *recheck,
Expand Down
55 changes: 55 additions & 0 deletions collector/process_collector.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
package collector

import (
"bufio"
"log"
"os"
"strconv"
"strings"
"time"

common "github.com/ncabatoff/process-exporter"
Expand Down Expand Up @@ -52,6 +56,12 @@ var (
[]string{"groupname", "ctxswitchtype"},
nil)

mmapCountDesc = prometheus.NewDesc(
"namedprocess_namegroup_mmap_count",
"maximum number of mmap entries in use",
[]string{"groupname"},
nil)

membytesDesc = prometheus.NewDesc(
"namedprocess_namegroup_memory_bytes",
"number of bytes of memory in use",
Expand Down Expand Up @@ -106,6 +116,12 @@ var (
nil,
nil)

maxMapCountDesc = prometheus.NewDesc(
"namedprocess_max_map_count",
"maximum number of map entries allowed per process",
nil,
nil)

threadWchanDesc = prometheus.NewDesc(
"namedprocess_namegroup_threads_wchan",
"Number of threads in this group waiting on each wchan",
Expand Down Expand Up @@ -159,6 +175,7 @@ type (
ProcFSPath string
Children bool
Threads bool
GatherMMaps bool
GatherSMaps bool
Namer common.MatchNamer
Recheck bool
Expand All @@ -172,7 +189,9 @@ type (
*proc.Grouper
threads bool
smaps bool
mmaps bool
source proc.Source
maxMapCount int
scrapeErrors int
scrapeProcReadErrors int
scrapePartialErrors int
Expand All @@ -186,12 +205,14 @@ func NewProcessCollector(options ProcessCollectorOption) (*NamedProcessCollector
return nil, err
}

fs.GatherMMaps = options.GatherMMaps
fs.GatherSMaps = options.GatherSMaps
p := &NamedProcessCollector{
scrapeChan: make(chan scrapeRequest),
Grouper: proc.NewGrouper(options.Namer, options.Children, options.Threads, options.Recheck, options.RecheckTimeLimit, options.Debug, options.RemoveEmptyGroups),
source: fs,
threads: options.Threads,
mmaps: options.GatherMMaps,
smaps: options.GatherSMaps,
debug: options.Debug,
}
Expand Down Expand Up @@ -253,6 +274,25 @@ func (p *NamedProcessCollector) start() {
}
}

func GetMaxMapCount() (int, error) {
r, err := os.Open("/proc/sys/vm/max_map_count")
if err != nil {
return 0, err
}
defer r.Close()
reader := bufio.NewReader(r)
rdln, err := reader.ReadString('\n')
if err != nil {
return 0, err
}
line := strings.Trim(rdln, "\n")
val, err := strconv.Atoi(line)
if err != nil {
return 0, err
}
return val, nil
}

func (p *NamedProcessCollector) scrape(ch chan<- prometheus.Metric) {
permErrs, groups, err := p.Update(p.source.AllProcs())
p.scrapePartialErrors += permErrs.Partial
Expand Down Expand Up @@ -309,6 +349,11 @@ func (p *NamedProcessCollector) scrape(ch chan<- prometheus.Metric) {
prometheus.GaugeValue, float64(count), gname, wchan)
}

if p.mmaps {
ch <- prometheus.MustNewConstMetric(mmapCountDesc,
prometheus.GaugeValue, float64(gcounts.Memory.MmapCount), gname)
}

if p.smaps {
ch <- prometheus.MustNewConstMetric(membytesDesc,
prometheus.GaugeValue, float64(gcounts.Memory.ProportionalBytes), gname, "proportionalResident")
Expand Down Expand Up @@ -349,6 +394,16 @@ func (p *NamedProcessCollector) scrape(ch chan<- prometheus.Metric) {
}
}
}
if p.mmaps {
max_map_count, err := GetMaxMapCount()
if err == nil {
p.maxMapCount = max_map_count
} else {
p.scrapePartialErrors++
}
}
ch <- prometheus.MustNewConstMetric(maxMapCountDesc,
prometheus.GaugeValue, float64(p.maxMapCount))
ch <- prometheus.MustNewConstMetric(scrapeErrorsDesc,
prometheus.CounterValue, float64(p.scrapeErrors))
ch <- prometheus.MustNewConstMetric(scrapeProcReadErrorsDesc,
Expand Down
3 changes: 3 additions & 0 deletions proc/grouper.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,9 @@ func groupadd(grp Group, ts Update) Group {
grp.Memory.ResidentBytes += ts.Memory.ResidentBytes
grp.Memory.VirtualBytes += ts.Memory.VirtualBytes
grp.Memory.VmSwapBytes += ts.Memory.VmSwapBytes
if grp.Memory.MmapCount < ts.Memory.MmapCount {
grp.Memory.MmapCount = ts.Memory.MmapCount
}
grp.Memory.ProportionalBytes += ts.Memory.ProportionalBytes
grp.Memory.ProportionalSwapBytes += ts.Memory.ProportionalSwapBytes
if ts.Filedesc.Open != -1 {
Expand Down
54 changes: 27 additions & 27 deletions proc/grouper_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,30 +45,30 @@ func TestGrouperBasic(t *testing.T) {
}{
{
[]IDInfo{
piinfost(p1, n1, Counts{1, 2, 3, 4, 5, 6, 0, 0}, Memory{7, 8, 0, 0, 0},
piinfost(p1, n1, Counts{1, 2, 3, 4, 5, 6, 0, 0}, Memory{7, 8, 0, 0, 0, 0},
Filedesc{4, 400}, 2, States{Other: 1}),
piinfost(p2, n2, Counts{2, 3, 4, 5, 6, 7, 0, 0}, Memory{8, 9, 0, 0, 0},
piinfost(p2, n2, Counts{2, 3, 4, 5, 6, 7, 0, 0}, Memory{8, 9, 0, 0, 0, 0},
Filedesc{40, 400}, 3, States{Waiting: 1}),
},
GroupByName{
"g1": Group{Counts{}, States{Other: 1}, msi{}, 1, Memory{7, 8, 0, 0, 0}, starttime,
"g1": Group{Counts{}, States{Other: 1}, msi{}, 1, Memory{7, 8, 0, 0, 0, 0}, starttime,
4, 0.01, 2, nil},
"g2": Group{Counts{}, States{Waiting: 1}, msi{}, 1, Memory{8, 9, 0, 0, 0}, starttime,
"g2": Group{Counts{}, States{Waiting: 1}, msi{}, 1, Memory{8, 9, 0, 0, 0, 0}, starttime,
40, 0.1, 3, nil},
},
},
{
[]IDInfo{
piinfost(p1, n1, Counts{2, 3, 4, 5, 6, 7, 0, 0},
Memory{6, 7, 0, 0, 0}, Filedesc{100, 400}, 4, States{Zombie: 1}),
Memory{6, 7, 0, 0, 0, 0}, Filedesc{100, 400}, 4, States{Zombie: 1}),
piinfost(p2, n2, Counts{4, 5, 6, 7, 8, 9, 0, 0},
Memory{9, 8, 0, 0, 0}, Filedesc{400, 400}, 2, States{Running: 1}),
Memory{9, 8, 0, 0, 0, 0}, Filedesc{400, 400}, 2, States{Running: 1}),
},
GroupByName{
"g1": Group{Counts{1, 1, 1, 1, 1, 1, 0, 0}, States{Zombie: 1}, msi{}, 1,
Memory{6, 7, 0, 0, 0}, starttime, 100, 0.25, 4, nil},
Memory{6, 7, 0, 0, 0, 0}, starttime, 100, 0.25, 4, nil},
"g2": Group{Counts{2, 2, 2, 2, 2, 2, 0, 0}, States{Running: 1}, msi{}, 1,
Memory{9, 8, 0, 0, 0}, starttime, 400, 1, 2, nil},
Memory{9, 8, 0, 0, 0, 0}, starttime, 400, 1, 2, nil},
},
},
}
Expand All @@ -95,35 +95,35 @@ func TestGrouperProcJoin(t *testing.T) {
}{
{
[]IDInfo{
piinfo(p1, n1, Counts{1, 2, 3, 4, 5, 6, 0, 0}, Memory{3, 4, 0, 0, 0}, Filedesc{4, 400}, 2),
piinfo(p1, n1, Counts{1, 2, 3, 4, 5, 6, 0, 0}, Memory{3, 4, 0, 0, 0, 0}, Filedesc{4, 400}, 2),
},
GroupByName{
"g1": Group{Counts{}, States{}, msi{}, 1, Memory{3, 4, 0, 0, 0}, starttime, 4, 0.01, 2, nil},
"g1": Group{Counts{}, States{}, msi{}, 1, Memory{3, 4, 0, 0, 0, 0}, starttime, 4, 0.01, 2, nil},
},
}, {
// The counts for pid2 won't be factored into the total yet because we only add
// to counts starting with the second time we see a proc. Memory and FDs are
// affected though.
[]IDInfo{
piinfost(p1, n1, Counts{3, 4, 5, 6, 7, 8, 0, 0},
Memory{3, 4, 0, 0, 0}, Filedesc{4, 400}, 2, States{Running: 1}),
Memory{3, 4, 0, 0, 0, 0}, Filedesc{4, 400}, 2, States{Running: 1}),
piinfost(p2, n2, Counts{1, 1, 1, 1, 1, 1, 0, 0},
Memory{1, 2, 0, 0, 0}, Filedesc{40, 400}, 3, States{Sleeping: 1}),
Memory{1, 2, 0, 0, 0, 0}, Filedesc{40, 400}, 3, States{Sleeping: 1}),
},
GroupByName{
"g1": Group{Counts{2, 2, 2, 2, 2, 2, 0, 0}, States{Running: 1, Sleeping: 1}, msi{}, 2,
Memory{4, 6, 0, 0, 0}, starttime, 44, 0.1, 5, nil},
Memory{4, 6, 0, 0, 0, 0}, starttime, 44, 0.1, 5, nil},
},
}, {
[]IDInfo{
piinfost(p1, n1, Counts{4, 5, 6, 7, 8, 9, 0, 0},
Memory{1, 5, 0, 0, 0}, Filedesc{4, 400}, 2, States{Running: 1}),
Memory{1, 5, 0, 0, 0, 0}, Filedesc{4, 400}, 2, States{Running: 1}),
piinfost(p2, n2, Counts{2, 2, 2, 2, 2, 2, 0, 0},
Memory{2, 4, 0, 0, 0}, Filedesc{40, 400}, 3, States{Running: 1}),
Memory{2, 4, 0, 0, 0, 0}, Filedesc{40, 400}, 3, States{Running: 1}),
},
GroupByName{
"g1": Group{Counts{4, 4, 4, 4, 4, 4, 0, 0}, States{Running: 2}, msi{}, 2,
Memory{3, 9, 0, 0, 0}, starttime, 44, 0.1, 5, nil},
Memory{3, 9, 0, 0, 0, 0}, starttime, 44, 0.1, 5, nil},
},
},
}
Expand All @@ -150,18 +150,18 @@ func TestGrouperNonDecreasing(t *testing.T) {
}{
{
[]IDInfo{
piinfo(p1, n1, Counts{3, 4, 5, 6, 7, 8, 0, 0}, Memory{3, 4, 0, 0, 0}, Filedesc{4, 400}, 2),
piinfo(p2, n2, Counts{1, 1, 1, 1, 1, 1, 0, 0}, Memory{1, 2, 0, 0, 0}, Filedesc{40, 400}, 3),
piinfo(p1, n1, Counts{3, 4, 5, 6, 7, 8, 0, 0}, Memory{3, 4, 0, 0, 0, 0}, Filedesc{4, 400}, 2),
piinfo(p2, n2, Counts{1, 1, 1, 1, 1, 1, 0, 0}, Memory{1, 2, 0, 0, 0, 0}, Filedesc{40, 400}, 3),
},
GroupByName{
"g1": Group{Counts{}, States{}, msi{}, 2, Memory{4, 6, 0, 0, 0}, starttime, 44, 0.1, 5, nil},
"g1": Group{Counts{}, States{}, msi{}, 2, Memory{4, 6, 0, 0, 0, 0}, starttime, 44, 0.1, 5, nil},
},
}, {
[]IDInfo{
piinfo(p1, n1, Counts{4, 5, 6, 7, 8, 9, 0, 0}, Memory{1, 5, 0, 0, 0}, Filedesc{4, 400}, 2),
piinfo(p1, n1, Counts{4, 5, 6, 7, 8, 9, 0, 0}, Memory{1, 5, 0, 0, 0, 0}, Filedesc{4, 400}, 2),
},
GroupByName{
"g1": Group{Counts{1, 1, 1, 1, 1, 1, 0, 0}, States{}, msi{}, 1, Memory{1, 5, 0, 0, 0}, starttime, 4, 0.01, 2, nil},
"g1": Group{Counts{1, 1, 1, 1, 1, 1, 0, 0}, States{}, msi{}, 1, Memory{1, 5, 0, 0, 0, 0}, starttime, 4, 0.01, 2, nil},
},
}, {
[]IDInfo{},
Expand Down Expand Up @@ -193,19 +193,19 @@ func TestGrouperRemoveEmptyGroups(t *testing.T) {
}{
{
[]IDInfo{
piinfo(p1, n1, Counts{3, 4, 5, 6, 7, 8, 0, 0}, Memory{3, 4, 0, 0, 0}, Filedesc{4, 400}, 2),
piinfo(p2, n2, Counts{1, 1, 1, 1, 1, 1, 0, 0}, Memory{1, 2, 0, 0, 0}, Filedesc{40, 400}, 3),
piinfo(p1, n1, Counts{3, 4, 5, 6, 7, 8, 0, 0}, Memory{3, 4, 0, 0, 0, 0}, Filedesc{4, 400}, 2),
piinfo(p2, n2, Counts{1, 1, 1, 1, 1, 1, 0, 0}, Memory{1, 2, 0, 0, 0, 0}, Filedesc{40, 400}, 3),
},
GroupByName{
n1: Group{Counts{}, States{}, msi{}, 1, Memory{3, 4, 0, 0, 0}, starttime, 4, 0.01, 2, nil},
n2: Group{Counts{}, States{}, msi{}, 1, Memory{1, 2, 0, 0, 0}, starttime, 40, 0.1, 3, nil},
n1: Group{Counts{}, States{}, msi{}, 1, Memory{3, 4, 0, 0, 0, 0}, starttime, 4, 0.01, 2, nil},
n2: Group{Counts{}, States{}, msi{}, 1, Memory{1, 2, 0, 0, 0, 0}, starttime, 40, 0.1, 3, nil},
},
}, {
[]IDInfo{
piinfo(p1, n1, Counts{4, 5, 6, 7, 8, 9, 0, 0}, Memory{1, 5, 0, 0, 0}, Filedesc{4, 400}, 2),
piinfo(p1, n1, Counts{4, 5, 6, 7, 8, 9, 0, 0}, Memory{1, 5, 0, 0, 0, 0}, Filedesc{4, 400}, 2),
},
GroupByName{
n1: Group{Counts{1, 1, 1, 1, 1, 1, 0, 0}, States{}, msi{}, 1, Memory{1, 5, 0, 0, 0}, starttime, 4, 0.01, 2, nil},
n1: Group{Counts{1, 1, 1, 1, 1, 1, 0, 0}, States{}, msi{}, 1, Memory{1, 5, 0, 0, 0, 0}, starttime, 4, 0.01, 2, nil},
},
}, {
[]IDInfo{},
Expand Down
Loading