Skip to content

Support process shared memory monitoring #353

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Support process shared memory monitoring #353

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions collector/process_collector.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,8 @@ func (p *NamedProcessCollector) scrape(ch chan<- prometheus.Metric) {
prometheus.GaugeValue, float64(gcounts.Memory.VirtualBytes), gname, "virtual")
ch <- prometheus.MustNewConstMetric(membytesDesc,
prometheus.GaugeValue, float64(gcounts.Memory.VmSwapBytes), gname, "swapped")
ch <- prometheus.MustNewConstMetric(membytesDesc,
prometheus.GaugeValue, float64(gcounts.Memory.SHRBytes), gname, "shared")
ch <- prometheus.MustNewConstMetric(startTimeDesc,
prometheus.GaugeValue, float64(gcounts.OldestStartTime.Unix()), gname)
ch <- prometheus.MustNewConstMetric(openFDsDesc,
Expand Down
1 change: 1 addition & 0 deletions fixtures/14804/statm
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0 0 1995 0 0 0 0
8 changes: 4 additions & 4 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,13 @@ go 1.23.0
toolchain go1.23.8

require (
github.com/google/go-cmp v0.6.0
github.com/google/go-cmp v0.7.0
github.com/ncabatoff/fakescraper v0.0.0-20201102132415-4b37ba603d65
github.com/ncabatoff/go-seq v0.0.0-20180805175032-b08ef85ed833
github.com/prometheus/client_golang v1.19.0
github.com/prometheus/common v0.52.3
github.com/prometheus/exporter-toolkit v0.11.0
github.com/prometheus/procfs v0.14.0
github.com/prometheus/procfs v0.17.0
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c
gopkg.in/yaml.v2 v2.4.0
)
Expand All @@ -33,8 +33,8 @@ require (
golang.org/x/crypto v0.35.0 // indirect
golang.org/x/net v0.36.0 // indirect
golang.org/x/oauth2 v0.18.0 // indirect
golang.org/x/sync v0.11.0 // indirect
golang.org/x/sys v0.30.0 // indirect
golang.org/x/sync v0.15.0 // indirect
golang.org/x/sys v0.33.0 // indirect
golang.org/x/text v0.22.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.33.0 // indirect
Expand Down
16 changes: 8 additions & 8 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
Expand All @@ -45,8 +45,8 @@ github.com/prometheus/common v0.52.3 h1:5f8uj6ZwHSscOGNdIQg6OiZv/ybiK2CO2q2drVZA
github.com/prometheus/common v0.52.3/go.mod h1:BrxBKv3FWBIGXw89Mg1AeBq7FSyRzXWI3l3e7W3RN5U=
github.com/prometheus/exporter-toolkit v0.11.0 h1:yNTsuZ0aNCNFQ3aFTD2uhPOvr4iD7fdBvKPAEGkNf+g=
github.com/prometheus/exporter-toolkit v0.11.0/go.mod h1:BVnENhnNecpwoTLiABx7mrPB/OLRIgN74qlQbV+FK1Q=
github.com/prometheus/procfs v0.14.0 h1:Lw4VdGGoKEZilJsayHf0B+9YgLGREba2C6xr+Fdfq6s=
github.com/prometheus/procfs v0.14.0/go.mod h1:XL+Iwz8k8ZabyZfMFHPiilCniixqQarAy5Mu67pHlNQ=
github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0=
github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
Expand All @@ -60,11 +60,11 @@ golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA=
golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I=
golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI=
golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8=
golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8=
golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
Expand Down
1 change: 1 addition & 0 deletions proc/grouper.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ func groupadd(grp Group, ts Update) Group {
grp.Memory.ResidentBytes += ts.Memory.ResidentBytes
grp.Memory.VirtualBytes += ts.Memory.VirtualBytes
grp.Memory.VmSwapBytes += ts.Memory.VmSwapBytes
grp.Memory.SHRBytes += ts.Memory.SHRBytes
grp.Memory.ProportionalBytes += ts.Memory.ProportionalBytes
grp.Memory.ProportionalSwapBytes += ts.Memory.ProportionalSwapBytes
if ts.Filedesc.Open != -1 {
Expand Down
54 changes: 27 additions & 27 deletions proc/grouper_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,30 +45,30 @@ func TestGrouperBasic(t *testing.T) {
}{
{
[]IDInfo{
piinfost(p1, n1, Counts{1, 2, 3, 4, 5, 6, 0, 0}, Memory{7, 8, 0, 0, 0},
piinfost(p1, n1, Counts{1, 2, 3, 4, 5, 6, 0, 0}, Memory{7, 8, 0, 0, 0, 1},
Filedesc{4, 400}, 2, States{Other: 1}),
piinfost(p2, n2, Counts{2, 3, 4, 5, 6, 7, 0, 0}, Memory{8, 9, 0, 0, 0},
piinfost(p2, n2, Counts{2, 3, 4, 5, 6, 7, 0, 0}, Memory{8, 9, 0, 0, 0, 1},
Filedesc{40, 400}, 3, States{Waiting: 1}),
},
GroupByName{
"g1": Group{Counts{}, States{Other: 1}, msi{}, 1, Memory{7, 8, 0, 0, 0}, starttime,
"g1": Group{Counts{}, States{Other: 1}, msi{}, 1, Memory{7, 8, 0, 0, 0, 1}, starttime,
4, 0.01, 2, nil},
"g2": Group{Counts{}, States{Waiting: 1}, msi{}, 1, Memory{8, 9, 0, 0, 0}, starttime,
"g2": Group{Counts{}, States{Waiting: 1}, msi{}, 1, Memory{8, 9, 0, 0, 0, 1}, starttime,
40, 0.1, 3, nil},
},
},
{
[]IDInfo{
piinfost(p1, n1, Counts{2, 3, 4, 5, 6, 7, 0, 0},
Memory{6, 7, 0, 0, 0}, Filedesc{100, 400}, 4, States{Zombie: 1}),
Memory{6, 7, 0, 0, 0, 1}, Filedesc{100, 400}, 4, States{Zombie: 1}),
piinfost(p2, n2, Counts{4, 5, 6, 7, 8, 9, 0, 0},
Memory{9, 8, 0, 0, 0}, Filedesc{400, 400}, 2, States{Running: 1}),
Memory{9, 8, 0, 0, 0, 1}, Filedesc{400, 400}, 2, States{Running: 1}),
},
GroupByName{
"g1": Group{Counts{1, 1, 1, 1, 1, 1, 0, 0}, States{Zombie: 1}, msi{}, 1,
Memory{6, 7, 0, 0, 0}, starttime, 100, 0.25, 4, nil},
Memory{6, 7, 0, 0, 0, 1}, starttime, 100, 0.25, 4, nil},
"g2": Group{Counts{2, 2, 2, 2, 2, 2, 0, 0}, States{Running: 1}, msi{}, 1,
Memory{9, 8, 0, 0, 0}, starttime, 400, 1, 2, nil},
Memory{9, 8, 0, 0, 0, 1}, starttime, 400, 1, 2, nil},
},
},
}
Expand All @@ -95,35 +95,35 @@ func TestGrouperProcJoin(t *testing.T) {
}{
{
[]IDInfo{
piinfo(p1, n1, Counts{1, 2, 3, 4, 5, 6, 0, 0}, Memory{3, 4, 0, 0, 0}, Filedesc{4, 400}, 2),
piinfo(p1, n1, Counts{1, 2, 3, 4, 5, 6, 0, 0}, Memory{3, 4, 0, 0, 0, 1}, Filedesc{4, 400}, 2),
},
GroupByName{
"g1": Group{Counts{}, States{}, msi{}, 1, Memory{3, 4, 0, 0, 0}, starttime, 4, 0.01, 2, nil},
"g1": Group{Counts{}, States{}, msi{}, 1, Memory{3, 4, 0, 0, 0, 1}, starttime, 4, 0.01, 2, nil},
},
}, {
// The counts for pid2 won't be factored into the total yet because we only add
// to counts starting with the second time we see a proc. Memory and FDs are
// affected though.
[]IDInfo{
piinfost(p1, n1, Counts{3, 4, 5, 6, 7, 8, 0, 0},
Memory{3, 4, 0, 0, 0}, Filedesc{4, 400}, 2, States{Running: 1}),
Memory{3, 4, 0, 0, 0, 1}, Filedesc{4, 400}, 2, States{Running: 1}),
piinfost(p2, n2, Counts{1, 1, 1, 1, 1, 1, 0, 0},
Memory{1, 2, 0, 0, 0}, Filedesc{40, 400}, 3, States{Sleeping: 1}),
Memory{1, 2, 0, 0, 0, 1}, Filedesc{40, 400}, 3, States{Sleeping: 1}),
},
GroupByName{
"g1": Group{Counts{2, 2, 2, 2, 2, 2, 0, 0}, States{Running: 1, Sleeping: 1}, msi{}, 2,
Memory{4, 6, 0, 0, 0}, starttime, 44, 0.1, 5, nil},
Memory{4, 6, 0, 0, 0, 2}, starttime, 44, 0.1, 5, nil},
},
}, {
[]IDInfo{
piinfost(p1, n1, Counts{4, 5, 6, 7, 8, 9, 0, 0},
Memory{1, 5, 0, 0, 0}, Filedesc{4, 400}, 2, States{Running: 1}),
Memory{1, 5, 0, 0, 0, 1}, Filedesc{4, 400}, 2, States{Running: 1}),
piinfost(p2, n2, Counts{2, 2, 2, 2, 2, 2, 0, 0},
Memory{2, 4, 0, 0, 0}, Filedesc{40, 400}, 3, States{Running: 1}),
Memory{2, 4, 0, 0, 0, 1}, Filedesc{40, 400}, 3, States{Running: 1}),
},
GroupByName{
"g1": Group{Counts{4, 4, 4, 4, 4, 4, 0, 0}, States{Running: 2}, msi{}, 2,
Memory{3, 9, 0, 0, 0}, starttime, 44, 0.1, 5, nil},
Memory{3, 9, 0, 0, 0, 2}, starttime, 44, 0.1, 5, nil},
},
},
}
Expand All @@ -150,18 +150,18 @@ func TestGrouperNonDecreasing(t *testing.T) {
}{
{
[]IDInfo{
piinfo(p1, n1, Counts{3, 4, 5, 6, 7, 8, 0, 0}, Memory{3, 4, 0, 0, 0}, Filedesc{4, 400}, 2),
piinfo(p2, n2, Counts{1, 1, 1, 1, 1, 1, 0, 0}, Memory{1, 2, 0, 0, 0}, Filedesc{40, 400}, 3),
piinfo(p1, n1, Counts{3, 4, 5, 6, 7, 8, 0, 0}, Memory{3, 4, 0, 0, 0, 1}, Filedesc{4, 400}, 2),
piinfo(p2, n2, Counts{1, 1, 1, 1, 1, 1, 0, 0}, Memory{1, 2, 0, 0, 0, 1}, Filedesc{40, 400}, 3),
},
GroupByName{
"g1": Group{Counts{}, States{}, msi{}, 2, Memory{4, 6, 0, 0, 0}, starttime, 44, 0.1, 5, nil},
"g1": Group{Counts{}, States{}, msi{}, 2, Memory{4, 6, 0, 0, 0, 2}, starttime, 44, 0.1, 5, nil},
},
}, {
[]IDInfo{
piinfo(p1, n1, Counts{4, 5, 6, 7, 8, 9, 0, 0}, Memory{1, 5, 0, 0, 0}, Filedesc{4, 400}, 2),
piinfo(p1, n1, Counts{4, 5, 6, 7, 8, 9, 0, 0}, Memory{1, 5, 0, 0, 0, 1}, Filedesc{4, 400}, 2),
},
GroupByName{
"g1": Group{Counts{1, 1, 1, 1, 1, 1, 0, 0}, States{}, msi{}, 1, Memory{1, 5, 0, 0, 0}, starttime, 4, 0.01, 2, nil},
"g1": Group{Counts{1, 1, 1, 1, 1, 1, 0, 0}, States{}, msi{}, 1, Memory{1, 5, 0, 0, 0, 1}, starttime, 4, 0.01, 2, nil},
},
}, {
[]IDInfo{},
Expand Down Expand Up @@ -193,19 +193,19 @@ func TestGrouperRemoveEmptyGroups(t *testing.T) {
}{
{
[]IDInfo{
piinfo(p1, n1, Counts{3, 4, 5, 6, 7, 8, 0, 0}, Memory{3, 4, 0, 0, 0}, Filedesc{4, 400}, 2),
piinfo(p2, n2, Counts{1, 1, 1, 1, 1, 1, 0, 0}, Memory{1, 2, 0, 0, 0}, Filedesc{40, 400}, 3),
piinfo(p1, n1, Counts{3, 4, 5, 6, 7, 8, 0, 0}, Memory{3, 4, 0, 0, 0, 1}, Filedesc{4, 400}, 2),
piinfo(p2, n2, Counts{1, 1, 1, 1, 1, 1, 0, 0}, Memory{1, 2, 0, 0, 0, 1}, Filedesc{40, 400}, 3),
},
GroupByName{
n1: Group{Counts{}, States{}, msi{}, 1, Memory{3, 4, 0, 0, 0}, starttime, 4, 0.01, 2, nil},
n2: Group{Counts{}, States{}, msi{}, 1, Memory{1, 2, 0, 0, 0}, starttime, 40, 0.1, 3, nil},
n1: Group{Counts{}, States{}, msi{}, 1, Memory{3, 4, 0, 0, 0, 1}, starttime, 4, 0.01, 2, nil},
n2: Group{Counts{}, States{}, msi{}, 1, Memory{1, 2, 0, 0, 0, 1}, starttime, 40, 0.1, 3, nil},
},
}, {
[]IDInfo{
piinfo(p1, n1, Counts{4, 5, 6, 7, 8, 9, 0, 0}, Memory{1, 5, 0, 0, 0}, Filedesc{4, 400}, 2),
piinfo(p1, n1, Counts{4, 5, 6, 7, 8, 9, 0, 0}, Memory{1, 5, 0, 0, 0, 1}, Filedesc{4, 400}, 2),
},
GroupByName{
n1: Group{Counts{1, 1, 1, 1, 1, 1, 0, 0}, States{}, msi{}, 1, Memory{1, 5, 0, 0, 0}, starttime, 4, 0.01, 2, nil},
n1: Group{Counts{1, 1, 1, 1, 1, 1, 0, 0}, States{}, msi{}, 1, Memory{1, 5, 0, 0, 0, 1}, starttime, 4, 0.01, 2, nil},
},
}, {
[]IDInfo{},
Expand Down
18 changes: 18 additions & 0 deletions proc/read.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ type (
VmSwapBytes uint64
ProportionalBytes uint64
ProportionalSwapBytes uint64
SHRBytes uint64
}

// Filedesc describes a proc's file descriptor usage and soft limit.
Expand Down Expand Up @@ -134,6 +135,7 @@ type (
procfs.Proc
procid *ID
stat *procfs.ProcStat
statm *procfs.ProcStatm
status *procfs.ProcStatus
cmdline []string
cgroups []procfs.Cgroup
Expand Down Expand Up @@ -289,6 +291,18 @@ func (p *proccache) getStat() (procfs.ProcStat, error) {
return *p.stat, nil
}

func (p *proccache) getStatm() (procfs.ProcStatm, error) {
if p.statm == nil {
statm, err := p.Proc.NewStatm()
if err != nil {
return procfs.ProcStatm{}, err
}
p.statm = &statm
}

return *p.statm, nil
}

func (p *proccache) getStatus() (procfs.ProcStatus, error) {
if p.status == nil {
status, err := p.Proc.NewStatus()
Expand Down Expand Up @@ -478,6 +492,9 @@ func (p proc) GetMetrics() (Metrics, int, error) {
// won't see the effect of the caching between calls.
stat, _ := p.getStat()

// Used for getting shared memory of process
statm, _ := p.getStatm()

// Ditto for states
states, _ := p.GetStates()

Expand All @@ -504,6 +521,7 @@ func (p proc) GetMetrics() (Metrics, int, error) {
ResidentBytes: uint64(stat.ResidentMemory()),
VirtualBytes: uint64(stat.VirtualMemory()),
VmSwapBytes: uint64(status.VmSwap),
SHRBytes: uint64(statm.SHRBytes()),
}

if p.proccache.fs.GatherSMaps {
Expand Down
1 change: 1 addition & 0 deletions proc/read_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ func TestReadFixture(t *testing.T) {
ResidentBytes: 0x7b1000,
VirtualBytes: 0x1061000,
VmSwapBytes: 0x2800,
SHRBytes: 0x7cb000,
},
Filedesc: Filedesc{
Open: 5,
Expand Down
8 changes: 4 additions & 4 deletions proc/tracker_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,15 +99,15 @@ func TestTrackerMetrics(t *testing.T) {
want Update
}{
{
piinfost(p, n, Counts{1, 2, 3, 4, 5, 6, 0, 0}, Memory{7, 8, 0, 0, 0},
piinfost(p, n, Counts{1, 2, 3, 4, 5, 6, 0, 0}, Memory{7, 8, 0, 0, 0, 1},
Filedesc{1, 10}, 9, States{Sleeping: 1}),
Update{n, Delta{}, Memory{7, 8, 0, 0, 0}, Filedesc{1, 10}, tm,
Update{n, Delta{}, Memory{7, 8, 0, 0, 0, 1}, Filedesc{1, 10}, tm,
9, States{Sleeping: 1}, msi{}, nil},
},
{
piinfost(p, n, Counts{2, 3, 4, 5, 6, 7, 0, 0}, Memory{1, 2, 0, 0, 0},
piinfost(p, n, Counts{2, 3, 4, 5, 6, 7, 0, 0}, Memory{1, 2, 0, 0, 0, 1},
Filedesc{2, 20}, 1, States{Running: 1}),
Update{n, Delta{1, 1, 1, 1, 1, 1, 0, 0}, Memory{1, 2, 0, 0, 0},
Update{n, Delta{1, 1, 1, 1, 1, 1, 0, 0}, Memory{1, 2, 0, 0, 0, 1},
Filedesc{2, 20}, tm, 1, States{Running: 1}, msi{}, nil},
},
}
Expand Down