Skip to content

Commit 7975d8d

Browse files
andrzej-stencelandrzej-stencel
committed
fix(update-collection-v3): fix multiline property rendering
Fixes #433
1 parent 06c9992 commit 7975d8d

File tree

6 files changed

+358
-2
lines changed

6 files changed

+358
-2
lines changed

‎src/go/cmd/update-collection-v3/README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ make build
4848
## Known issues
4949

5050
1. This package is using github.com/go-yaml/yaml which unfortunately doesn't allow
51-
to maintain `yaml` comments and order when using user defined structutres.
51+
to maintain `yaml` comments and order when using user defined structures.
5252

5353
This could be done when we'd use [`yaml.Node`][yaml_node] instead of customized structs
5454
which reflect the schema of `values.yaml` used in `sumologic-kubernetes-collection`

‎src/go/cmd/update-collection-v3/main.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,7 @@ func reorderYaml(input string, original string) (string, error) {
181181

182182
sortByBlueprint(outputMapSlice, originalMapSlice)
183183

184-
outputBytes, err := yaml.MarshalWithOptions(outputMapSlice, yaml.Indent(2))
184+
outputBytes, err := yaml.MarshalWithOptions(outputMapSlice, yaml.Indent(2), yaml.UseLiteralStyleIfMultiline(true))
185185

186186
return string(outputBytes), err
187187
}

‎src/go/cmd/update-collection-v3/testdata/containerd-multiline.input.yaml

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
sumologic:
2+
accessId: xxx
3+
accessKey: yyy
4+
clusterName: containerd-multiline
5+
6+
fluent-bit:
7+
config:
8+
customParsers: |
9+
[PARSER]
10+
Name containerd_multiline_pattern
11+
Format regex
12+
Regex (?<time>^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}[.]\d+Z) (?<stream>stdout|stderr) (?<logtag>[P|F]) (?<log>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.* .*)
13+
Time_Key time
14+
Time_Format %Y-%m-%dT%H:%M:%S.%LZ
15+
filters: |
16+
[FILTER]
17+
Name lua
18+
Match containers.*
19+
script /fluent-bit/scripts/parse_logs.lua
20+
call parse_log
21+
inputs: |
22+
[INPUT]
23+
Name tail
24+
Path /var/log/containers/*.log
25+
Read_from_head true
26+
Multiline On
27+
Parser_Firstline containerd_multiline_pattern
28+
Multiline_Flush 60
29+
Tag containers.*
30+
Refresh_Interval 1
31+
Rotate_Wait 60
32+
Mem_Buf_Limit 30MB
33+
Skip_Long_Lines On
34+
DB /tail-db/tail-containers-state-sumo.db
35+
DB.Sync Normal
36+
[INPUT]
37+
Name systemd
38+
Tag host.*
39+
DB /tail-db/systemd-state-sumo.db
40+
Systemd_Filter _SYSTEMD_UNIT=addon-config.service
41+
Systemd_Filter _SYSTEMD_UNIT=addon-run.service
42+
Systemd_Filter _SYSTEMD_UNIT=cfn-etcd-environment.service
43+
Systemd_Filter _SYSTEMD_UNIT=cfn-signal.service
44+
Systemd_Filter _SYSTEMD_UNIT=clean-ca-certificates.service
45+
Systemd_Filter _SYSTEMD_UNIT=containerd.service
46+
Systemd_Filter _SYSTEMD_UNIT=coreos-metadata.service
47+
Systemd_Filter _SYSTEMD_UNIT=coreos-setup-environment.service
48+
Systemd_Filter _SYSTEMD_UNIT=coreos-tmpfiles.service
49+
Systemd_Filter _SYSTEMD_UNIT=dbus.service
50+
Systemd_Filter _SYSTEMD_UNIT=docker.service
51+
Systemd_Filter _SYSTEMD_UNIT=efs.service
52+
Systemd_Filter _SYSTEMD_UNIT=etcd-member.service
53+
Systemd_Filter _SYSTEMD_UNIT=etcd.service
54+
Systemd_Filter _SYSTEMD_UNIT=etcd2.service
55+
Systemd_Filter _SYSTEMD_UNIT=etcd3.service
56+
Systemd_Filter _SYSTEMD_UNIT=etcdadm-check.service
57+
Systemd_Filter _SYSTEMD_UNIT=etcdadm-reconfigure.service
58+
Systemd_Filter _SYSTEMD_UNIT=etcdadm-save.service
59+
Systemd_Filter _SYSTEMD_UNIT=etcdadm-update-status.service
60+
Systemd_Filter _SYSTEMD_UNIT=flanneld.service
61+
Systemd_Filter _SYSTEMD_UNIT=format-etcd2-volume.service
62+
Systemd_Filter _SYSTEMD_UNIT=kube-node-taint-and-uncordon.service
63+
Systemd_Filter _SYSTEMD_UNIT=kubelet.service
64+
Systemd_Filter _SYSTEMD_UNIT=ldconfig.service
65+
Systemd_Filter _SYSTEMD_UNIT=locksmithd.service
66+
Systemd_Filter _SYSTEMD_UNIT=logrotate.service
67+
Systemd_Filter _SYSTEMD_UNIT=lvm2-monitor.service
68+
Systemd_Filter _SYSTEMD_UNIT=mdmon.service
69+
Systemd_Filter _SYSTEMD_UNIT=nfs-idmapd.service
70+
Systemd_Filter _SYSTEMD_UNIT=nfs-mountd.service
71+
Systemd_Filter _SYSTEMD_UNIT=nfs-server.service
72+
Systemd_Filter _SYSTEMD_UNIT=nfs-utils.service
73+
Systemd_Filter _SYSTEMD_UNIT=node-problem-detector.service
74+
Systemd_Filter _SYSTEMD_UNIT=ntp.service
75+
Systemd_Filter _SYSTEMD_UNIT=oem-cloudinit.service
76+
Systemd_Filter _SYSTEMD_UNIT=rkt-gc.service
77+
Systemd_Filter _SYSTEMD_UNIT=rkt-metadata.service
78+
Systemd_Filter _SYSTEMD_UNIT=rpc-idmapd.service
79+
Systemd_Filter _SYSTEMD_UNIT=rpc-mountd.service
80+
Systemd_Filter _SYSTEMD_UNIT=rpc-statd.service
81+
Systemd_Filter _SYSTEMD_UNIT=rpcbind.service
82+
Systemd_Filter _SYSTEMD_UNIT=set-aws-environment.service
83+
Systemd_Filter _SYSTEMD_UNIT=system-cloudinit.service
84+
Systemd_Filter _SYSTEMD_UNIT=systemd-timesyncd.service
85+
Systemd_Filter _SYSTEMD_UNIT=update-ca-certificates.service
86+
Systemd_Filter _SYSTEMD_UNIT=user-cloudinit.service
87+
Systemd_Filter _SYSTEMD_UNIT=var-lib-etcd2.service
88+
Max_Entries 1000
89+
Read_From_Tail true
90+
outputs: |
91+
[OUTPUT]
92+
Name forward
93+
Match *
94+
Host ${FLUENTD_LOGS_SVC}.${NAMESPACE}.svc.cluster.local.
95+
Port 24321
96+
Retry_Limit False
97+
tls off
98+
tls.verify on
99+
tls.debug 1
100+
# Disable keepalive for better load balancing
101+
net.keepalive off
102+
service: |
103+
[SERVICE]
104+
Flush 1
105+
Daemon Off
106+
Log_Level info
107+
Parsers_File parsers.conf
108+
Parsers_File custom_parsers.conf
109+
HTTP_Server On
110+
HTTP_Listen 0.0.0.0
111+
HTTP_Port 2020
112+
luaScripts:
113+
parse_logs.lua: |
114+
local function split(s, delimiter)
115+
result = {};
116+
for match in (s..delimiter):gmatch('(.-)'..delimiter) do
117+
table.insert(result, match);
118+
end
119+
return result;
120+
end
121+
122+
function get_log_content(line)
123+
-- remove elements specific containerd log format and get log as single string
124+
table.remove(line, 1) -- remove date
125+
table.remove(line, 1) -- remove stream
126+
table.remove(line, 1) -- remove log tag
127+
return table.concat(line, " ")
128+
end
129+
130+
function adjust_first_line(record, first_line)
131+
-- adjust the first line to containerd format, it comes without date, stream and logtag
132+
-- 'fake-date' string at the beginning is used only to have proper log format
133+
first_line = 'fake-date' .. ' ' .. record['stream'] .. ' ' .. record['logtag'] .. ' ' .. first_line
134+
return first_line
135+
end
136+
137+
function parse_log(tag, timestamp, record)
138+
if record['log'] == nil or record['stream'] == nil or record['logtag'] == nil then
139+
-- log does not contain required attributes ('log', 'stream', 'logtag') to be processed by parse_log function
140+
-- the record will not be modified
141+
return 0, timestamp, record
142+
end
143+
144+
log_lines = split(record['log'], '\n')
145+
log_lines[1] = adjust_first_line(record, log_lines[1])
146+
147+
new_lines = {}
148+
buffer = ''
149+
150+
for k, v in pairs(log_lines) do
151+
line = split(v, ' ')
152+
log_tag = line[3]
153+
buffer = buffer .. get_log_content(line)
154+
155+
if log_tag == 'F' then
156+
table.insert(new_lines, buffer)
157+
buffer = ""
158+
end
159+
end
160+
161+
new_log = table.concat(new_lines, "\n")
162+
record['log'] = new_log
163+
return 2, timestamp, record
164+
end
165+
166+
fluentd:
167+
logs:
168+
containers:
169+
multiline:
170+
enabled: false

‎src/go/cmd/update-collection-v3/testdata/containerd-multiline.output.yaml

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
sumologic:
2+
accessId: xxx
3+
accessKey: yyy
4+
clusterName: containerd-multiline
5+
fluent-bit:
6+
config:
7+
customParsers: |
8+
[PARSER]
9+
Name containerd_multiline_pattern
10+
Format regex
11+
Regex (?<time>^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}[.]\d+Z) (?<stream>stdout|stderr) (?<logtag>[P|F]) (?<log>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.* .*)
12+
Time_Key time
13+
Time_Format %Y-%m-%dT%H:%M:%S.%LZ
14+
filters: |
15+
[FILTER]
16+
Name lua
17+
Match containers.*
18+
script /fluent-bit/scripts/parse_logs.lua
19+
call parse_log
20+
inputs: |
21+
[INPUT]
22+
Name tail
23+
Path /var/log/containers/*.log
24+
Read_from_head true
25+
Multiline On
26+
Parser_Firstline containerd_multiline_pattern
27+
Multiline_Flush 60
28+
Tag containers.*
29+
Refresh_Interval 1
30+
Rotate_Wait 60
31+
Mem_Buf_Limit 30MB
32+
Skip_Long_Lines On
33+
DB /tail-db/tail-containers-state-sumo.db
34+
DB.Sync Normal
35+
[INPUT]
36+
Name systemd
37+
Tag host.*
38+
DB /tail-db/systemd-state-sumo.db
39+
Systemd_Filter _SYSTEMD_UNIT=addon-config.service
40+
Systemd_Filter _SYSTEMD_UNIT=addon-run.service
41+
Systemd_Filter _SYSTEMD_UNIT=cfn-etcd-environment.service
42+
Systemd_Filter _SYSTEMD_UNIT=cfn-signal.service
43+
Systemd_Filter _SYSTEMD_UNIT=clean-ca-certificates.service
44+
Systemd_Filter _SYSTEMD_UNIT=containerd.service
45+
Systemd_Filter _SYSTEMD_UNIT=coreos-metadata.service
46+
Systemd_Filter _SYSTEMD_UNIT=coreos-setup-environment.service
47+
Systemd_Filter _SYSTEMD_UNIT=coreos-tmpfiles.service
48+
Systemd_Filter _SYSTEMD_UNIT=dbus.service
49+
Systemd_Filter _SYSTEMD_UNIT=docker.service
50+
Systemd_Filter _SYSTEMD_UNIT=efs.service
51+
Systemd_Filter _SYSTEMD_UNIT=etcd-member.service
52+
Systemd_Filter _SYSTEMD_UNIT=etcd.service
53+
Systemd_Filter _SYSTEMD_UNIT=etcd2.service
54+
Systemd_Filter _SYSTEMD_UNIT=etcd3.service
55+
Systemd_Filter _SYSTEMD_UNIT=etcdadm-check.service
56+
Systemd_Filter _SYSTEMD_UNIT=etcdadm-reconfigure.service
57+
Systemd_Filter _SYSTEMD_UNIT=etcdadm-save.service
58+
Systemd_Filter _SYSTEMD_UNIT=etcdadm-update-status.service
59+
Systemd_Filter _SYSTEMD_UNIT=flanneld.service
60+
Systemd_Filter _SYSTEMD_UNIT=format-etcd2-volume.service
61+
Systemd_Filter _SYSTEMD_UNIT=kube-node-taint-and-uncordon.service
62+
Systemd_Filter _SYSTEMD_UNIT=kubelet.service
63+
Systemd_Filter _SYSTEMD_UNIT=ldconfig.service
64+
Systemd_Filter _SYSTEMD_UNIT=locksmithd.service
65+
Systemd_Filter _SYSTEMD_UNIT=logrotate.service
66+
Systemd_Filter _SYSTEMD_UNIT=lvm2-monitor.service
67+
Systemd_Filter _SYSTEMD_UNIT=mdmon.service
68+
Systemd_Filter _SYSTEMD_UNIT=nfs-idmapd.service
69+
Systemd_Filter _SYSTEMD_UNIT=nfs-mountd.service
70+
Systemd_Filter _SYSTEMD_UNIT=nfs-server.service
71+
Systemd_Filter _SYSTEMD_UNIT=nfs-utils.service
72+
Systemd_Filter _SYSTEMD_UNIT=node-problem-detector.service
73+
Systemd_Filter _SYSTEMD_UNIT=ntp.service
74+
Systemd_Filter _SYSTEMD_UNIT=oem-cloudinit.service
75+
Systemd_Filter _SYSTEMD_UNIT=rkt-gc.service
76+
Systemd_Filter _SYSTEMD_UNIT=rkt-metadata.service
77+
Systemd_Filter _SYSTEMD_UNIT=rpc-idmapd.service
78+
Systemd_Filter _SYSTEMD_UNIT=rpc-mountd.service
79+
Systemd_Filter _SYSTEMD_UNIT=rpc-statd.service
80+
Systemd_Filter _SYSTEMD_UNIT=rpcbind.service
81+
Systemd_Filter _SYSTEMD_UNIT=set-aws-environment.service
82+
Systemd_Filter _SYSTEMD_UNIT=system-cloudinit.service
83+
Systemd_Filter _SYSTEMD_UNIT=systemd-timesyncd.service
84+
Systemd_Filter _SYSTEMD_UNIT=update-ca-certificates.service
85+
Systemd_Filter _SYSTEMD_UNIT=user-cloudinit.service
86+
Systemd_Filter _SYSTEMD_UNIT=var-lib-etcd2.service
87+
Max_Entries 1000
88+
Read_From_Tail true
89+
outputs: |
90+
[OUTPUT]
91+
Name forward
92+
Match *
93+
Host ${FLUENTD_LOGS_SVC}.${NAMESPACE}.svc.cluster.local.
94+
Port 24321
95+
Retry_Limit False
96+
tls off
97+
tls.verify on
98+
tls.debug 1
99+
# Disable keepalive for better load balancing
100+
net.keepalive off
101+
service: |
102+
[SERVICE]
103+
Flush 1
104+
Daemon Off
105+
Log_Level info
106+
Parsers_File parsers.conf
107+
Parsers_File custom_parsers.conf
108+
HTTP_Server On
109+
HTTP_Listen 0.0.0.0
110+
HTTP_Port 2020
111+
luaScripts:
112+
parse_logs.lua: |
113+
local function split(s, delimiter)
114+
result = {};
115+
for match in (s..delimiter):gmatch('(.-)'..delimiter) do
116+
table.insert(result, match);
117+
end
118+
return result;
119+
end
120+
121+
function get_log_content(line)
122+
-- remove elements specific containerd log format and get log as single string
123+
table.remove(line, 1) -- remove date
124+
table.remove(line, 1) -- remove stream
125+
table.remove(line, 1) -- remove log tag
126+
return table.concat(line, " ")
127+
end
128+
129+
function adjust_first_line(record, first_line)
130+
-- adjust the first line to containerd format, it comes without date, stream and logtag
131+
-- 'fake-date' string at the beginning is used only to have proper log format
132+
first_line = 'fake-date' .. ' ' .. record['stream'] .. ' ' .. record['logtag'] .. ' ' .. first_line
133+
return first_line
134+
end
135+
136+
function parse_log(tag, timestamp, record)
137+
if record['log'] == nil or record['stream'] == nil or record['logtag'] == nil then
138+
-- log does not contain required attributes ('log', 'stream', 'logtag') to be processed by parse_log function
139+
-- the record will not be modified
140+
return 0, timestamp, record
141+
end
142+
143+
log_lines = split(record['log'], '\n')
144+
log_lines[1] = adjust_first_line(record, log_lines[1])
145+
146+
new_lines = {}
147+
buffer = ''
148+
149+
for k, v in pairs(log_lines) do
150+
line = split(v, ' ')
151+
log_tag = line[3]
152+
buffer = buffer .. get_log_content(line)
153+
154+
if log_tag == 'F' then
155+
table.insert(new_lines, buffer)
156+
buffer = ""
157+
end
158+
end
159+
160+
new_log = table.concat(new_lines, "\n")
161+
record['log'] = new_log
162+
return 2, timestamp, record
163+
end
164+
fluentd:
165+
logs:
166+
containers:
167+
multiline:
168+
enabled: false

‎src/go/cmd/update-collection-v3/testdata/multiline.input.yaml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
sumologic:
2+
accessId: xxx
3+
accessKey: yyy
4+
multiline_key: |-
5+
line 1
6+
line 2
7+
8+
line 3
9+
another_key: yes

‎src/go/cmd/update-collection-v3/testdata/multiline.output.yaml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
sumologic:
2+
accessId: xxx
3+
accessKey: yyy
4+
multiline_key: |-
5+
line 1
6+
line 2
7+
8+
line 3
9+
another_key: yes

0 commit comments

Comments
 (0)