Skip to content

Commit dbfbf7a

Browse files
dasfmidasfmi
authored
Merge pull request #28 from axiomhq/capture-newlines-in-message
Capture newlines in message
2 parents f2e4a52 + 913e5eb commit dbfbf7a

File tree

5 files changed

+148
-24
lines changed

5 files changed

+148
-24
lines changed

‎.github/workflows/ci.yaml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,19 @@ jobs:
2828
go-version: ${{ env.GOVERSION }}
2929
- uses: golangci/golangci-lint-action@v3
3030

31+
test:
32+
name: Test
33+
runs-on: ubuntu-latest
34+
needs:
35+
- lint
36+
steps:
37+
- uses: actions/checkout@v3
38+
- uses: actions/setup-go@v3
39+
with:
40+
go-version: ${{ env.GOVERSION }}
41+
- name: unit tests
42+
run: make test
43+
3144
build:
3245
name: Build
3346
runs-on: ubuntu-latest

‎Makefile

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
GOOS=linux
22

3+
test:
4+
GOOS=${GOOS} GOARCH=${GOARCH} go test ./...
5+
36
build:
47
mkdir -p bin/extensions
58
GOOS=${GOOS} GOARCH=${GOARCH} go build -o bin/extensions/axiom-lambda-extension .

‎server/server.go

Lines changed: 33 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ var (
3838
axiomMetaInfo = map[string]string{}
3939
)
4040

41-
var logLineRgx, _ = regexp.Compile(`^([0-9.:TZ-]{20,})\s+([0-9a-f-]{36})\s+(ERROR|INFO|WARN|DEBUG|TRACE)\s+(.*)`)
41+
var logLineRgx, _ = regexp.Compile(`^([0-9.:TZ-]{20,})\s+([0-9a-f-]{36})\s+(ERROR|INFO|WARN|DEBUG|TRACE)\s+(?s:(.*))`)
4242

4343
func init() {
4444
logger, _ = zap.NewProduction()
@@ -91,28 +91,7 @@ func httpHandler(ax *flusher.Axiom, runtimeDone chan struct{}) http.HandlerFunc
9191
e["_time"], e["time"] = e["time"], nil
9292

9393
if e["type"] == "function" {
94-
e["message"] = e["record"]
95-
if recordStr, ok := e["record"].(string); ok && len(recordStr) > 0 {
96-
recordStr = strings.Trim(recordStr, "\n")
97-
// parse the record
98-
// first check if the record is a json object, if not parse it as a text log line
99-
if recordStr[0] == '{' && recordStr[len(recordStr)-1] == '}' {
100-
var record map[string]any
101-
err = json.Unmarshal([]byte(recordStr), &record)
102-
if err != nil {
103-
logger.Error("Error unmarshalling record:", zap.Error(err))
104-
// do not return, we want to continue processing the event
105-
} else {
106-
e["record"] = record
107-
}
108-
} else {
109-
matches := logLineRgx.FindStringSubmatch(recordStr)
110-
if len(matches) == 5 {
111-
e["record"] = map[string]any{"requestId": matches[2], "message": matches[4], "timestamp": matches[1], "level": e["level"]}
112-
e["level"] = strings.ToLower(matches[3])
113-
}
114-
}
115-
}
94+
extractEventMessage(e)
11695
}
11796

11897
// decide if the handler should notify the extension that the runtime is done
@@ -136,3 +115,34 @@ func httpHandler(ax *flusher.Axiom, runtimeDone chan struct{}) http.HandlerFunc
136115
}
137116
}
138117
}
118+
119+
// extractEventMessage extracts the message from the record field and puts it in the message field
120+
// it detects if the record is a json string or a text log line that confirms to AWS log line formatting.
121+
func extractEventMessage(e map[string]any) {
122+
e["message"] = e["record"]
123+
if recordStr, ok := e["record"].(string); ok && len(recordStr) > 0 {
124+
recordStr = strings.Trim(recordStr, "\n")
125+
// parse the record
126+
// first check if the record is a json object, if not parse it as a text log line
127+
if recordStr[0] == '{' && recordStr[len(recordStr)-1] == '}' {
128+
var record map[string]any
129+
err := json.Unmarshal([]byte(recordStr), &record)
130+
if err != nil {
131+
logger.Error("Error unmarshalling record:", zap.Error(err))
132+
// do not return, we want to continue processing the event
133+
} else {
134+
if level, ok := record["level"].(string); ok {
135+
record["level"] = strings.ToLower(level)
136+
}
137+
e["level"] = record["level"]
138+
e["record"] = record
139+
}
140+
} else {
141+
matches := logLineRgx.FindStringSubmatch(recordStr)
142+
if len(matches) == 5 {
143+
e["level"] = strings.ToLower(matches[3])
144+
e["record"] = map[string]any{"requestId": matches[2], "message": matches[4], "timestamp": matches[1], "level": e["level"]}
145+
}
146+
}
147+
}
148+
}

‎server/server_test.go

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
package server
2+
3+
import (
4+
"testing"
5+
)
6+
7+
func TestMessageExtraction(t *testing.T) {
8+
testCases := []struct {
9+
name string
10+
input string
11+
expected map[string]any
12+
}{
13+
{
14+
name: "error messages on multiple lines",
15+
input: "2024-01-16T08:53:51.919Z 4b995efa-75f8-4fdc-92af-0882c79f47a1 ERROR testing sending an error\nand this is a new line inside the error \n and a new line \n bye",
16+
expected: map[string]any{
17+
"level": "error",
18+
"message": "SAME_AS_INPUT_NO_NEED_TO_DUPLICATE_INPUT_HERE",
19+
"record": map[string]any{"requestId": "4b995efa-75f8-4fdc-92af-0882c79f47a1", "message": "testing sending an error\nand this is a new line inside the error \n and a new line \n bye", "timestamp": "2024-01-16T08:53:51.919Z", "level": "error"},
20+
},
21+
},
22+
{
23+
name: "info messages",
24+
input: "2024-01-16T08:53:51.919Z 4b995efa-75f8-4fdc-92af-0882c79f47a2 INFO Hello, world!",
25+
expected: map[string]any{
26+
"level": "info",
27+
"message": "SAME_AS_INPUT_NO_NEED_TO_DUPLICATE_INPUT_HERE",
28+
"record": map[string]any{"requestId": "4b995efa-75f8-4fdc-92af-0882c79f47a2", "message": "Hello, world!", "timestamp": "2024-01-16T08:53:51.919Z", "level": "info"},
29+
},
30+
},
31+
{
32+
name: "warn messages",
33+
input: "2024-01-16T08:53:51.919Z 4b995efa-75f8-4fdc-92af-0882c79f47a3 WARN head my warning",
34+
expected: map[string]any{
35+
"level": "warn",
36+
"message": "SAME_AS_INPUT_NO_NEED_TO_DUPLICATE_INPUT_HERE",
37+
"record": map[string]any{"requestId": "4b995efa-75f8-4fdc-92af-0882c79f47a3", "message": "head my warning", "timestamp": "2024-01-16T08:53:51.919Z", "level": "warn"},
38+
},
39+
},
40+
{
41+
name: "trace messages",
42+
input: "2024-01-16T08:53:51.919Z 4b995efa-75f8-4fdc-92af-0882c79f47a4 TRACE this is a trace \n with information on a new line.",
43+
expected: map[string]any{
44+
"level": "trace",
45+
"message": "SAME_AS_INPUT_NO_NEED_TO_DUPLICATE_INPUT_HERE",
46+
"record": map[string]any{"requestId": "4b995efa-75f8-4fdc-92af-0882c79f47a4", "message": "this is a trace \n with information on a new line.", "timestamp": "2024-01-16T08:53:51.919Z", "level": "trace"},
47+
},
48+
},
49+
{
50+
name: "debug messages",
51+
input: "2024-01-16T08:53:51.919Z 4b995efa-75f8-4fdc-92af-0882c79f47a5 DEBUG Debugging is fun!",
52+
expected: map[string]any{
53+
"level": "debug",
54+
"message": "SAME_AS_INPUT_NO_NEED_TO_DUPLICATE_INPUT_HERE",
55+
"record": map[string]any{"requestId": "4b995efa-75f8-4fdc-92af-0882c79f47a5", "message": "Debugging is fun!", "timestamp": "2024-01-16T08:53:51.919Z", "level": "debug"},
56+
},
57+
},
58+
{
59+
name: "testing json messages",
60+
input: `{"timestamp":"2024-01-08T16:48:45.316Z","level":"INFO","requestId":"de126cf0-6124-426c-818a-174983fbfc4b","message":"foo != bar"}`,
61+
expected: map[string]any{
62+
"level": "info",
63+
"message": "SAME_AS_INPUT_NO_NEED_TO_DUPLICATE_INPUT_HERE",
64+
"record": map[string]any{"requestId": "de126cf0-6124-426c-818a-174983fbfc4b", "message": "foo != bar", "timestamp": "2024-01-08T16:48:45.316Z", "level": "info"},
65+
},
66+
},
67+
}
68+
69+
for _, testCase := range testCases {
70+
t.Run(testCase.name, func(t *testing.T) {
71+
e := make(map[string]any)
72+
e["record"] = testCase.input
73+
extractEventMessage(e)
74+
if e["level"] != testCase.expected["level"] {
75+
t.Errorf("Expected level to be %s, got %s", testCase.expected["level"], e["level"])
76+
}
77+
if e["message"] != testCase.input { // the message field should contain the original input
78+
t.Errorf("Expected message to be %s, got %s", testCase.input, e["message"])
79+
}
80+
81+
expectedRecord := testCase.expected["record"].(map[string]any)
82+
outputRecord := e["record"].(map[string]any)
83+
84+
if outputRecord["timestamp"] != expectedRecord["timestamp"] {
85+
t.Errorf("Expected timestamp to be %s, got %s", testCase.expected["timestamp"], e["timestamp"])
86+
}
87+
if outputRecord["level"] != expectedRecord["level"] {
88+
t.Errorf("Expected record.level to be %s, got %s", expectedRecord["level"], outputRecord["level"])
89+
}
90+
if outputRecord["requestId"] != expectedRecord["requestId"] {
91+
t.Errorf("Expected record.requestId to be %s, got %s", expectedRecord["requestId"], outputRecord["requestId"])
92+
}
93+
if outputRecord["message"] != expectedRecord["message"] {
94+
t.Errorf("Expected record.message to be %s, got %s", expectedRecord["message"], outputRecord["message"])
95+
}
96+
})
97+
}
98+
}

‎version/version.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
package version
22

33
// manually set constant version
4-
const version string = "v10"
4+
const version string = "v11"
55

66
// Get returns the Go module version of the axiom-go module.
77
func Get() string {

0 commit comments

Comments
 (0)