Skip to content

Commit b8c2a57

Browse files
authored
feat: add structured log writer (#246)
1 parent 298bc02 commit b8c2a57

File tree

7 files changed

+409
-2
lines changed

7 files changed

+409
-2
lines changed

funcframework/framework.go

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -198,7 +198,7 @@ func wrapHTTPFunction(fn func(http.ResponseWriter, *http.Request)) (http.Handler
198198
defer fmt.Println()
199199
defer fmt.Fprintln(os.Stderr)
200200
}
201-
r, cancel := setContextTimeoutIfRequested(r)
201+
r, cancel := setupRequestContext(r)
202202
if cancel != nil {
203203
defer cancel()
204204
}
@@ -218,7 +218,7 @@ func wrapEventFunction(fn interface{}) (http.Handler, error) {
218218
defer fmt.Println()
219219
defer fmt.Fprintln(os.Stderr)
220220
}
221-
r, cancel := setContextTimeoutIfRequested(r)
221+
r, cancel := setupRequestContext(r)
222222
if cancel != nil {
223223
defer cancel()
224224
}
@@ -398,6 +398,12 @@ func writeHTTPErrorResponse(w http.ResponseWriter, statusCode int, status, msg s
398398
fmt.Fprint(w, msg)
399399
}
400400

401+
func setupRequestContext(r *http.Request) (*http.Request, func()) {
402+
r, cancel := setContextTimeoutIfRequested(r)
403+
r = addLoggingIDsToRequest(r)
404+
return r, cancel
405+
}
406+
401407
// setContextTimeoutIfRequested replaces the request's context with a cancellation if requested
402408
func setContextTimeoutIfRequested(r *http.Request) (*http.Request, func()) {
403409
timeoutStr := os.Getenv("CLOUD_RUN_TIMEOUT_SECONDS")

funcframework/logging.go

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
package funcframework
2+
3+
import (
4+
"bufio"
5+
"context"
6+
"encoding/json"
7+
"io"
8+
"net/http"
9+
"os"
10+
"regexp"
11+
"sync"
12+
)
13+
14+
var (
15+
loggingIDsContextKey contextKey = "loggingIDs"
16+
validXCloudTraceContext = regexp.MustCompile(
17+
// Matches on "TRACE_ID"
18+
`([a-f\d]+)?` +
19+
// Matches on "/SPAN_ID"
20+
`(?:/([a-f\d]+))?` +
21+
// Matches on ";0=TRACE_TRUE"
22+
`(?:;o=(\d))?`)
23+
)
24+
25+
type loggingIDs struct {
26+
trace string
27+
spanID string
28+
executionID string
29+
}
30+
31+
type contextKey string
32+
33+
func addLoggingIDsToRequest(r *http.Request) *http.Request {
34+
executionID := r.Header.Get("Function-Execution-Id")
35+
traceID, spanID, _ := deconstructXCloudTraceContext(r.Header.Get("X-Cloud-Trace-Context"))
36+
37+
if executionID == "" && traceID == "" && spanID == "" {
38+
return r
39+
}
40+
41+
r = r.WithContext(contextWithLoggingIDs(r.Context(), &loggingIDs{
42+
trace: traceID,
43+
spanID: spanID,
44+
executionID: executionID,
45+
}))
46+
47+
return r
48+
}
49+
50+
func contextWithLoggingIDs(ctx context.Context, loggingIDs *loggingIDs) context.Context {
51+
return context.WithValue(ctx, loggingIDsContextKey, loggingIDs)
52+
}
53+
54+
func loggingIDsFromContext(ctx context.Context) *loggingIDs {
55+
val := ctx.Value(loggingIDsContextKey)
56+
if val == nil {
57+
return nil
58+
}
59+
return val.(*loggingIDs)
60+
}
61+
62+
func TraceIDFromContext(ctx context.Context) string {
63+
ids := loggingIDsFromContext(ctx)
64+
if ids == nil {
65+
return ""
66+
}
67+
return ids.trace
68+
}
69+
70+
func ExecutionIDFromContext(ctx context.Context) string {
71+
ids := loggingIDsFromContext(ctx)
72+
if ids == nil {
73+
return ""
74+
}
75+
return ids.executionID
76+
}
77+
78+
func SpanIDFromContext(ctx context.Context) string {
79+
ids := loggingIDsFromContext(ctx)
80+
if ids == nil {
81+
return ""
82+
}
83+
return ids.spanID
84+
}
85+
86+
func deconstructXCloudTraceContext(s string) (traceID, spanID string, traceSampled bool) {
87+
// As per the format described at https://cloud.google.com/trace/docs/setup#force-trace
88+
// "X-Cloud-Trace-Context: TRACE_ID/SPAN_ID;o=TRACE_TRUE"
89+
// for example:
90+
// "X-Cloud-Trace-Context: 105445aa7843bc8bf206b120001000/1;o=1"
91+
matches := validXCloudTraceContext.FindStringSubmatch(s)
92+
if matches != nil {
93+
traceID, spanID, traceSampled = matches[1], matches[2], matches[3] == "1"
94+
}
95+
if spanID == "0" {
96+
spanID = ""
97+
}
98+
return
99+
}
100+
101+
// structuredLogEvent declares a subset of the fields supported by cloudlogging structured log events.
102+
// See https://cloud.google.com/logging/docs/structured-logging.
103+
type structuredLogEvent struct {
104+
Message string `json:"message"`
105+
Trace string `json:"logging.googleapis.com/trace,omitempty"`
106+
SpanID string `json:"logging.googleapis.com/spanId,omitempty"`
107+
Labels map[string]string `json:"logging.googleapis.com/labels,omitempty"`
108+
}
109+
110+
// structuredLogWriter writes structured logs
111+
type structuredLogWriter struct {
112+
mu sync.Mutex
113+
w io.Writer
114+
loggingIDs loggingIDs
115+
buf []byte
116+
}
117+
118+
func (w *structuredLogWriter) writeStructuredLog(loggingIDs loggingIDs, message string) (int, error) {
119+
event := structuredLogEvent{
120+
Message: message,
121+
Trace: loggingIDs.trace,
122+
SpanID: loggingIDs.spanID,
123+
}
124+
if loggingIDs.executionID != "" {
125+
event.Labels = map[string]string{
126+
"execution_id": loggingIDs.executionID,
127+
}
128+
}
129+
130+
marshalled, err := json.Marshal(event)
131+
if err != nil {
132+
return 0, err
133+
}
134+
marshalled = append(marshalled, '\n')
135+
return w.w.Write(marshalled)
136+
}
137+
138+
func (w *structuredLogWriter) Write(output []byte) (int, error) {
139+
w.mu.Lock()
140+
defer w.mu.Unlock()
141+
142+
w.buf = append(w.buf, output...)
143+
buf := w.buf
144+
wroteLines := 0
145+
for {
146+
advance, token, err := bufio.ScanLines(buf, false)
147+
if token == nil || err != nil {
148+
break
149+
}
150+
buf = buf[advance:]
151+
if _, err := w.writeStructuredLog(w.loggingIDs, string(token)); err != nil {
152+
return 0, err
153+
}
154+
wroteLines += 1
155+
}
156+
157+
if wroteLines > 0 {
158+
// Compact the buffer by copying remaining bytes to the start.
159+
w.buf = append(w.buf[:0], buf...)
160+
}
161+
162+
return len(output), nil
163+
}
164+
165+
func (w *structuredLogWriter) Close() error {
166+
if len(w.buf) == 0 {
167+
return nil
168+
}
169+
_, err := w.writeStructuredLog(w.loggingIDs, string(w.buf))
170+
return err
171+
}
172+
173+
// LogWriter returns an io.Writer as a log sink for the request context.
174+
// One log event is generated for each new line terminated byte sequence
175+
// written to the io.Writer.
176+
//
177+
// This can be used with common logging frameworks, for example:
178+
//
179+
// import (
180+
// "log"
181+
// "github.com/GoogleCloudPlatform/functions-framework-go/funcframework"
182+
// )
183+
// ...
184+
// func helloWorld(w http.ResponseWriter, r *http.Request) {
185+
// l := logger.New(funcframework.LogWriter(r.Context()))
186+
// l.Println("hello world!")
187+
// }
188+
func LogWriter(ctx context.Context) io.WriteCloser {
189+
loggingIDs := loggingIDsFromContext(ctx)
190+
if loggingIDs == nil {
191+
return os.Stderr
192+
}
193+
194+
return &structuredLogWriter{
195+
w: os.Stderr,
196+
loggingIDs: *loggingIDs,
197+
}
198+
}

funcframework/logging_test.go

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
package funcframework
2+
3+
import (
4+
"bytes"
5+
"fmt"
6+
"log"
7+
"net/http/httptest"
8+
"testing"
9+
)
10+
11+
func TestLoggingIDExtraction(t *testing.T) {
12+
tcs := []struct {
13+
name string
14+
headers map[string]string
15+
wantTraceID string
16+
wantSpanID string
17+
wantExecutionID string
18+
}{
19+
{
20+
name: "no IDs",
21+
headers: map[string]string{},
22+
},
23+
{
24+
name: "execution ID only",
25+
headers: map[string]string{
26+
"Function-Execution-Id": "exec id",
27+
},
28+
wantExecutionID: "exec id",
29+
},
30+
{
31+
name: "malformatted X-Cloud-Trace-Context",
32+
headers: map[string]string{
33+
"X-Cloud-Trace-Context": "$*#$(v434)",
34+
},
35+
},
36+
{
37+
name: "trace ID only",
38+
headers: map[string]string{
39+
"X-Cloud-Trace-Context": "0123456789abcdef",
40+
},
41+
wantTraceID: "0123456789abcdef",
42+
},
43+
{
44+
name: "trace ID and span ID",
45+
headers: map[string]string{
46+
"X-Cloud-Trace-Context": "0123456789abcdef/aaaaaa",
47+
},
48+
wantTraceID: "0123456789abcdef",
49+
wantSpanID: "aaaaaa",
50+
},
51+
{
52+
name: "all",
53+
headers: map[string]string{
54+
"X-Cloud-Trace-Context": "a/b",
55+
"Function-Execution-Id": "c",
56+
},
57+
wantTraceID: "a",
58+
wantSpanID: "b",
59+
wantExecutionID: "c",
60+
},
61+
}
62+
63+
for _, tc := range tcs {
64+
t.Run(tc.name, func(t *testing.T) {
65+
r := httptest.NewRequest("POST", "/", bytes.NewReader(nil))
66+
for k, v := range tc.headers {
67+
r.Header.Set(k, v)
68+
}
69+
r = addLoggingIDsToRequest(r)
70+
ctx := r.Context()
71+
72+
if tid := TraceIDFromContext(ctx); tid != tc.wantTraceID {
73+
t.Errorf("expected trace id %q but got %q", tc.wantTraceID, tid)
74+
}
75+
76+
if spid := SpanIDFromContext(ctx); spid != tc.wantSpanID {
77+
t.Errorf("expected span id %q but got %q", tc.wantSpanID, spid)
78+
}
79+
80+
if eid := ExecutionIDFromContext(ctx); eid != tc.wantExecutionID {
81+
t.Errorf("expected execution id %q but got %q", tc.wantExecutionID, eid)
82+
}
83+
})
84+
}
85+
}
86+
87+
func TestStructuredLogWriter(t *testing.T) {
88+
output := bytes.NewBuffer(nil)
89+
90+
w := &structuredLogWriter{
91+
w: output,
92+
loggingIDs: loggingIDs{
93+
spanID: "a",
94+
trace: "b",
95+
executionID: "c",
96+
},
97+
}
98+
99+
fmt.Fprintf(w, "hello world!\n")
100+
fmt.Fprintf(w, "this is another log line!\n")
101+
102+
wantOutput := `{"message":"hello world!","logging.googleapis.com/trace":"b","logging.googleapis.com/spanId":"a","logging.googleapis.com/labels":{"execution_id":"c"}}
103+
{"message":"this is another log line!","logging.googleapis.com/trace":"b","logging.googleapis.com/spanId":"a","logging.googleapis.com/labels":{"execution_id":"c"}}
104+
`
105+
if output.String() != wantOutput {
106+
t.Errorf("expected output %q got %q", wantOutput, output.String())
107+
}
108+
}
109+
110+
func TestLogPackageCompat(t *testing.T) {
111+
output := bytes.NewBuffer(nil)
112+
w := &structuredLogWriter{
113+
w: output,
114+
loggingIDs: loggingIDs{
115+
spanID: "a",
116+
trace: "b",
117+
executionID: "c",
118+
},
119+
}
120+
121+
l := log.New(w, "", 0)
122+
l.Print("go logger line")
123+
l.Print("a second log line")
124+
l.Print("a multiline\nstring in a single log\ncall")
125+
126+
wantOutput := `{"message":"go logger line","logging.googleapis.com/trace":"b","logging.googleapis.com/spanId":"a","logging.googleapis.com/labels":{"execution_id":"c"}}
127+
{"message":"a second log line","logging.googleapis.com/trace":"b","logging.googleapis.com/spanId":"a","logging.googleapis.com/labels":{"execution_id":"c"}}
128+
{"message":"a multiline","logging.googleapis.com/trace":"b","logging.googleapis.com/spanId":"a","logging.googleapis.com/labels":{"execution_id":"c"}}
129+
{"message":"string in a single log","logging.googleapis.com/trace":"b","logging.googleapis.com/spanId":"a","logging.googleapis.com/labels":{"execution_id":"c"}}
130+
{"message":"call","logging.googleapis.com/trace":"b","logging.googleapis.com/spanId":"a","logging.googleapis.com/labels":{"execution_id":"c"}}
131+
`
132+
if output.String() != wantOutput {
133+
t.Errorf("expected output %q got %q", wantOutput, output.String())
134+
}
135+
}

go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ go 1.11
44

55
require (
66
cloud.google.com/go/functions v1.16.2
7+
cloud.google.com/go/logging v1.10.0 // indirect
78
github.com/cloudevents/sdk-go/v2 v2.14.0
89
github.com/google/go-cmp v0.6.0
910
)

0 commit comments

Comments
 (0)