Skip to content

Commit 58066be

Browse files
dasfmidasfmi
authored
Merge pull request #11 from axiomhq/fix-timeout-error
handle exit signals and shutdown http server
2 parents 3a8a79a + 5b298c7 commit 58066be

File tree

8 files changed

+378
-619
lines changed

8 files changed

+378
-619
lines changed

‎Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ package: build
88
cd bin && zip -r extension.zip extensions
99

1010
publish: package
11-
aws lambda publish-layer-version --layer-name axiom-development-lambda-extension-go --region eu-west-1 --zip-file "fileb://bin/extension.zip"
11+
aws lambda publish-layer-version --layer-name axiom-development-lambda-extension-go --region eu-west-1 --zip-file "fileb://bin/extension.zip" --compatible-architectures ${GOARCH}
1212

1313
clean:
1414
rm -r ./bin

‎flusher/flusher.go

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
package flusher
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"log"
7+
"os"
8+
"sync"
9+
"time"
10+
11+
"github.com/axiomhq/axiom-lambda-extension/version"
12+
13+
"github.com/axiomhq/axiom-go/axiom"
14+
)
15+
16+
// Axiom Config
17+
var (
18+
axiomToken = os.Getenv("AXIOM_TOKEN")
19+
axiomDataset = os.Getenv("AXIOM_DATASET")
20+
batchSize = 1000
21+
flushInterval = 1 * time.Second
22+
)
23+
24+
type Axiom struct {
25+
client *axiom.Client
26+
events []axiom.Event
27+
eventsLock sync.Mutex
28+
lastFlushTime time.Time
29+
}
30+
31+
func New() (*Axiom, error) {
32+
client, err := axiom.NewClient(
33+
axiom.SetAPITokenConfig(axiomToken),
34+
axiom.SetUserAgent(fmt.Sprintf("axiom-lambda-extension/%s", version.Get())),
35+
axiom.SetNoRetry(),
36+
)
37+
if err != nil {
38+
return nil, err
39+
}
40+
41+
f := &Axiom{
42+
client: client,
43+
events: make([]axiom.Event, 0),
44+
}
45+
46+
return f, nil
47+
}
48+
49+
func (f *Axiom) ShouldFlush() bool {
50+
f.eventsLock.Lock()
51+
defer f.eventsLock.Unlock()
52+
53+
return len(f.events) > batchSize || f.lastFlushTime.IsZero() || time.Since(f.lastFlushTime) > flushInterval
54+
}
55+
56+
func (f *Axiom) Queue(event axiom.Event) {
57+
f.eventsLock.Lock()
58+
defer f.eventsLock.Unlock()
59+
60+
f.events = append(f.events, event)
61+
}
62+
63+
func (f *Axiom) QueueEvents(events []axiom.Event) {
64+
f.eventsLock.Lock()
65+
defer f.eventsLock.Unlock()
66+
67+
f.events = append(f.events, events...)
68+
}
69+
70+
func (f *Axiom) Flush() {
71+
f.eventsLock.Lock()
72+
var batch []axiom.Event
73+
// create a copy of the batch, clear the original
74+
batch, f.events = f.events, []axiom.Event{}
75+
f.eventsLock.Unlock()
76+
77+
f.lastFlushTime = time.Now()
78+
if len(f.events) == 0 {
79+
return
80+
}
81+
82+
res, err := f.client.IngestEvents(context.Background(), axiomDataset, batch)
83+
if err != nil {
84+
log.Println(fmt.Errorf("failed to ingest events: %w", err))
85+
// allow this batch to be retried again, put them back
86+
f.eventsLock.Lock()
87+
defer f.eventsLock.Unlock()
88+
f.events = append(batch, f.events...)
89+
90+
return
91+
} else if res.Failed > 0 {
92+
log.Printf("%d failures during ingesting, %s", res.Failed, res.Failures[0].Error)
93+
}
94+
}

‎go.mod

Lines changed: 48 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,17 @@ module github.com/axiomhq/axiom-lambda-extension
33
go 1.18
44

55
require (
6-
github.com/axiomhq/axiom-go v0.14.0
7-
github.com/golangci/golangci-lint v1.50.1
6+
github.com/axiomhq/axiom-go v0.14.1
7+
github.com/axiomhq/pkg v0.3.0
8+
github.com/golangci/golangci-lint v1.51.1
89
github.com/peterbourgon/ff/v2 v2.0.1
9-
go.uber.org/zap v1.23.0
10+
go.uber.org/zap v1.24.0
1011
)
1112

1213
require (
13-
4d63.com/gochecknoglobals v0.1.0 // indirect
14-
github.com/Abirdcfly/dupword v0.0.7 // indirect
14+
4d63.com/gocheckcompilerdirectives v1.2.1 // indirect
15+
4d63.com/gochecknoglobals v0.2.1 // indirect
16+
github.com/Abirdcfly/dupword v0.0.9 // indirect
1517
github.com/Antonboom/errname v0.1.7 // indirect
1618
github.com/Antonboom/nilnil v0.1.1 // indirect
1719
github.com/BurntSushi/toml v1.2.1 // indirect
@@ -30,17 +32,17 @@ require (
3032
github.com/breml/bidichk v0.2.3 // indirect
3133
github.com/breml/errchkjson v0.3.0 // indirect
3234
github.com/butuzov/ireturn v0.1.1 // indirect
33-
github.com/cenkalti/backoff/v4 v4.1.3 // indirect
35+
github.com/cenkalti/backoff/v4 v4.2.0 // indirect
3436
github.com/cespare/xxhash/v2 v2.1.2 // indirect
3537
github.com/charithe/durationcheck v0.0.9 // indirect
36-
github.com/chavacava/garif v0.0.0-20220630083739-93517212f375 // indirect
38+
github.com/chavacava/garif v0.0.0-20221024190013-b3ef35877348 // indirect
3739
github.com/curioswitch/go-reassign v0.2.0 // indirect
38-
github.com/daixiang0/gci v0.8.1 // indirect
40+
github.com/daixiang0/gci v0.9.0 // indirect
3941
github.com/davecgh/go-spew v1.1.1 // indirect
4042
github.com/denis-tingaikin/go-header v0.4.3 // indirect
4143
github.com/esimonov/ifshort v1.0.4 // indirect
4244
github.com/ettle/strcase v0.1.1 // indirect
43-
github.com/fatih/color v1.13.0 // indirect
45+
github.com/fatih/color v1.14.1 // indirect
4446
github.com/fatih/structtag v1.2.0 // indirect
4547
github.com/felixge/httpsnoop v1.0.3 // indirect
4648
github.com/firefart/nonamedreturns v1.0.4 // indirect
@@ -50,13 +52,13 @@ require (
5052
github.com/go-logr/logr v1.2.3 // indirect
5153
github.com/go-logr/stdr v1.2.2 // indirect
5254
github.com/go-toolsmith/astcast v1.0.0 // indirect
53-
github.com/go-toolsmith/astcopy v1.0.2 // indirect
55+
github.com/go-toolsmith/astcopy v1.0.3 // indirect
5456
github.com/go-toolsmith/astequal v1.0.3 // indirect
5557
github.com/go-toolsmith/astfmt v1.0.0 // indirect
5658
github.com/go-toolsmith/astp v1.0.0 // indirect
5759
github.com/go-toolsmith/strparse v1.0.0 // indirect
5860
github.com/go-toolsmith/typep v1.0.2 // indirect
59-
github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b // indirect
61+
github.com/go-xmlfmt/xmlfmt v1.1.2 // indirect
6062
github.com/gobwas/glob v0.2.3 // indirect
6163
github.com/gofrs/flock v0.8.1 // indirect
6264
github.com/golang/protobuf v1.5.2 // indirect
@@ -66,12 +68,12 @@ require (
6668
github.com/golangci/gofmt v0.0.0-20220901101216-f2edd75033f2 // indirect
6769
github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 // indirect
6870
github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca // indirect
69-
github.com/golangci/misspell v0.3.5 // indirect
71+
github.com/golangci/misspell v0.4.0 // indirect
7072
github.com/golangci/revgrep v0.0.0-20220804021717-745bb2f7c2e6 // indirect
7173
github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 // indirect
7274
github.com/google/go-cmp v0.5.9 // indirect
7375
github.com/google/go-querystring v1.1.0 // indirect
74-
github.com/gordonklaus/ineffassign v0.0.0-20210914165742-4cc7213b9bc8 // indirect
76+
github.com/gordonklaus/ineffassign v0.0.0-20230107090616-13ace0543b28 // indirect
7577
github.com/gostaticanalysis/analysisutil v0.7.1 // indirect
7678
github.com/gostaticanalysis/comment v1.4.2 // indirect
7779
github.com/gostaticanalysis/forcetypeassert v0.1.0 // indirect
@@ -86,41 +88,42 @@ require (
8688
github.com/jingyugao/rowserrcheck v1.1.1 // indirect
8789
github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af // indirect
8890
github.com/julz/importas v0.1.0 // indirect
89-
github.com/kisielk/errcheck v1.6.2 // indirect
91+
github.com/junk1tm/musttag v0.4.4 // indirect
92+
github.com/kisielk/errcheck v1.6.3 // indirect
9093
github.com/kisielk/gotool v1.0.0 // indirect
9194
github.com/kkHAIKE/contextcheck v1.1.3 // indirect
92-
github.com/klauspost/compress v1.15.12 // indirect
95+
github.com/klauspost/compress v1.15.15 // indirect
9396
github.com/kulti/thelper v0.6.3 // indirect
9497
github.com/kunwardeep/paralleltest v1.0.6 // indirect
95-
github.com/kyoh86/exportloopref v0.1.8 // indirect
98+
github.com/kyoh86/exportloopref v0.1.11 // indirect
9699
github.com/ldez/gomoddirectives v0.2.3 // indirect
97-
github.com/ldez/tagliatelle v0.3.1 // indirect
98-
github.com/leonklingele/grouper v1.1.0 // indirect
100+
github.com/ldez/tagliatelle v0.4.0 // indirect
101+
github.com/leonklingele/grouper v1.1.1 // indirect
99102
github.com/lufeee/execinquery v1.2.1 // indirect
100103
github.com/magiconair/properties v1.8.6 // indirect
101104
github.com/maratori/testableexamples v1.0.0 // indirect
102105
github.com/maratori/testpackage v1.1.0 // indirect
103106
github.com/matoous/godox v0.0.0-20210227103229-6504466cf951 // indirect
104107
github.com/mattn/go-colorable v0.1.13 // indirect
105-
github.com/mattn/go-isatty v0.0.16 // indirect
108+
github.com/mattn/go-isatty v0.0.17 // indirect
106109
github.com/mattn/go-runewidth v0.0.14 // indirect
107110
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
108111
github.com/mbilski/exhaustivestruct v1.2.0 // indirect
109-
github.com/mgechev/revive v1.2.4 // indirect
112+
github.com/mgechev/revive v1.2.5 // indirect
110113
github.com/mitchellh/go-homedir v1.1.0 // indirect
111114
github.com/mitchellh/mapstructure v1.5.0 // indirect
112115
github.com/moricho/tparallel v0.2.1 // indirect
113116
github.com/nakabonne/nestif v0.3.1 // indirect
114117
github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354 // indirect
115-
github.com/nishanths/exhaustive v0.8.3 // indirect
118+
github.com/nishanths/exhaustive v0.9.5 // indirect
116119
github.com/nishanths/predeclared v0.2.2 // indirect
120+
github.com/nunnatsa/ginkgolinter v0.8.1 // indirect
117121
github.com/olekukonko/tablewriter v0.0.5 // indirect
118122
github.com/pelletier/go-toml v1.9.5 // indirect
119123
github.com/pelletier/go-toml/v2 v2.0.5 // indirect
120-
github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d // indirect
121124
github.com/pkg/errors v0.9.1 // indirect
122125
github.com/pmezard/go-difflib v1.0.0 // indirect
123-
github.com/polyfloyd/go-errorlint v1.0.5 // indirect
126+
github.com/polyfloyd/go-errorlint v1.0.6 // indirect
124127
github.com/prometheus/client_golang v1.13.0 // indirect
125128
github.com/prometheus/client_model v0.2.0 // indirect
126129
github.com/prometheus/common v0.37.0 // indirect
@@ -129,23 +132,23 @@ require (
129132
github.com/quasilyte/gogrep v0.0.0-20220828223005-86e4605de09f // indirect
130133
github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 // indirect
131134
github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 // indirect
132-
github.com/rivo/uniseg v0.4.2 // indirect
133-
github.com/ryancurrah/gomodguard v1.2.4 // indirect
134-
github.com/ryanrolds/sqlclosecheck v0.3.0 // indirect
135+
github.com/rivo/uniseg v0.4.3 // indirect
136+
github.com/ryancurrah/gomodguard v1.3.0 // indirect
137+
github.com/ryanrolds/sqlclosecheck v0.4.0 // indirect
135138
github.com/sanposhiho/wastedassign/v2 v2.0.7 // indirect
136139
github.com/sashamelentyev/interfacebloat v1.1.0 // indirect
137-
github.com/sashamelentyev/usestdlibvars v1.20.0 // indirect
138-
github.com/securego/gosec/v2 v2.13.1 // indirect
140+
github.com/sashamelentyev/usestdlibvars v1.21.1 // indirect
141+
github.com/securego/gosec/v2 v2.14.0 // indirect
139142
github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c // indirect
140143
github.com/sirupsen/logrus v1.9.0 // indirect
141144
github.com/sivchari/containedctx v1.0.2 // indirect
142145
github.com/sivchari/nosnakecase v1.7.0 // indirect
143-
github.com/sivchari/tenv v1.7.0 // indirect
146+
github.com/sivchari/tenv v1.7.1 // indirect
144147
github.com/sonatard/noctx v0.0.1 // indirect
145-
github.com/sourcegraph/go-diff v0.6.1 // indirect
148+
github.com/sourcegraph/go-diff v0.7.0 // indirect
146149
github.com/spf13/afero v1.8.2 // indirect
147150
github.com/spf13/cast v1.5.0 // indirect
148-
github.com/spf13/cobra v1.6.0 // indirect
151+
github.com/spf13/cobra v1.6.1 // indirect
149152
github.com/spf13/jwalterweatherman v1.1.0 // indirect
150153
github.com/spf13/pflag v1.0.5 // indirect
151154
github.com/spf13/viper v1.12.0 // indirect
@@ -154,40 +157,40 @@ require (
154157
github.com/stretchr/objx v0.5.0 // indirect
155158
github.com/stretchr/testify v1.8.1 // indirect
156159
github.com/subosito/gotenv v1.4.1 // indirect
157-
github.com/sylvia7788/contextcheck v1.0.4 // indirect
160+
github.com/t-yuki/gocover-cobertura v0.0.0-20180217150009-aaee18c8195c // indirect
158161
github.com/tdakkota/asciicheck v0.1.1 // indirect
159162
github.com/tetafro/godot v1.4.11 // indirect
160-
github.com/timakin/bodyclose v0.0.0-20210704033933-f49887972144 // indirect
163+
github.com/timakin/bodyclose v0.0.0-20221125081123-e39cf3fc478e // indirect
161164
github.com/timonwong/loggercheck v0.9.3 // indirect
162-
github.com/tomarrell/wrapcheck/v2 v2.7.0 // indirect
165+
github.com/tomarrell/wrapcheck/v2 v2.8.0 // indirect
163166
github.com/tommy-muehle/go-mnd/v2 v2.5.1 // indirect
164167
github.com/ultraware/funlen v0.0.3 // indirect
165168
github.com/ultraware/whitespace v0.0.5 // indirect
166169
github.com/uudashr/gocognit v1.0.6 // indirect
167170
github.com/yagipy/maintidx v1.0.0 // indirect
168171
github.com/yeya24/promlinter v0.2.0 // indirect
169172
gitlab.com/bosi/decorder v0.2.3 // indirect
170-
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.36.4 // indirect
171-
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.36.4 // indirect
172-
go.opentelemetry.io/otel v1.11.1 // indirect
173-
go.opentelemetry.io/otel/metric v0.33.0 // indirect
174-
go.opentelemetry.io/otel/trace v1.11.1 // indirect
173+
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.39.0 // indirect
174+
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.39.0 // indirect
175+
go.opentelemetry.io/otel v1.13.0 // indirect
176+
go.opentelemetry.io/otel/metric v0.36.0 // indirect
177+
go.opentelemetry.io/otel/trace v1.13.0 // indirect
175178
go.uber.org/atomic v1.9.0 // indirect
176179
go.uber.org/multierr v1.7.0 // indirect
177180
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect
178-
golang.org/x/exp/typeparams v0.0.0-20220827204233-334a2380cb91 // indirect
181+
golang.org/x/exp/typeparams v0.0.0-20221208152030-732eee02a75a // indirect
179182
golang.org/x/mod v0.7.0 // indirect
180183
golang.org/x/sync v0.1.0 // indirect
181-
golang.org/x/sys v0.2.0 // indirect
182-
golang.org/x/text v0.4.0 // indirect
183-
golang.org/x/tools v0.3.0 // indirect
184+
golang.org/x/sys v0.4.0 // indirect
185+
golang.org/x/text v0.6.0 // indirect
186+
golang.org/x/tools v0.5.0 // indirect
184187
google.golang.org/protobuf v1.28.1 // indirect
185188
gopkg.in/ini.v1 v1.67.0 // indirect
186189
gopkg.in/yaml.v2 v2.4.0 // indirect
187190
gopkg.in/yaml.v3 v3.0.1 // indirect
188-
honnef.co/go/tools v0.3.3 // indirect
191+
honnef.co/go/tools v0.4.0 // indirect
189192
mvdan.cc/gofumpt v0.4.0 // indirect
190193
mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed // indirect
191194
mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b // indirect
192-
mvdan.cc/unparam v0.0.0-20220706161116-678bad134442 // indirect
195+
mvdan.cc/unparam v0.0.0-20221223090309-7455f1af531d // indirect
193196
)

0 commit comments

Comments
 (0)