From abd890c0d719e4c040df46379dbe3304a9a33864 Mon Sep 17 00:00:00 2001 From: Fuchun Zhang Date: Fri, 1 Dec 2023 14:56:47 +0800 Subject: [PATCH 1/3] 1.add /metrics for this app; 2.add to avoid too many alerts; --- .gitignore | 2 ++ Makefile | 3 ++ README.md | 4 +++ cmd/prometheus-webhook-dingtalk/main.go | 27 ++++++++++++++++- go.mod | 5 +++- go.sum | 8 +++++ web/dingtalk/dingtalk.go | 39 +++++++++++++++++++++---- web/web.go | 6 ++++ 8 files changed, 87 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index ecf6c682..35419615 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ /.vscode /.build /vendor +build/ +test/ *.tar.gz coverage.txt diff --git a/Makefile b/Makefile index cdb11a58..16ebc7b2 100644 --- a/Makefile +++ b/Makefile @@ -61,3 +61,6 @@ test: common-test .PHONY: clean clean: - @rm -rf "$(REACT_APP_OUTPUT_DIR)"s + +go-build: + go build -o build/dingtalk-webhook cmd/prometheus-webhook-dingtalk/main.go diff --git a/README.md b/README.md index 1cec2822..2f139a78 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,10 @@ Flags: --config.file=config.yml Path to the configuration file. --log.level=info Only log messages with the given severity or above. One of: [debug, info, warn, error] --log.format=logfmt Output format of log messages. One of: [logfmt, json] + --pushmetrics.extraLabel extraLabel for push metrics. see: https://github.com/VictoriaMetrics/metrics + --pushmetrics.interval=15s interval for push metrics. + --pushmetrics.url urls for push metrics.eg: http://host.docker.internal:8480/insert/0/prometheus/api/v1/import/prometheus. + --maxalertcount=30 max alert count to send to ding talk. --version Show application version. ``` diff --git a/cmd/prometheus-webhook-dingtalk/main.go b/cmd/prometheus-webhook-dingtalk/main.go index 49dbe093..7db352cf 100644 --- a/cmd/prometheus-webhook-dingtalk/main.go +++ b/cmd/prometheus-webhook-dingtalk/main.go @@ -16,6 +16,7 @@ import ( "github.com/prometheus/common/version" "gopkg.in/alecthomas/kingpin.v2" + "github.com/VictoriaMetrics/metrics" "github.com/timonwong/prometheus-webhook-dingtalk/config" "github.com/timonwong/prometheus-webhook-dingtalk/template" "github.com/timonwong/prometheus-webhook-dingtalk/web" @@ -43,6 +44,24 @@ func run() int { "config.file", "Path to the configuration file.", ).Default("config.yml").ExistingFile() + // for push metrics + extraLabel = kingpin.Flag( + "pushmetrics.extraLabel", + "extraLabel for push metrics.", + ).Default("").String() + intervalForPushMetrics = kingpin.Flag( + "pushmetrics.interval", + "interval for push metrics.", + ).Default("15s").Duration() + urlForPushMetrics = kingpin.Flag( + "pushmetrics.url", + "urls for push metrics.", + ).Default("").String() + // aviod too many alert + maxAlertCount = kingpin.Flag( + "maxalertcount", + "max alert count to send to ding talk.", + ).Default("30").Uint16() ) // DO NOT REMOVE. For compatibility purpose @@ -59,6 +78,11 @@ func run() int { level.Info(logger).Log("msg", "Starting prometheus-webhook-dingtalk", "version", version.Info()) level.Info(logger).Log("msg", "Build context", version.BuildContext()) + if len(*urlForPushMetrics) > 0 { + metrics.InitPush(*urlForPushMetrics, + *intervalForPushMetrics, *extraLabel, true) + } + flagsMap := map[string]string{} // Exclude kingpin default flags to expose only Prometheus ones. boilerplateFlags := kingpin.New("", "").Version("") @@ -87,7 +111,8 @@ func run() int { BuildDate: version.BuildDate, GoVersion: version.GoVersion, }, - Flags: flagsMap, + Flags: flagsMap, + MaxAlertCount: *maxAlertCount, }) configLogger := log.With(logger, "component", "configuration") diff --git a/go.mod b/go.mod index 2830dba3..a1a455b7 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( require ( github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.1.1 // indirect + github.com/VictoriaMetrics/metrics v1.25.3 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -34,7 +35,9 @@ require ( github.com/prometheus/procfs v0.7.3 // indirect github.com/shopspring/decimal v1.2.0 // indirect github.com/spf13/cast v1.3.1 // indirect + github.com/valyala/fastrand v1.1.0 // indirect + github.com/valyala/histogram v1.2.0 // indirect golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect - golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect + golang.org/x/sys v0.15.0 // indirect google.golang.org/protobuf v1.26.0 // indirect ) diff --git a/go.sum b/go.sum index c78058a1..c3ddf9a5 100644 --- a/go.sum +++ b/go.sum @@ -39,6 +39,8 @@ github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030I github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8= github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk= +github.com/VictoriaMetrics/metrics v1.25.3 h1:Zcxyj8JbAB6CQU51Er3D7RBRupcP55DevVQi9cFqo2Q= +github.com/VictoriaMetrics/metrics v1.25.3/go.mod h1:ZKmlI+QN6b9LUC0OiHNp2LiGQGlBy4U1re6Slooln1o= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -232,6 +234,10 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8= +github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= +github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ= +github.com/valyala/histogram v1.2.0/go.mod h1:Hb4kBwb4UxsaNbbbh+RRz8ZR6pdodR57tzWUS3BUzXY= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -366,6 +372,8 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/web/dingtalk/dingtalk.go b/web/dingtalk/dingtalk.go index dfaf403f..dafd1ad0 100644 --- a/web/dingtalk/dingtalk.go +++ b/web/dingtalk/dingtalk.go @@ -2,6 +2,7 @@ package dingtalk import ( "encoding/json" + "fmt" "io" "net/http" "sync" @@ -11,6 +12,7 @@ import ( "github.com/go-kit/log" "github.com/go-kit/log/level" + "github.com/VictoriaMetrics/metrics" "github.com/timonwong/prometheus-webhook-dingtalk/config" "github.com/timonwong/prometheus-webhook-dingtalk/notifier" "github.com/timonwong/prometheus-webhook-dingtalk/pkg/chilog" @@ -27,6 +29,8 @@ type API struct { targets map[string]config.Target httpClient *http.Client logger log.Logger + + MaxAlertCount uint16 // to avoid too many alert } func NewAPI(logger log.Logger) *API { @@ -60,6 +64,7 @@ func (api *API) Routes() chi.Router { } func (api *API) serveSend(w http.ResponseWriter, r *http.Request) { + metrics.GetOrCreateCounter("http_requests_total{path=\"" + r.RequestURI + "\"}").Inc() api.mtx.RLock() targets := api.targets conf := api.conf @@ -72,36 +77,60 @@ func (api *API) serveSend(w http.ResponseWriter, r *http.Request) { target, ok := targets[targetName] if !ok { - level.Warn(logger).Log("msg", "target not found") + const reason = "target not found" + level.Warn(logger).Log("msg", reason) http.NotFound(w, r) + metrics.GetOrCreateCounter( + fmt.Sprintf(`http_requests_error{path="%s",reason="%s"}`, r.RequestURI, reason)).Inc() return } var promMessage models.WebhookMessage if err := json.NewDecoder(r.Body).Decode(&promMessage); err != nil { - level.Error(logger).Log("msg", "Cannot decode prometheus webhook JSON request", "err", err) + const reason = "Cannot decode prometheus webhook JSON request" + level.Error(logger).Log("msg", reason, "err", err) http.Error(w, "Bad Request", http.StatusBadRequest) + metrics.GetOrCreateCounter( + fmt.Sprintf(`http_requests_error{path="%s",reason="%s"}`, r.RequestURI, reason)).Inc() return } + metrics.GetOrCreateCounter( + fmt.Sprintf(`alert_count{path="%s"}`, r.RequestURI)).Add(len(promMessage.Alerts)) + if api.MaxAlertCount > 0 && len(promMessage.Alerts) > int(api.MaxAlertCount) { + metrics.GetOrCreateCounter( + fmt.Sprintf(`alert_drop_count{path="%s"}`, r.RequestURI)).Add(len(promMessage.Alerts) - int(api.MaxAlertCount)) + promMessage.Alerts = promMessage.Alerts[:api.MaxAlertCount] + } + builder := notifier.NewDingNotificationBuilder(tmpl, conf, &target) notification, err := builder.Build(&promMessage) if err != nil { - level.Error(logger).Log("msg", "Failed to build notification", "err", err) + const reason = "Failed to build notification" + level.Error(logger).Log("msg", reason, "err", err) http.Error(w, "Bad Request", http.StatusBadRequest) + metrics.GetOrCreateCounter( + fmt.Sprintf(`http_requests_error{path="%s",reason="%s"}`, r.RequestURI, reason)).Inc() return } robotResp, err := notifier.SendNotification(notification, httpClient, &target) if err != nil { - level.Error(logger).Log("msg", "Failed to send notification", "err", err) + const reason = "Failed to send notification" + level.Error(logger).Log("msg", reason, "err", err) http.Error(w, "Bad Request", http.StatusBadRequest) + metrics.GetOrCreateCounter( + fmt.Sprintf(`http_requests_error{path="%s",reason="%s"}`, r.RequestURI, reason)).Inc() return } if robotResp.ErrorCode != 0 { - level.Error(logger).Log("msg", "Failed to send notification to DingTalk", "respCode", robotResp.ErrorCode, "respMsg", robotResp.ErrorMessage) + const reason = "Failed to send notification to DingTalk" + level.Error(logger).Log("msg", reason, "respCode", robotResp.ErrorCode, "respMsg", robotResp.ErrorMessage) http.Error(w, "Unable to talk to DingTalk", http.StatusBadRequest) + metrics.GetOrCreateCounter( + fmt.Sprintf(`http_requests_error{path="%s",reason="%s",respCode="%d",respMsg="%s"}`, + r.RequestURI, reason, robotResp.ErrorCode, robotResp.ErrorMessage)).Inc() return } diff --git a/web/web.go b/web/web.go index 06746ca1..5dcd3086 100644 --- a/web/web.go +++ b/web/web.go @@ -34,6 +34,7 @@ import ( "github.com/prometheus/common/server" "go.uber.org/atomic" + "github.com/VictoriaMetrics/metrics" "github.com/timonwong/prometheus-webhook-dingtalk/config" "github.com/timonwong/prometheus-webhook-dingtalk/template" "github.com/timonwong/prometheus-webhook-dingtalk/web/apiv1" @@ -57,6 +58,7 @@ type Options struct { EnableLifecycle bool Version *VersionInfo Flags map[string]string + MaxAlertCount uint16 } type VersionInfo = apiv1.VersionInfo @@ -120,6 +122,7 @@ func New(logger log.Logger, o *Options) *Handler { h.runtimeInfo, ) h.dingTalk = dingtalk.NewAPI(logger) + h.dingTalk.MaxAlertCount = o.MaxAlertCount router.Mount("/dingtalk", h.dingTalk.Routes()) @@ -189,6 +192,9 @@ func New(logger log.Logger, o *Options) *Handler { }) } + router.Get("/metrics", func(w http.ResponseWriter, r *http.Request) { + metrics.WritePrometheus(w, true) + }) return h } From 88ff3eb305921bb9173f119ff9e16f6f79cdf9de Mon Sep 17 00:00:00 2001 From: Fuchun Zhang Date: Fri, 1 Dec 2023 14:59:27 +0800 Subject: [PATCH 2/3] 1.add /metrics for this app; 2.add to avoid too many alerts; 3.modify readme --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2f139a78..a15372e6 100644 --- a/README.md +++ b/README.md @@ -58,10 +58,11 @@ Flags: --config.file=config.yml Path to the configuration file. --log.level=info Only log messages with the given severity or above. One of: [debug, info, warn, error] --log.format=logfmt Output format of log messages. One of: [logfmt, json] - --pushmetrics.extraLabel extraLabel for push metrics. see: https://github.com/VictoriaMetrics/metrics - --pushmetrics.interval=15s interval for push metrics. - --pushmetrics.url urls for push metrics.eg: http://host.docker.internal:8480/insert/0/prometheus/api/v1/import/prometheus. - --maxalertcount=30 max alert count to send to ding talk. + --pushmetrics.extraLabel Extra label for push metrics. see: https://github.com/VictoriaMetrics/metrics + --pushmetrics.interval=15s + Interval for push metrics. + --pushmetrics.url Urls for push metrics.eg: http://host.docker.internal:8480/insert/0/prometheus/api/v1/import/prometheus. + --maxalertcount=30 Max alert count to send to ding talk. --version Show application version. ``` From e3a5f0381bd00fa52d79beffe14ba1dbcf06c5a4 Mon Sep 17 00:00:00 2001 From: Fuchun Zhang Date: Mon, 4 Dec 2023 10:52:10 +0800 Subject: [PATCH 3/3] add docker build cmd --- Dockerfile_linux_amd64 | 3 +++ Dockerfile_linux_arm64 | 3 +++ Makefile | 21 +++++++++++++++++++-- 3 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 Dockerfile_linux_amd64 create mode 100644 Dockerfile_linux_arm64 diff --git a/Dockerfile_linux_amd64 b/Dockerfile_linux_amd64 new file mode 100644 index 00000000..ad560012 --- /dev/null +++ b/Dockerfile_linux_amd64 @@ -0,0 +1,3 @@ +FROM alpine:3.16 +ADD build/dingtalk-webhook-linux-amd64 /dingtalk-webhook-linux-amd64 +ENTRYPOINT [ "/dingtalk-webhook-linux-amd64"] diff --git a/Dockerfile_linux_arm64 b/Dockerfile_linux_arm64 new file mode 100644 index 00000000..83cd2ef9 --- /dev/null +++ b/Dockerfile_linux_arm64 @@ -0,0 +1,3 @@ +FROM alpine:3.16 +ADD build/dingtalk-webhook-linux-arm64 /dingtalk-webhook-linux-arm64 +ENTRYPOINT [ "/dingtalk-webhook-linux-arm64"] diff --git a/Makefile b/Makefile index 16ebc7b2..bfe8f5c0 100644 --- a/Makefile +++ b/Makefile @@ -62,5 +62,22 @@ test: common-test clean: - @rm -rf "$(REACT_APP_OUTPUT_DIR)"s -go-build: - go build -o build/dingtalk-webhook cmd/prometheus-webhook-dingtalk/main.go +go-build-linux-arm64: + GOOS=linux GOARCH=arm64 \ + go build -o build/dingtalk-webhook-linux-arm64 cmd/prometheus-webhook-dingtalk/main.go + +go-build-linux-amd64: + GOOS=linux GOARCH=amd64 \ + go build -o build/dingtalk-webhook-linux-amd64 cmd/prometheus-webhook-dingtalk/main.go + +docker-build-linux-arm64: + docker build -t dingtalk-webhook:v2.1.1 --platform=linux/arm64 -f Dockerfile_linux_arm64 . + +docker-build-linux-amd64: + docker build -t dingtalk-webhook:v2.1.1 --platform=linux/amd64 -f Dockerfile_linux_amd64 . + +docker-tag: + docker tag dingtalk-webhook:v2.1.1 docker.io/ahfuzhang/dingtalk-webhook:v2.1.1 + +docker-push: + docker push docker.io/ahfuzhang/dingtalk-webhook:v2.1.1