Skip to content

Commit f3341c1

Browse files
blotusblotus
authored
Appsec: properly populate event (#2943)
1 parent 9088f31 commit f3341c1

File tree

19 files changed

+333
-142
lines changed

19 files changed

+333
-142
lines changed

‎cmd/crowdsec-cli/alerts.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -493,6 +493,7 @@ func (cli *cliAlerts) NewInspectCmd() *cobra.Command {
493493
switch cfg.Cscli.Output {
494494
case "human":
495495
if err := cli.displayOneAlert(alert, details); err != nil {
496+
log.Warnf("unable to display alert with id %s: %s", alertID, err)
496497
continue
497498
}
498499
case "json":

‎cmd/crowdsec/crowdsec.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919
"github.com/crowdsecurity/crowdsec/pkg/appsec"
2020
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
2121
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
22+
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
2223
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
2324
"github.com/crowdsecurity/crowdsec/pkg/parser"
2425
"github.com/crowdsecurity/crowdsec/pkg/types"
@@ -32,6 +33,13 @@ func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, []
3233
return nil, nil, fmt.Errorf("while loading context: %w", err)
3334
}
3435

36+
err = exprhelpers.GeoIPInit(hub.GetDataDir())
37+
38+
if err != nil {
39+
//GeoIP databases are not mandatory, do not make crowdsec fail if they are not present
40+
log.Warnf("unable to initialize GeoIP: %s", err)
41+
}
42+
3543
// Start loading configs
3644
csParsers := parser.NewParsers(hub)
3745
if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {

‎cmd/crowdsec/serve.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,9 @@ func ShutdownCrowdsecRoutines() error {
177177
// He's dead, Jim.
178178
crowdsecTomb.Kill(nil)
179179

180+
// close the potential geoips reader we have to avoid leaking ressources on reload
181+
exprhelpers.GeoIPClose()
182+
180183
return reterr
181184
}
182185

‎pkg/acquisition/modules/appsec/utils.go

Lines changed: 133 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,46 @@
11
package appsecacquisition
22

33
import (
4-
"encoding/json"
54
"fmt"
5+
"net"
6+
"slices"
7+
"strconv"
68
"time"
79

810
"github.com/crowdsecurity/coraza/v3/collection"
911
"github.com/crowdsecurity/coraza/v3/types/variables"
12+
"github.com/crowdsecurity/crowdsec/pkg/alertcontext"
1013
"github.com/crowdsecurity/crowdsec/pkg/appsec"
14+
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
1115
"github.com/crowdsecurity/crowdsec/pkg/models"
1216
"github.com/crowdsecurity/crowdsec/pkg/types"
1317
"github.com/crowdsecurity/go-cs-lib/ptr"
18+
"github.com/oschwald/geoip2-golang"
1419
"github.com/prometheus/client_golang/prometheus"
1520
log "github.com/sirupsen/logrus"
1621
)
1722

23+
var appsecMetaKeys = []string{
24+
"id",
25+
"name",
26+
"method",
27+
"uri",
28+
"matched_zones",
29+
"msg",
30+
}
31+
32+
func appendMeta(meta models.Meta, key string, value string) models.Meta {
33+
if value == "" {
34+
return meta
35+
}
36+
37+
meta = append(meta, &models.MetaItems0{
38+
Key: key,
39+
Value: value,
40+
})
41+
return meta
42+
}
43+
1844
func AppsecEventGeneration(inEvt types.Event) (*types.Event, error) {
1945
//if the request didnd't trigger inband rules, we don't want to generate an event to LAPI/CAPI
2046
if !inEvt.Appsec.HasInBandMatches {
@@ -23,48 +49,127 @@ func AppsecEventGeneration(inEvt types.Event) (*types.Event, error) {
2349
evt := types.Event{}
2450
evt.Type = types.APPSEC
2551
evt.Process = true
52+
sourceIP := inEvt.Parsed["source_ip"]
2653
source := models.Source{
27-
Value: ptr.Of(inEvt.Parsed["source_ip"]),
28-
IP: inEvt.Parsed["source_ip"],
54+
Value: &sourceIP,
55+
IP: sourceIP,
2956
Scope: ptr.Of(types.Ip),
3057
}
3158

59+
asndata, err := exprhelpers.GeoIPASNEnrich(sourceIP)
60+
61+
if err != nil {
62+
log.Errorf("Unable to enrich ip '%s' for ASN: %s", sourceIP, err)
63+
} else if asndata != nil {
64+
record := asndata.(*geoip2.ASN)
65+
source.AsName = record.AutonomousSystemOrganization
66+
source.AsNumber = fmt.Sprintf("%d", record.AutonomousSystemNumber)
67+
}
68+
69+
cityData, err := exprhelpers.GeoIPEnrich(sourceIP)
70+
if err != nil {
71+
log.Errorf("Unable to enrich ip '%s' for geo data: %s", sourceIP, err)
72+
} else if cityData != nil {
73+
record := cityData.(*geoip2.City)
74+
source.Cn = record.Country.IsoCode
75+
source.Latitude = float32(record.Location.Latitude)
76+
source.Longitude = float32(record.Location.Longitude)
77+
}
78+
79+
rangeData, err := exprhelpers.GeoIPRangeEnrich(sourceIP)
80+
if err != nil {
81+
log.Errorf("Unable to enrich ip '%s' for range: %s", sourceIP, err)
82+
} else if rangeData != nil {
83+
record := rangeData.(*net.IPNet)
84+
source.Range = record.String()
85+
}
86+
3287
evt.Overflow.Sources = make(map[string]models.Source)
33-
evt.Overflow.Sources["ip"] = source
88+
evt.Overflow.Sources[sourceIP] = source
3489

3590
alert := models.Alert{}
3691
alert.Capacity = ptr.Of(int32(1))
37-
alert.Events = make([]*models.Event, 0)
38-
alert.Meta = make(models.Meta, 0)
39-
for _, key := range []string{"target_uri", "method"} {
92+
alert.Events = make([]*models.Event, len(evt.Appsec.GetRuleIDs()))
4093

41-
valueByte, err := json.Marshal([]string{inEvt.Parsed[key]})
42-
if err != nil {
43-
log.Debugf("unable to serialize key %s", key)
94+
now := ptr.Of(time.Now().UTC().Format(time.RFC3339))
95+
96+
tmpAppsecContext := make(map[string][]string)
97+
98+
for _, matched_rule := range inEvt.Appsec.MatchedRules {
99+
evtRule := models.Event{}
100+
101+
evtRule.Timestamp = now
102+
103+
evtRule.Meta = make(models.Meta, 0)
104+
105+
for _, key := range appsecMetaKeys {
106+
107+
if tmpAppsecContext[key] == nil {
108+
tmpAppsecContext[key] = make([]string, 0)
109+
}
110+
111+
switch value := matched_rule[key].(type) {
112+
case string:
113+
evtRule.Meta = appendMeta(evtRule.Meta, key, value)
114+
if value != "" && !slices.Contains(tmpAppsecContext[key], value) {
115+
tmpAppsecContext[key] = append(tmpAppsecContext[key], value)
116+
}
117+
case int:
118+
val := strconv.Itoa(value)
119+
evtRule.Meta = appendMeta(evtRule.Meta, key, val)
120+
if val != "" && !slices.Contains(tmpAppsecContext[key], val) {
121+
tmpAppsecContext[key] = append(tmpAppsecContext[key], val)
122+
}
123+
case []string:
124+
for _, v := range value {
125+
evtRule.Meta = appendMeta(evtRule.Meta, key, v)
126+
if v != "" && !slices.Contains(tmpAppsecContext[key], v) {
127+
tmpAppsecContext[key] = append(tmpAppsecContext[key], v)
128+
}
129+
}
130+
case []int:
131+
for _, v := range value {
132+
val := strconv.Itoa(v)
133+
evtRule.Meta = appendMeta(evtRule.Meta, key, val)
134+
if val != "" && !slices.Contains(tmpAppsecContext[key], val) {
135+
tmpAppsecContext[key] = append(tmpAppsecContext[key], val)
136+
}
137+
138+
}
139+
default:
140+
val := fmt.Sprintf("%v", value)
141+
evtRule.Meta = appendMeta(evtRule.Meta, key, val)
142+
if val != "" && !slices.Contains(tmpAppsecContext[key], val) {
143+
tmpAppsecContext[key] = append(tmpAppsecContext[key], val)
144+
}
145+
146+
}
147+
}
148+
alert.Events = append(alert.Events, &evtRule)
149+
}
150+
151+
metas := make([]*models.MetaItems0, 0)
152+
153+
for key, values := range tmpAppsecContext {
154+
if len(values) == 0 {
44155
continue
45156
}
46157

158+
valueStr, err := alertcontext.TruncateContext(values, alertcontext.MaxContextValueLen)
159+
if err != nil {
160+
log.Warningf(err.Error())
161+
}
162+
47163
meta := models.MetaItems0{
48164
Key: key,
49-
Value: string(valueByte),
50-
}
51-
alert.Meta = append(alert.Meta, &meta)
52-
}
53-
matchedZones := inEvt.Appsec.GetMatchedZones()
54-
if matchedZones != nil {
55-
valueByte, err := json.Marshal(matchedZones)
56-
if err != nil {
57-
log.Debugf("unable to serialize key matched_zones")
58-
} else {
59-
meta := models.MetaItems0{
60-
Key: "matched_zones",
61-
Value: string(valueByte),
62-
}
63-
alert.Meta = append(alert.Meta, &meta)
165+
Value: valueStr,
64166
}
167+
metas = append(metas, &meta)
65168
}
66169

67-
alert.EventsCount = ptr.Of(int32(1))
170+
alert.Meta = metas
171+
172+
alert.EventsCount = ptr.Of(int32(len(alert.Events)))
68173
alert.Leakspeed = ptr.Of("")
69174
alert.Scenario = ptr.Of(inEvt.Appsec.MatchedRules.GetName())
70175
alert.ScenarioHash = ptr.Of(inEvt.Appsec.MatchedRules.GetHash())
@@ -200,7 +305,7 @@ func (r *AppsecRunner) AccumulateTxToEvent(evt *types.Event, req *appsec.ParsedR
200305
})
201306

202307
for _, rule := range req.Tx.MatchedRules() {
203-
if rule.Message() == "" || rule.DisruptiveAction() == "pass" || rule.DisruptiveAction() == "allow" {
308+
if rule.Message() == "" {
204309
r.logger.Tracef("discarding rule %d (action: %s)", rule.Rule().ID(), rule.DisruptiveAction())
205310
continue
206311
}
@@ -242,7 +347,7 @@ func (r *AppsecRunner) AccumulateTxToEvent(evt *types.Event, req *appsec.ParsedR
242347

243348
corazaRule := map[string]interface{}{
244349
"id": rule.Rule().ID(),
245-
"uri": evt.Parsed["uri"],
350+
"uri": evt.Parsed["target_uri"],
246351
"rule_type": kind,
247352
"method": evt.Parsed["method"],
248353
"disruptive": rule.Disruptive(),

‎pkg/alertcontext/alertcontext.go

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import (
1616
)
1717

1818
const (
19-
maxContextValueLen = 4000
19+
MaxContextValueLen = 4000
2020
)
2121

2222
var alertContext = Context{}
@@ -46,13 +46,13 @@ func NewAlertContext(contextToSend map[string][]string, valueLength int) error {
4646
}
4747

4848
if valueLength == 0 {
49-
clog.Debugf("No console context value length provided, using default: %d", maxContextValueLen)
50-
valueLength = maxContextValueLen
49+
clog.Debugf("No console context value length provided, using default: %d", MaxContextValueLen)
50+
valueLength = MaxContextValueLen
5151
}
5252

53-
if valueLength > maxContextValueLen {
54-
clog.Debugf("Provided console context value length (%d) is higher than the maximum, using default: %d", valueLength, maxContextValueLen)
55-
valueLength = maxContextValueLen
53+
if valueLength > MaxContextValueLen {
54+
clog.Debugf("Provided console context value length (%d) is higher than the maximum, using default: %d", valueLength, MaxContextValueLen)
55+
valueLength = MaxContextValueLen
5656
}
5757

5858
alertContext = Context{
@@ -85,7 +85,7 @@ func NewAlertContext(contextToSend map[string][]string, valueLength int) error {
8585
return nil
8686
}
8787

88-
func truncate(values []string, contextValueLen int) (string, error) {
88+
func TruncateContext(values []string, contextValueLen int) (string, error) {
8989
valueByte, err := json.Marshal(values)
9090
if err != nil {
9191
return "", fmt.Errorf("unable to dump metas: %w", err)
@@ -159,7 +159,7 @@ func EventToContext(events []types.Event) (models.Meta, []error) {
159159
continue
160160
}
161161

162-
valueStr, err := truncate(values, alertContext.ContextValueLen)
162+
valueStr, err := TruncateContext(values, alertContext.ContextValueLen)
163163
if err != nil {
164164
log.Warningf(err.Error())
165165
}

‎pkg/exprhelpers/expr_lib.go

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
package exprhelpers
22

33
import (
4+
"net"
45
"time"
56

67
"github.com/crowdsecurity/crowdsec/pkg/cticlient"
8+
"github.com/oschwald/geoip2-golang"
79
)
810

911
type exprCustomFunc struct {
@@ -469,6 +471,27 @@ var exprFuncs = []exprCustomFunc{
469471
new(func(string) bool),
470472
},
471473
},
474+
{
475+
name: "GeoIPEnrich",
476+
function: GeoIPEnrich,
477+
signature: []interface{}{
478+
new(func(string) *geoip2.City),
479+
},
480+
},
481+
{
482+
name: "GeoIPASNEnrich",
483+
function: GeoIPASNEnrich,
484+
signature: []interface{}{
485+
new(func(string) *geoip2.ASN),
486+
},
487+
},
488+
{
489+
name: "GeoIPRangeEnrich",
490+
function: GeoIPRangeEnrich,
491+
signature: []interface{}{
492+
new(func(string) *net.IPNet),
493+
},
494+
},
472495
}
473496

474497
//go 1.20 "CutPrefix": strings.CutPrefix,

0 commit comments

Comments
 (0)