Merge pull request #35 from coreruleset/feat/default-config

feat: default global config

Author: fzipi

crslang_test.go

@@ -4,62 +4,19 @@ import (
 	"os"
 	"testing"
 
-	"github.com/antlr4-go/antlr/v4"
-	"github.com/coreruleset/crslang/listener"
 	"github.com/coreruleset/crslang/types"
-	"github.com/coreruleset/seclang_parser/parser"
 	"go.yaml.in/yaml/v4"
 )
 
-var testFiles = []string{
-	"testdata/crs-setup.conf",
-	"testdata/crs/REQUEST-900-EXCLUSION-RULES-BEFORE-CRS.conf",
-	"testdata/crs/REQUEST-901-INITIALIZATION.conf",
-	"testdata/crs/REQUEST-905-COMMON-EXCEPTIONS.conf",
-	"testdata/crs/REQUEST-911-METHOD-ENFORCEMENT.conf",
-	"testdata/crs/REQUEST-913-SCANNER-DETECTION.conf",
-	"testdata/crs/REQUEST-920-PROTOCOL-ENFORCEMENT.conf",
-	"testdata/crs/REQUEST-921-PROTOCOL-ATTACK.conf",
-	"testdata/crs/REQUEST-922-MULTIPART-ATTACK.conf",
-	"testdata/crs/REQUEST-930-APPLICATION-ATTACK-LFI.conf",
-	"testdata/crs/REQUEST-931-APPLICATION-ATTACK-RFI.conf",
-	"testdata/crs/REQUEST-932-APPLICATION-ATTACK-RCE.conf",
-	"testdata/crs/REQUEST-933-APPLICATION-ATTACK-PHP.conf",
-	"testdata/crs/REQUEST-934-APPLICATION-ATTACK-GENERIC.conf",
-	"testdata/crs/REQUEST-941-APPLICATION-ATTACK-XSS.conf",
-	"testdata/crs/REQUEST-942-APPLICATION-ATTACK-SQLI.conf",
-	"testdata/crs/REQUEST-943-APPLICATION-ATTACK-SESSION-FIXATION.conf",
-	"testdata/crs/REQUEST-944-APPLICATION-ATTACK-JAVA.conf",
-	"testdata/crs/REQUEST-949-BLOCKING-EVALUATION.conf",
-	"testdata/crs/RESPONSE-950-DATA-LEAKAGES.conf",
-	"testdata/crs/RESPONSE-951-DATA-LEAKAGES-SQL.conf",
-	"testdata/crs/RESPONSE-952-DATA-LEAKAGES-JAVA.conf",
-	"testdata/crs/RESPONSE-953-DATA-LEAKAGES-PHP.conf",
-	"testdata/crs/RESPONSE-954-DATA-LEAKAGES-IIS.conf",
-	"testdata/crs/RESPONSE-959-BLOCKING-EVALUATION.conf",
-	"testdata/crs/RESPONSE-980-CORRELATION.conf",
-}
+var testFilepath = "testdata/crs/"
 
 func TestLoadCRS(t *testing.T) {
-	resultConfigs := []types.DirectiveList{}
-	for _, file := range testFiles {
-		input, err := antlr.NewFileStream(file)
-		if err != nil {
-			panic("Error reading file" + file)
-		}
-		lexer := parser.NewSecLangLexer(input)
-		stream := antlr.NewCommonTokenStream(lexer, 0)
-		p := parser.NewSecLangParser(stream)
-		start := p.Configuration()
-		var seclangListener listener.ExtendedSeclangParserListener
-		antlr.ParseTreeWalkerDefault.Walk(&seclangListener, start)
-		resultConfigs = append(resultConfigs, seclangListener.ConfigurationList.DirectiveList...)
-	}
-	configList := types.ConfigurationList{DirectiveList: resultConfigs}
 
-	configListWithConditions := types.ToDirectiveWithConditions(configList)
+	configList := LoadSeclang(testFilepath)
+
+	configList = *ToCRSLang(configList)
 
-	yamlFile, err := yaml.Marshal(configListWithConditions.DirectiveList)
+	yamlFile, err := yaml.Marshal(configList)
 	if err != nil {
 		t.Errorf("Error marshalling yaml: %v", err)
 	}
@@ -68,12 +25,8 @@ func TestLoadCRS(t *testing.T) {
 
 	defer os.Remove("tmp_crslang.yaml")
 
-	if err != nil {
-		t.Errorf("Error writing file: %v", err)
-	}
-
 	loadedConfigList := types.LoadDirectivesWithConditionsFromFile("tmp_crslang.yaml")
-	yamlLoadedFile, err := yaml.Marshal(loadedConfigList.DirectiveList)
+	yamlLoadedFile, err := yaml.Marshal(loadedConfigList)
 	if err != nil {
 		t.Errorf("Error writing file: %v", err)
 	}
@@ -84,32 +37,16 @@ func TestLoadCRS(t *testing.T) {
 }
 
 func TestFromCRSLangToSeclang(t *testing.T) {
-	resultConfigs := []types.DirectiveList{}
-	for _, file := range testFiles {
-		input, err := antlr.NewFileStream(file)
-		if err != nil {
-			panic("Error reading file" + file)
-		}
-		lexer := parser.NewSecLangLexer(input)
-		stream := antlr.NewCommonTokenStream(lexer, 0)
-		p := parser.NewSecLangParser(stream)
-		start := p.Configuration()
-		var seclangListener listener.ExtendedSeclangParserListener
-		antlr.ParseTreeWalkerDefault.Walk(&seclangListener, start)
-		resultConfigs = append(resultConfigs, seclangListener.ConfigurationList.DirectiveList...)
-	}
-	configList := types.ConfigurationList{DirectiveList: resultConfigs}
+	configList := LoadSeclang(testFilepath)
 
 	seclangDirectives := types.ToSeclang(configList)
 
-	configListWithConditions := types.ToDirectiveWithConditions(configList)
-
-	configListFromConditions := types.FromCRSLangToUnformattedDirectives(*configListWithConditions)
-
-	seclangDirectivesFromConditions := types.ToSeclang(*configListFromConditions)
+	crslangConfigList := ToCRSLang(configList)
+	unformattedDirectives := types.FromCRSLangToUnformattedDirectives(*crslangConfigList)
+	seclangDirectivesFromConditions := types.ToSeclang(*unformattedDirectives)
 
-	if seclangDirectives != seclangDirectivesFromConditions {
-		t.Errorf("Error in CRSLang to Seclang directives convertion. Expected length: %v, got: %v", len(seclangDirectives), len(seclangDirectivesFromConditions))
+	if len(seclangDirectives) != len(seclangDirectivesFromConditions) {
+		t.Errorf("Error in CRSLang to Seclang directives conversion. Expected length: %v, got: %v", len(seclangDirectives), len(seclangDirectivesFromConditions))
 	}
 
 }

main.go

@@ -47,38 +47,15 @@ Flags:
 	}
 
 	if !*toSeclang {
-		resultConfigs := []types.DirectiveList{}
-		filepath.Walk(pathArg, func(path string, info os.FileInfo, err error) error {
-			if err != nil {
-				return err
-			}
-			if !info.IsDir() && filepath.Ext(info.Name()) == ".conf" {
-				input, err := antlr.NewFileStream(path)
-				if err != nil {
-					panic("Error reading file" + path)
-				}
-				lexer := parser.NewSecLangLexer(input)
-				stream := antlr.NewCommonTokenStream(lexer, 0)
-				p := parser.NewSecLangParser(stream)
-				start := p.Configuration()
-				var seclangListener listener.ExtendedSeclangParserListener
-				antlr.ParseTreeWalkerDefault.Walk(&seclangListener, start)
-				for i := range seclangListener.ConfigurationList.DirectiveList {
-					seclangListener.ConfigurationList.DirectiveList[i].Id = strings.TrimSuffix(filepath.Base(info.Name()), filepath.Ext(info.Name()))
-					if len(seclangListener.ConfigurationList.DirectiveList) > 1 {
-						seclangListener.ConfigurationList.DirectiveList[i].Id += "_" + strconv.Itoa(i+1)
-					}
-				}
-				resultConfigs = append(resultConfigs, seclangListener.ConfigurationList.DirectiveList...)
-			}
-			return nil
-		})
-		configList := types.ConfigurationList{DirectiveList: resultConfigs}
+		configList := LoadSeclang(pathArg)
+
+		configList = *ToCRSLang(configList)
 
 		if *output == "" {
 			*output = "crslang"
 		}
-		err := printCRSLang(configList, *output+".yaml")
+
+		err := printYAML(configList, *output+".yaml")
 		if err != nil {
 			log.Fatal(err.Error())
 		}
@@ -89,25 +66,53 @@ Flags:
 
 		configList := types.LoadDirectivesWithConditionsFromFile(pathArg)
 
-		err := printSeclang(configList, *output)
+		err := PrintSeclang(configList, *output)
 		if err != nil {
 			log.Fatal(err.Error())
 		}
 	}
 }
 
-// printSeclang writes seclang directives to files specified in directive list ids.
-func printSeclang(configList types.ConfigurationList, dir string) error {
-	unfDirs := types.FromCRSLangToUnformattedDirectives(configList)
-
-	for _, dirList := range unfDirs.DirectiveList {
-		f, err := os.Create(dir + dirList.Id + ".conf")
+// LoadSeclang loads seclang directives from an input file or folder and returns a ConfigurationList
+// if a folder is specified it loads all .conf files in the folder
+func LoadSeclang(input string) types.ConfigurationList {
+	resultConfigs := []types.DirectiveList{}
+	filepath.Walk(input, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
-		seclangDirectives := dirList.ToSeclang()
+		if ext := filepath.Ext(info.Name()); !info.IsDir() && (ext == ".conf" || (ext == ".example" && filepath.Ext(strings.TrimSuffix(info.Name(), ext)) == ".conf")) {
+			input, err := antlr.NewFileStream(path)
+			if err != nil {
+				panic("Error reading file" + path)
+			}
+			lexer := parser.NewSecLangLexer(input)
+			stream := antlr.NewCommonTokenStream(lexer, 0)
+			p := parser.NewSecLangParser(stream)
+			start := p.Configuration()
+			var seclangListener listener.ExtendedSeclangParserListener
+			antlr.ParseTreeWalkerDefault.Walk(&seclangListener, start)
+			for i := range seclangListener.ConfigurationList.DirectiveList {
+				seclangListener.ConfigurationList.DirectiveList[i].Id = strings.TrimSuffix(filepath.Base(info.Name()), filepath.Ext(info.Name()))
+				if len(seclangListener.ConfigurationList.DirectiveList) > 1 {
+					seclangListener.ConfigurationList.DirectiveList[i].Id += "_" + strconv.Itoa(i+1)
+				}
+			}
+			resultConfigs = append(resultConfigs, seclangListener.ConfigurationList.DirectiveList...)
+		}
+		return nil
+	})
+	configList := types.ConfigurationList{DirectiveList: resultConfigs}
+	return configList
+}
 
-		_, err = io.WriteString(f, seclangDirectives)
+// PrintSeclang writes seclang directives to files specified in directive list ids.
+func PrintSeclang(configList types.ConfigurationList, dir string) error {
+	unfDirs := types.FromCRSLangToUnformattedDirectives(configList)
+
+	for _, dirList := range unfDirs.DirectiveList {
+		seclangDirectives := dirList.ToSeclang()
+		err := writeToFile([]byte(seclangDirectives), dir+dirList.Id+".conf")
 		if err != nil {
 			return err
 		}
@@ -116,28 +121,17 @@ func printSeclang(configList types.ConfigurationList, dir string) error {
 	return nil
 }
 
-// printSeclangToFile writes seclang format directives to a file
-func printSeclangToFile(configList types.ConfigurationList, filename string) error {
-	f, err := os.Create(filename)
-	if err != nil {
-		return err
-	}
-
-	seclangDirectives := types.ToSeclang(configList)
-
-	_, err = io.WriteString(f, seclangDirectives)
-	if err != nil {
-		return err
-	}
+// ToCRSLang process previously loaded seclang directives to CRSLang schema directives
+func ToCRSLang(configList types.ConfigurationList) *types.ConfigurationList {
+	configListWithConditions := types.ToDirectiveWithConditions(configList)
 
-	return nil
+	configListWithConditions.ExtractDefaultValues()
+	return configListWithConditions
 }
 
-// printCRSLang writes crslang format directives (directives with conditions) to a file
-func printCRSLang(configList types.ConfigurationList, filename string) error {
-	configListWithConditions := types.ToDirectiveWithConditions(configList)
-
-	yamlFile, err := yaml.Marshal(configListWithConditions.DirectiveList)
+// printYAML marshal and write structures to a yaml file
+func printYAML(input any, filename string) error {
+	yamlFile, err := yaml.Marshal(input)
 	if err != nil {
 		return err
 	}

testdata/crs/REQUEST-900-EXCLUSION-RULES-BEFORE-CRS.conf

@@ -199,9 +199,9 @@
 #           ctl:ruleRemoveById=920350,\
 #           ctl:ruleRemoveByTag=attack-disclosure"
 
-SecRule REQUEST_FILENAME "@endsWith wp-login.php" \
-    "id:1003,\
-    phase:2,\
-    pass,\
-    nolog,\
-    ctl:ruleRemoveTargetByTag=OWASP_CRS;ARGS:pwd"
+# SecRule REQUEST_FILENAME "@endsWith wp-login.php" \
+#     "id:1003,\
+#     phase:2,\
+#     pass,\
+#     nolog,\
+#     ctl:ruleRemoveTargetByTag=OWASP_CRS;ARGS:pwd"

types/condition_directives.go

@@ -47,10 +47,14 @@ type RuleWithCondition struct {
 	ChainedRule *RuleWithCondition `yaml:"chainedRule,omitempty"`
 }
 
-func (s RuleWithCondition) ToSeclang() string {
+func (s *RuleWithCondition) ToSeclang() string {
 	return "New sec rule with conditions"
 }
 
+func (s *RuleWithCondition) GetKind() Kind {
+	return s.Kind
+}
+
 func ToDirectiveWithConditions(configList ConfigurationList) *ConfigurationList {
 	result := new(ConfigurationList)
 	for _, config := range configList.DirectiveList {
@@ -81,7 +85,7 @@ func ToDirectiveWithConditions(configList ConfigurationList) *ConfigurationList
 	return result
 }
 
-func RuleToCondition(directive ChainableDirective) RuleWithCondition {
+func RuleToCondition(directive ChainableDirective) *RuleWithCondition {
 	var ruleWithCondition RuleWithCondition
 	switch directive.(type) {
 	case *SecRule:
@@ -131,7 +135,7 @@ func RuleToCondition(directive ChainableDirective) RuleWithCondition {
 	if directive.GetChainedDirective() != nil {
 		chainedConditionRule := RuleToCondition(directive.GetChainedDirective())
 		if directive.NonDisruptiveActionsCount() > 0 {
-			ruleWithCondition.ChainedRule = &chainedConditionRule
+			ruleWithCondition.ChainedRule = chainedConditionRule
 		} else {
 			ruleWithCondition.Conditions = append(ruleWithCondition.Conditions, chainedConditionRule.Conditions...)
 			ruleWithCondition.Actions.NonDisruptiveActions = chainedConditionRule.Actions.NonDisruptiveActions
@@ -140,7 +144,13 @@ func RuleToCondition(directive ChainableDirective) RuleWithCondition {
 			}
 		}
 	}
-	return ruleWithCondition
+	return &ruleWithCondition
+}
+
+// configurationYamlLoader is a auxiliary struct to load the whole yaml file
+type configurationYamlLoader struct {
+	Global        DefaultConfigs             `yaml:"global,omitempty"`
+	DirectiveList []yamlLoaderConditionRules `yaml:"directivelist,omitempty"`
 }
 
 // yamlLoaderConditionRules is a auxiliary struct to load and iterate over the yaml file
@@ -224,8 +234,9 @@ func LoadDirectivesWithConditionsFromFile(filename string) ConfigurationList {
 
 // LoadDirectivesWithConditions loads condition format directives from a yaml file
 func LoadDirectivesWithConditions(yamlFile []byte) ConfigurationList {
-	var configs []yamlLoaderConditionRules
-	err := yaml.Unmarshal(yamlFile, &configs)
+	var config configurationYamlLoader
+	err := yaml.Unmarshal(yamlFile, &config)
+	configs := config.DirectiveList
 	if err != nil {
 		panic(err)
 	}
@@ -242,7 +253,7 @@ func LoadDirectivesWithConditions(yamlFile []byte) ConfigurationList {
 		}
 		resultConfigs = append(resultConfigs, DirectiveList{Id: config.Id, Directives: directives, Marker: config.Marker})
 	}
-	return ConfigurationList{DirectiveList: resultConfigs}
+	return ConfigurationList{Global: config.Global, DirectiveList: resultConfigs}
 }
 
 // loadConditionDirective loads the different kind of directives
@@ -324,7 +335,7 @@ func loadConditionDirective(yamlDirective yaml.Node) SeclangDirective {
 }
 
 // loadRuleWithConditions loads a rule with conditions in a recursive way
-func loadRuleWithConditions(yamlDirective yaml.Node) RuleWithCondition {
+func loadRuleWithConditions(yamlDirective yaml.Node) *RuleWithCondition {
 	rawDirective := []byte{}
 	var err error
 
@@ -340,7 +351,7 @@ func loadRuleWithConditions(yamlDirective yaml.Node) RuleWithCondition {
 		print(string(rawDirective))
 		panic(err)
 	}
-	directive := RuleWithCondition{
+	directive := &RuleWithCondition{
 		Kind:     RuleKind,
 		Metadata: loaderDirective.Metadata,
 		Actions:  loaderDirective.Actions,
@@ -351,10 +362,10 @@ func loadRuleWithConditions(yamlDirective yaml.Node) RuleWithCondition {
 			directive.Conditions = append(directive.Conditions, loadedCondition)
 		}
 	}
-	var loadedChainedRule RuleWithCondition
+	var loadedChainedRule *RuleWithCondition
 	if len(loaderDirective.ChainedRule.Content) > 0 {
 		loadedChainedRule = loadRuleWithConditions(loaderDirective.ChainedRule)
-		directive.ChainedRule = &loadedChainedRule
+		directive.ChainedRule = loadedChainedRule
 	}
 	return directive
 }
@@ -400,8 +411,15 @@ func FromCRSLangToUnformattedDirectives(configListWrapped ConfigurationList) *Co
 				directive = directiveWrapped.(CommentDirective).Metadata
 			case DefaultAction:
 				directive = directiveWrapped
-			case RuleWithCondition:
-				directive = FromConditionToUnmorfattedDirective(directiveWrapped.(RuleWithCondition))
+			case *RuleWithCondition:
+				chainableDir := FromConditionToUnmorfattedDirective(*directiveWrapped.(*RuleWithCondition))
+				if configListWrapped.Global.Version != "" {
+					chainableDir.GetMetadata().SetVer(configListWrapped.Global.Version)
+				}
+				for _, tag := range configListWrapped.Global.Tags {
+					chainableDir.GetMetadata().AddTag(tag)
+				}
+				directive = chainableDir
 			case ConfigurationDirective:
 				directive = ConfigurationDirective{
 					Metadata:  directiveWrapped.(ConfigurationDirective).Metadata,