From 3334d2a6b26d560ad6d8b820ee6d3fb56e327a75 Mon Sep 17 00:00:00 2001 From: Ervin Hegedus Date: Tue, 5 Feb 2019 22:52:54 +0000 Subject: [PATCH 1/5] Regex search in collection data case insensitive fix --- src/collection/backend/in_memory-per_process.cc | 2 +- src/utils/regex.cc | 12 ++++++++++++ src/utils/regex.h | 3 ++- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/collection/backend/in_memory-per_process.cc b/src/collection/backend/in_memory-per_process.cc index 9f13fa589f..ca9107619f 100644 --- a/src/collection/backend/in_memory-per_process.cc +++ b/src/collection/backend/in_memory-per_process.cc @@ -134,7 +134,7 @@ void InMemoryPerProcess::resolveRegularExpression(const std::string& var, //std::string name = std::string(var, var.find(":") + 2, // var.size() - var.find(":") - 3); //size_t keySize = col.size(); - Utils::Regex r(var); + Utils::Regex r(var, PCRE_CASELESS); for (const auto& x : *this) { //if (x.first.size() <= keySize + 1) { diff --git a/src/utils/regex.cc b/src/utils/regex.cc index 461f0288a7..128284b612 100644 --- a/src/utils/regex.cc +++ b/src/utils/regex.cc @@ -49,6 +49,18 @@ Regex::Regex(const std::string& pattern_) m_pce = pcre_study(m_pc, pcre_study_opt, &errptr); } +Regex::Regex(const std::string& pattern_, int flags) + : pattern(pattern_.empty() ? ".*" : pattern_){ + const char *errptr = NULL; + int erroffset; + + flags |= (PCRE_DOTALL|PCRE_MULTILINE); + + m_pc = pcre_compile(pattern.c_str(), flags, + &errptr, &erroffset, NULL); + + m_pce = pcre_study(m_pc, pcre_study_opt, &errptr); +} Regex::~Regex() { if (m_pc != NULL) { diff --git a/src/utils/regex.h b/src/utils/regex.h index 147b48f48e..08205a9726 100644 --- a/src/utils/regex.h +++ b/src/utils/regex.h @@ -27,7 +27,7 @@ namespace modsecurity { namespace Utils { -#define OVECCOUNT 30 +#define OVECCOUNT 50 class SMatch { public: @@ -51,6 +51,7 @@ class SMatch { class Regex { public: explicit Regex(const std::string& pattern_); + explicit Regex(const std::string& pattern_, int flags); ~Regex(); // m_pc and m_pce can't be easily copied From 4e1b99ad90a0cfa07bf856984c2c657e12fa2cd2 Mon Sep 17 00:00:00 2001 From: Ervin Hegedus Date: Tue, 5 Feb 2019 23:03:33 +0000 Subject: [PATCH 2/5] Fixed regex case insensitive search bug in LMDB --- src/collection/backend/lmdb.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/collection/backend/lmdb.cc b/src/collection/backend/lmdb.cc index 2b8724c786..d789967499 100644 --- a/src/collection/backend/lmdb.cc +++ b/src/collection/backend/lmdb.cc @@ -538,7 +538,7 @@ void LMDB::resolveRegularExpression(const std::string& var, MDB_cursor *cursor; size_t pos; - Utils::Regex r(var); + Utils::Regex r(var, PCRE_CASELESS); rc = mdb_txn_begin(m_env, NULL, 0, &txn); lmdb_debug(rc, "txn", "resolveRegularExpression"); From 640daa49025a59aee039dfda0a1f34f137415f32 Mon Sep 17 00:00:00 2001 From: Ervin Hegedus Date: Tue, 5 Feb 2019 23:09:20 +0000 Subject: [PATCH 3/5] Added test cases --- test/test-cases/regression/rule-920450.json | 90 +++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 test/test-cases/regression/rule-920450.json diff --git a/test/test-cases/regression/rule-920450.json b/test/test-cases/regression/rule-920450.json new file mode 100644 index 0000000000..91bcb72e48 --- /dev/null +++ b/test/test-cases/regression/rule-920450.json @@ -0,0 +1,90 @@ +[ + { + "enabled":1, + "version_min":300000, + "title":"Testing setvar :: OWASP CRS id:920450 ('Translate' match)", + "client":{ + "ip":"200.249.12.31", + "port":123 + }, + "server":{ + "ip":"200.249.12.31", + "port":80 + }, + "request":{ + "headers":{ + "Translate": "test" + }, + "uri":"/", + "method":"GET" + }, + "response":{ + "headers":{ + "Date":"Mon, 13 Jul 2015 20:02:41 GMT", + "Last-Modified":"Sun, 26 Oct 2014 22:33:37 GMT", + "Content-Type":"text/html" + }, + "body":[ + "no need." + ] + }, + "expected":{ + "http_code":400 + }, + "rules":[ + "SecRuleEngine On", + "SecDefaultAction \"phase:2,deny,block,status:400,log\"", + "SecRule &TX:restricted_headers \"@eq 0\" \"id:901165,phase:1,pass,nolog,setvar:'tx.restricted_headers=/proxy/ /lock-token/ /content-range/ /translate/ /if/'\"", + "SecRule REQUEST_HEADERS_NAMES \"@rx ^.*$\" \"id:920450,phase:2,block,capture,t:none,t:lowercase,msg:'HTTP header is restricted by policy (%{MATCHED_VAR})',logdata:' Restricted header detected %{MATCHED_VAR}',ver:'OWASP_CRS/3.1.0',severity:'CRITICAL',setvar:'tx.header_name_%{tx.0}=/%{tx.0}/',chain", + "SecRule TX:/^HEADER_NAME_/ \"@within %{tx.restricted_headers}\" \"setvar:'tx.msg=%{rule.msg}',setvar:'tx.anomaly_score_pl1=+%{tx.critical_anomaly_score}',setvar:'tx.%{rule.id}-OWASP_CRS/POLICY/HEADERS_RESTRICTED-%{MATCHED_VAR_NAME}=%{MATCHED_VAR}'" + ] + }, + { + "enabled":1, + "version_min":300000, + "title":"Testing setvar :: OWASP CRS id:920450 ('Proxy-Connection' not match)", + "client":{ + "ip":"200.249.12.31", + "port":123 + }, + "server":{ + "ip":"200.249.12.31", + "port":80 + }, + "request":{ + "headers":{ + "Accept": "text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5", + "Accept-Charset": "ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Encoding": "gzip,deflate", + "Accept-Language": "en-us,en;q=0.5", + "Host": "localhost", + "Keep-Alive": "300", + "Proxy-Connection": "keep-alive", + "User-Agent": "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv" + }, + "uri":"/", + "method":"GET" + }, + "response":{ + "headers":{ + "Date":"Mon, 13 Jul 2015 20:02:41 GMT", + "Last-Modified":"Sun, 26 Oct 2014 22:33:37 GMT", + "Content-Type":"text/html" + }, + "body":[ + "no need." + ] + }, + "expected":{ + "http_code":200 + }, + "rules":[ + "SecRuleEngine On", + "SecDefaultAction \"phase:2,deny,block,status:400,log\"", + "SecRule &TX:restricted_headers \"@eq 0\" \"id:901165,phase:1,pass,nolog,setvar:'tx.restricted_headers=/proxy/ /lock-token/ /content-range/ /translate/ /if/'\"", + "SecRule REQUEST_HEADERS_NAMES \"@rx ^.*$\" \"id:920450,phase:2,block,capture,t:none,t:lowercase,msg:'HTTP header is restricted by policy (%{MATCHED_VAR})',logdata:' Restricted header detected %{MATCHED_VAR}',ver:'OWASP_CRS/3.1.0',severity:'CRITICAL',setvar:'tx.header_name_%{tx.0}=/%{tx.0}/',chain", + "SecRule TX:/^HEADER_NAME_/ \"@within %{tx.restricted_headers}\" \"setvar:'tx.msg=%{rule.msg}',setvar:'tx.anomaly_score_pl1=+%{tx.critical_anomaly_score}',setvar:'tx.%{rule.id}-OWASP_CRS/POLICY/HEADERS_RESTRICTED-%{MATCHED_VAR_NAME}=%{MATCHED_VAR}'" + ] + } +] + From ea937cf572dcf9ef5d964fa87dd6aca831d24d77 Mon Sep 17 00:00:00 2001 From: Ervin Hegedus Date: Thu, 7 Feb 2019 13:49:04 +0000 Subject: [PATCH 4/5] Increment OVECCOUNT value for bigger regex's --- src/utils/regex.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/regex.h b/src/utils/regex.h index 08205a9726..296a71499c 100644 --- a/src/utils/regex.h +++ b/src/utils/regex.h @@ -27,7 +27,7 @@ namespace modsecurity { namespace Utils { -#define OVECCOUNT 50 +#define OVECCOUNT 900 class SMatch { public: From 80409045125387b5a5e1f405855dbf03d245d30b Mon Sep 17 00:00:00 2001 From: Ervin Hegedus Date: Tue, 19 Feb 2019 20:47:52 +0000 Subject: [PATCH 5/5] Removed pcre dependency from outside of regex class --- .../backend/in_memory-per_process.cc | 2 +- src/collection/backend/lmdb.cc | 2 +- src/utils/regex.cc | 19 +++++-------------- src/utils/regex.h | 3 +-- 4 files changed, 8 insertions(+), 18 deletions(-) diff --git a/src/collection/backend/in_memory-per_process.cc b/src/collection/backend/in_memory-per_process.cc index ca9107619f..14e7e4744c 100644 --- a/src/collection/backend/in_memory-per_process.cc +++ b/src/collection/backend/in_memory-per_process.cc @@ -134,7 +134,7 @@ void InMemoryPerProcess::resolveRegularExpression(const std::string& var, //std::string name = std::string(var, var.find(":") + 2, // var.size() - var.find(":") - 3); //size_t keySize = col.size(); - Utils::Regex r(var, PCRE_CASELESS); + Utils::Regex r(var, true); for (const auto& x : *this) { //if (x.first.size() <= keySize + 1) { diff --git a/src/collection/backend/lmdb.cc b/src/collection/backend/lmdb.cc index d789967499..189c94b9de 100644 --- a/src/collection/backend/lmdb.cc +++ b/src/collection/backend/lmdb.cc @@ -538,7 +538,7 @@ void LMDB::resolveRegularExpression(const std::string& var, MDB_cursor *cursor; size_t pos; - Utils::Regex r(var, PCRE_CASELESS); + Utils::Regex r(var, true); rc = mdb_txn_begin(m_env, NULL, 0, &txn); lmdb_debug(rc, "txn", "resolveRegularExpression"); diff --git a/src/utils/regex.cc b/src/utils/regex.cc index 128284b612..e6b45c9d18 100644 --- a/src/utils/regex.cc +++ b/src/utils/regex.cc @@ -38,24 +38,15 @@ namespace modsecurity { namespace Utils { -Regex::Regex(const std::string& pattern_) +Regex::Regex(const std::string& pattern_, bool caseSensitive) : pattern(pattern_.empty() ? ".*" : pattern_) { const char *errptr = NULL; int erroffset; + int flags = (PCRE_DOTALL|PCRE_MULTILINE); - m_pc = pcre_compile(pattern.c_str(), PCRE_DOTALL|PCRE_MULTILINE, - &errptr, &erroffset, NULL); - - m_pce = pcre_study(m_pc, pcre_study_opt, &errptr); -} - -Regex::Regex(const std::string& pattern_, int flags) - : pattern(pattern_.empty() ? ".*" : pattern_){ - const char *errptr = NULL; - int erroffset; - - flags |= (PCRE_DOTALL|PCRE_MULTILINE); - + if (caseSensitive == true) { + flags |= PCRE_CASELESS; + } m_pc = pcre_compile(pattern.c_str(), flags, &errptr, &erroffset, NULL); diff --git a/src/utils/regex.h b/src/utils/regex.h index 296a71499c..d10ba47d1d 100644 --- a/src/utils/regex.h +++ b/src/utils/regex.h @@ -50,8 +50,7 @@ class SMatch { class Regex { public: - explicit Regex(const std::string& pattern_); - explicit Regex(const std::string& pattern_, int flags); + explicit Regex(const std::string& pattern_, bool caseSensitive = false); ~Regex(); // m_pc and m_pce can't be easily copied