From 04a41ef8647072a14535e6522032bb13f843c302 Mon Sep 17 00:00:00 2001 From: WinfredLIN Date: Thu, 6 Jun 2024 13:25:47 +0800 Subject: [PATCH 1/3] fix: regular expressions not matching multiple lines --- sqle/utils/util.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/sqle/utils/util.go b/sqle/utils/util.go index bb75f5f99a..c6361d604c 100644 --- a/sqle/utils/util.go +++ b/sqle/utils/util.go @@ -317,9 +317,20 @@ func IsPrefixSubStrArray(arr []string, prefix []string) bool { return true } -// 全模糊匹配字符串，并且对大小写不敏感 +// 全模糊匹配字符串，对大小写不敏感，匹配多行，且防止正则注入 func FullFuzzySearchRegexp(str string) *regexp.Regexp { - return regexp.MustCompile(`^.*(?i)` + regexp.QuoteMeta(str) + `.*$`) + /* + 1. (?is)是一个正则表达式修饰符,其中： + i表示忽略大小写(case-insensitive) + s表示让.匹配任何字符,包括换行符(\n) + 2. ^.*匹配字符串的开头,其中： + ^表示起始位置, + .表示匹配任何字符(除了换行符) + *表示匹配前面的模式零次或多次 + 3. .*$匹配字符串的结尾,其中： + $表示结束位置 + */ + return regexp.MustCompile(`(?is)^.*` + regexp.QuoteMeta(str) + `.*$`) } var ErrUnknownEncoding = errors.New("unknown encoding") From e92754dab3bf7e1e85356a6d6d83fb30934d50eb Mon Sep 17 00:00:00 2001 From: WinfredLIN Date: Thu, 6 Jun 2024 13:26:42 +0800 Subject: [PATCH 2/3] test: unit test for regular expressions and black list --- sqle/api/controller/v1/audit_plan_test.go | 8 ++++++++ sqle/utils/util_test.go | 7 +++++++ 2 files changed, 15 insertions(+) diff --git a/sqle/api/controller/v1/audit_plan_test.go b/sqle/api/controller/v1/audit_plan_test.go index 280bfc9d63..f6f13ca493 100644 --- a/sqle/api/controller/v1/audit_plan_test.go +++ b/sqle/api/controller/v1/audit_plan_test.go @@ -15,6 +15,9 @@ func TestIsSqlInBlackList(t *testing.T) { }, { FilterContent: "table_1", FilterType: "SQL", + },{ + FilterContent: "ignored_service", + FilterType: "SQL", }, }) @@ -22,6 +25,9 @@ func TestIsSqlInBlackList(t *testing.T) { "SELECT * FROM users", "DELETE From tAble_1", "SELECT COUNT(*) FROM table_2", + `/* this is a comment, Service: ignored_service */ + select * from table_ignored where id < 123;`, + `/* this is a comment, Service: ignored_service */ update * from table_ignored where id < 123;`, } for _, matchSql := range matchSqls { if !filter.IsSqlInBlackList(matchSql) { @@ -32,6 +38,8 @@ func TestIsSqlInBlackList(t *testing.T) { "INSERT INTO users VALUES (1, 'John')", "DELETE From schools", "SHOW CREATE TABLE table_2", + `/* this is a comment, Service: ignored_ + service */ update * from table_ignored where id < 123;`, } for _, notMatchSql := range notMatchSqls { if filter.IsSqlInBlackList(notMatchSql) { diff --git a/sqle/utils/util_test.go b/sqle/utils/util_test.go index 4fb40319e7..a44d03d10b 100644 --- a/sqle/utils/util_test.go +++ b/sqle/utils/util_test.go @@ -259,6 +259,13 @@ func TestFullFuzzySearchRegexp(t *testing.T) { ".*(?i)", []string{"GoLang .*(?i) awesome", "I love GO^.*(?i)SING", "GoLangGO.*(?i)Golang"}, []string{"language", "hi", "heyHelloCode", "HElLO", "Sun_hello", "HelLo_Jack"}, + },{ + "ignored_service", + []string{`/* this is a comment, Service: ignored_service */ + select * from table_ignored where id < 123;' + `,`/* this is a comment, Service: ignored_service */ select * from table_ignored where id < 123;`}, + []string{"any sql","",`/* this is a comment, Service: ignored + _service */ select * from table_ignored where id < 123;`}, }, } From 798c09bbebf143545f3ee43a748e634b12035271 Mon Sep 17 00:00:00 2001 From: WinfredLIN Date: Thu, 6 Jun 2024 13:50:30 +0800 Subject: [PATCH 3/3] docs: modify comment --- sqle/utils/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqle/utils/util.go b/sqle/utils/util.go index c6361d604c..5a6da8bd5c 100644 --- a/sqle/utils/util.go +++ b/sqle/utils/util.go @@ -322,7 +322,7 @@ func FullFuzzySearchRegexp(str string) *regexp.Regexp { /* 1. (?is)是一个正则表达式修饰符,其中： i表示忽略大小写(case-insensitive) - s表示让.匹配任何字符,包括换行符(\n) + s表示开启单行模式，开启后.可以匹配换行符，让整个字符串作为一行 2. ^.*匹配字符串的开头,其中： ^表示起始位置, .表示匹配任何字符(除了换行符)