diff --git a/mapping.yaml b/mapping.yaml index ab6e837..7c6f19b 100644 --- a/mapping.yaml +++ b/mapping.yaml @@ -35,3 +35,14 @@ mappings: query: SELECT url, COUNT(*) as count, SUM(size) as sum, AVG(size), MAX(response) FROM log WHERE url LIKE '/surveys/%' AND response > 200 GROUP BY url ORDER BY count LIMIT 5; +- name: internalLog + tables: + - 'CREATE TABLE nonAuth(reason VARCHAR, uri VARCHAR, referer VARCHAR, clientIp VARCHAR, + userAgent VARCHAR) WITH FIELDS IDENTIFIED BY ''^{\\Dreason\\D: \\D(?P[^\\\'']*)\\D, + \\Duri\\D: \\D(?P[^\\\'']*)\\D, \\Dreferer\\D: \\D(?P[^\\\'']*)\\D, + \\DclientIp\\D: \\D(?P[^\\\'']*)\\D, \\Duser_agent\\D: \\D(?P[^\\\'']*)\\D}$'' + LINES TERMINATED BY ''\n'';' + queries: + - name: basic + query: SELECT reason, URLIFY(uri) as url, COUNT(*) as count, URLIFY(referer), clientIp, userAgent from + nonAuth GROUP BY url, reason, clientIp ORDER BY count ASC; diff --git a/tablestream/sql_test.go b/tablestream/sql_test.go index cd6c64e..c5e2aea 100644 --- a/tablestream/sql_test.go +++ b/tablestream/sql_test.go @@ -77,6 +77,10 @@ func (s *Suite) TestPrepareCreate(c *check.C) { c.Assert(err, check.IsNil) field := table.Field("col2") c.Assert(field.fieldType, check.Equals, DATETIME) + + err = stream.Query(`CREATE TABLE nonAuth(reason VARCHAR, uri VARCHAR, referer VARCHAR, clientIp VARCHAR, userAgent VARCHAR) WITH FIELDS IDENTIFIED BY '^{\Dreason\D: \D(?P[^\\\']*)\D, \Duri\D: \D(?P[^\\\']*)\D, \Dreferer\D: \D(?P[^\\\']*)\D, \DclientIp\D: \D(?P[^\\\']*)\D, \Duser_agent\D: \D(?P[^\\\']*)\D}$' LINES TERMINATED BY '\n';`) + c.Assert(err, check.IsNil) + } // ORDER BY column1, column2, ... ASC|DESC; diff --git a/tablestream/tableregex.go b/tablestream/tableregex.go index e2f6ff0..fb37f4d 100644 --- a/tablestream/tableregex.go +++ b/tablestream/tableregex.go @@ -52,6 +52,9 @@ func regexMapping(stmt *sqlparser.DDL) (t Table, fields int, err error) { } regexMapTrimmed := strings.TrimPrefix(strings.TrimPrefix(regexMap[1], "'"), "\"") regexMapTrimmed = strings.TrimSuffix(strings.TrimSuffix(regexMapTrimmed, "'"), "\"") + + regexMapTrimmed = strings.Replace(regexMapTrimmed, "\\'", "'", -1) + table.fieldRegexMap, err = regexp.Compile(regexMapTrimmed) if err != nil { return t, fields, fmt.Errorf("regex present on FIELDS IDENTIFIED by failed to compile: %s", err.Error()) diff --git a/tablestream/viewdata_aggregated_test.go b/tablestream/viewdata_aggregated_test.go index 2ab08f7..4f9cf3b 100644 --- a/tablestream/viewdata_aggregated_test.go +++ b/tablestream/viewdata_aggregated_test.go @@ -328,3 +328,41 @@ func (s *Suite) TestGroupBYWindowsExecution(c *check.C) { c.Assert(allRows[2][5], check.Equals, "200") } + +func (s *Suite) TestSingleQuotesRegex(c *check.C) { + query := `CREATE TABLE nonAuth(reason VARCHAR, uri VARCHAR, referer VARCHAR, clientIp VARCHAR, userAgent VARCHAR) WITH FIELDS IDENTIFIED BY '^{\\Dreason\\D: \\D(?P[^\\\']*)\\D, \\Duri\\D: \\D(?P[^\\\']*)\\D, \\Dreferer\\D: \\D(?P[^\\\']*)\\D, \\DclientIp\\D: \\D(?P[^\\\']*)\\D, \\Duser_agent\\D: \\D(?P[^\\\']*)\\D}$' LINES TERMINATED BY '\n';` + + stream := &Stream{} + + err := stream.Query(query) + c.Assert(err, check.IsNil) + + query = `SELECT reason, uri, COUNT(*) as count, referer, clientIp, userAgent from + nonAuth GROUP BY uri, reason ORDER BY count ASC;` + + err = stream.Query(query) + c.Assert(err, check.IsNil) + + table, err := stream.Table("nonAuth") + c.Assert(err, check.IsNil) + + err = table.AddRow(`{'reason': 'No cookie', 'uri': '/config', 'referer': 'https://example.com', 'clientIp': '33.44.44.6', 'user_agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0.2 Safari/605.1.15'}`) + c.Assert(err, check.IsNil) + err = table.AddRow(`{'reason': 'Invalid SSO cookie signature', 'uri': '/chair', 'referer': 'https://example.com', 'clientIp': '33.44.44.6', 'user_agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36'}`) + c.Assert(err, check.IsNil) + err = table.AddRow(`{'reason': 'Invalid SSO cookie signature', 'uri': '/bleom/sdsds', 'referer': 'https://example.com', 'clientIp': '33.44.44.6', 'user_agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko'}`) + c.Assert(err, check.IsNil) + + err = table.AddRow(`{'reason': 'Invalid SSO cookie signature', 'uri': '/config', 'referer': 'https://example.com', 'clientIp': '33.44.44.4', 'user_agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko'}`) + c.Assert(err, check.IsNil) + err = table.AddRow(`{'reason': 'Invalid SSO cookie signature', 'uri': '/bleom/sdsds', 'referer': 'https://example.com', 'clientIp': '33.44.44.4', 'user_agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko'}`) + c.Assert(err, check.IsNil) + + // Time to flush the channels + time.Sleep(1000 * time.Millisecond) + + allRows := stream.views[0].FetchAllRows() + + c.Assert(len(allRows), check.Equals, 5) + +}