Merge pull request #324 from JayBizzle/issue_323

Fix false positives and new add new bot
JayBizzle · Apr 5, 2019 · af6a36e · af6a36e
2 parents 80144ca + 8d1e328
commit af6a36e
Show file tree

Hide file tree

Showing 8 changed files with 11 additions and 4 deletions.
diff --git a/raw/Crawlers.json b/raw/Crawlers.json
diff --git a/raw/Crawlers.txt b/raw/Crawlers.txt
@@ -1018,6 +1018,7 @@ topster
 touche\.com
 Traackr\.com
 tracemyfile
+Trackuity
 TrapitAgent
 Trendiction
 Trendsmap
@@ -1207,7 +1208,7 @@ Zemanta Aggregator
 Zend_Http_Client
 Zend\\Http\\Client
 Zermelo
-Zeus
+Zeus 
 zgrab
 ZnajdzFoto
 Zombie\.js

diff --git a/raw/Exclusions.json b/raw/Exclusions.json
@@ -1 +1 @@
-["Safari.[\\d\\.]*","Firefox.[\\d\\.]*"," Chrome.[\\d\\.]*","Chromium.[\\d\\.]*","MSIE.[\\d\\.]","Opera\\\/[\\d\\.]*","Mozilla.[\\d\\.]*","AppleWebKit.[\\d\\.]*","Trident.[\\d\\.]*","Windows NT.[\\d\\.]*","Android [\\d\\.]*","Macintosh.","Ubuntu","Linux","[ ]Intel","Mac OS X [\\d_]*","(like )?Gecko(.[\\d\\.]*)?","KHTML,","CriOS.[\\d\\.]*","CPU iPhone OS ([0-9_])* like Mac OS X","CPU OS ([0-9_])* like Mac OS X","iPod","compatible","x86_..","i686","x64","X11","rv:[\\d\\.]*","Version.[\\d\\.]*","WOW64","Win64","Dalvik.[\\d\\.]*"," \\.NET CLR [\\d\\.]*","Presto.[\\d\\.]*","Media Center PC","BlackBerry","Build","Opera Mini\\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]*\\\/\\d{1,2}\\.","Opera"," \\.NET[\\d\\.]*","cubot","; M bot","; B bot","; IDbot","; ID bot","; POWER BOT",";"]
+["Safari.[\\d\\.]*","Firefox.[\\d\\.]*"," Chrome.[\\d\\.]*","Chromium.[\\d\\.]*","MSIE.[\\d\\.]","Opera\\\/[\\d\\.]*","Mozilla.[\\d\\.]*","AppleWebKit.[\\d\\.]*","Trident.[\\d\\.]*","Windows NT.[\\d\\.]*","Android [\\d\\.]*","Macintosh.","Ubuntu","Linux","[ ]Intel","Mac OS X [\\d_]*","(like )?Gecko(.[\\d\\.]*)?","KHTML,","CriOS.[\\d\\.]*","CPU iPhone OS ([0-9_])* like Mac OS X","CPU OS ([0-9_])* like Mac OS X","iPod","compatible","x86_..","i686","x64","X11","rv:[\\d\\.]*","Version.[\\d\\.]*","WOW64","Win64","Dalvik.[\\d\\.]*"," \\.NET CLR [\\d\\.]*","Presto.[\\d\\.]*","Media Center PC","BlackBerry","Build","Opera Mini\\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]*\\\/\\d{1,2}\\.","Opera"," \\.NET[\\d\\.]*","cubot","; M bot","; CRONO","; B bot","; IDbot","; ID bot","; POWER BOT",";"]
diff --git a/raw/Exclusions.txt b/raw/Exclusions.txt
@@ -40,6 +40,7 @@ Opera
  \.NET[\d\.]*
 cubot
 ; M bot
+; CRONO
 ; B bot
 ; IDbot
 ; ID bot

diff --git a/src/Fixtures/Crawlers.php b/src/Fixtures/Crawlers.php
@@ -1039,6 +1039,7 @@ class Crawlers extends AbstractProvider
         'touche\.com',
         'Traackr\.com',
         'tracemyfile',
+        'Trackuity',
         'TrapitAgent',
         'Trendiction',
         'Trendsmap',
@@ -1228,7 +1229,7 @@ class Crawlers extends AbstractProvider
         'Zend_Http_Client',
         'Zend\\\\Http\\\\Client',
         'Zermelo',
-        'Zeus',
+        'Zeus ',
         'zgrab',
         'ZnajdzFoto',
         'Zombie\.js',

diff --git a/src/Fixtures/Exclusions.php b/src/Fixtures/Exclusions.php
@@ -62,6 +62,7 @@ class Exclusions extends AbstractProvider
         ' \.NET[\d\.]*',
         'cubot',
         '; M bot',
+        '; CRONO',
         '; B bot',
         '; IDbot',
         '; ID bot',

diff --git a/tests/crawlers.txt b/tests/crawlers.txt
@@ -3438,3 +3438,4 @@ https://www.nominet.uk/privacy-notice
 <a href='http://www.unchaos.com/'> UnChaos </a> From Chaos To Order Hybrid Web Search Engine.([email protected])
 Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36(KHTML, like Gecko) Chrome/69.0.3464.0 Safari/537.36 Chrome-Lighthouse
 Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5 Build/MRA58N) AppleWebKit/537.36(KHTML, like Gecko) Chrome/61.0.3116.0 Mobile Safari/537.36 Chrome-Lighthouse
+Trackuity (+http://trackuity.com)
diff --git a/tests/devices.txt b/tests/devices.txt
@@ -165978,3 +165978,5 @@ Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/604.4.7 (KHTML, like
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36
 Mozilla/5.0 (Linux; Android 8.1.0; Pixel Build/OPM4.171019.016.B1; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.158 Mobile Safari/537.36 [FB_IAB/FB4A;FBAV/171.0.0.49.92;]
 Mozilla/5.0 (iPhone; CPU iPhone OS 11_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E302 [FBAN/FBIOS;FBAV/171.0.0.49.95;FBBV/107251038;FBDV/iPhone7,2;FBMD/iPhone;FBSN/iOS;FBSV/11.3.1;FBSS/2;FBCR/EE;FBID/phone;FBLC/en_GB;FBOP/5;FBRV/108659124]
+Mozilla/5.0 (Linux; Android 5.1; CRONO 54 Build/LMY47I) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.111 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 4.2.1; MAJESTIC Zeus21 Build/JOP40D) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.72 Mobile Safari/537.36
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		["Safari.[\\d\\.]","Firefox.[\\d\\.]"," Chrome.[\\d\\.]","Chromium.[\\d\\.]","MSIE.[\\d\\.]","Opera\\\/[\\d\\.]","Mozilla.[\\d\\.]","AppleWebKit.[\\d\\.]","Trident.[\\d\\.]","Windows NT.[\\d\\.]","Android [\\d\\.]","Macintosh.","Ubuntu","Linux","[ ]Intel","Mac OS X [\\d_]","(like )?Gecko(.[\\d\\.])?","KHTML,","CriOS.[\\d\\.]","CPU iPhone OS ([0-9_]) like Mac OS X","CPU OS ([0-9_])* like Mac OS X","iPod","compatible","x86_..","i686","x64","X11","rv:[\\d\\.]","Version.[\\d\\.]","WOW64","Win64","Dalvik.[\\d\\.]"," \\.NET CLR [\\d\\.]","Presto.[\\d\\.]","Media Center PC","BlackBerry","Build","Opera Mini\\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]\\\/\\d{1,2}\\.","Opera"," \\.NET[\\d\\.]*","cubot","; M bot","; B bot","; IDbot","; ID bot","; POWER BOT",";"]
		["Safari.[\\d\\.]","Firefox.[\\d\\.]"," Chrome.[\\d\\.]","Chromium.[\\d\\.]","MSIE.[\\d\\.]","Opera\\\/[\\d\\.]","Mozilla.[\\d\\.]","AppleWebKit.[\\d\\.]","Trident.[\\d\\.]","Windows NT.[\\d\\.]","Android [\\d\\.]","Macintosh.","Ubuntu","Linux","[ ]Intel","Mac OS X [\\d_]","(like )?Gecko(.[\\d\\.])?","KHTML,","CriOS.[\\d\\.]","CPU iPhone OS ([0-9_]) like Mac OS X","CPU OS ([0-9_])* like Mac OS X","iPod","compatible","x86_..","i686","x64","X11","rv:[\\d\\.]","Version.[\\d\\.]","WOW64","Win64","Dalvik.[\\d\\.]"," \\.NET CLR [\\d\\.]","Presto.[\\d\\.]","Media Center PC","BlackBerry","Build","Opera Mini\\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]\\\/\\d{1,2}\\.","Opera"," \\.NET[\\d\\.]*","cubot","; M bot","; CRONO","; B bot","; IDbot","; ID bot","; POWER BOT",";"]
-Original file line number
+Diff line change
@@ Expand Up / @@ -40,6 +40,7 @@ Opera @@
      \.NET[\d\.]*
     cubot
     ; M bot
+    ; CRONO
     ; B bot
     ; IDbot
     ; ID bot
@@ Expand Down @@