Merge pull request #21 from JayBizzle/develop

Added a generic bot detector regex
JayBizzle · Sep 9, 2015 · fdc19a1 · fdc19a1
2 parents 4ac4e61 + 97c947d
commit fdc19a1
Show file tree

Hide file tree

Showing 3 changed files with 2 additions and 1 deletion.
diff --git a/src/CrawlerDetect.php b/src/CrawlerDetect.php
@@ -407,6 +407,7 @@ class CrawlerDetect
         'Zealbot',
         'zspider',
         'ZyBorg',
+        '[a-z0-9\\-_]*((?<!cu)bot|crawler|archiver|transcoder|spider)',
     );
 
     /**

diff --git a/tests/crawlers.txt b/tests/crawlers.txt
@@ -720,3 +720,4 @@ Scrapy/0.21.0-160-g3c1e226 (+http://scrapy.org)
 UnwindFetchor/1.0 (+http://www.gnip.com/)
 urlresolver
 Mozilla/5.0 (compatible; XML Sitemaps Generator; https://www.xml-sitemaps.com) Gecko XML-Sitemaps/1.0
+Mozilla/5.0 (en-us) AppleWebKit/534.14 (KHTML, like Gecko; Google Wireless Transcoder) Chrome/9.0.597 Safari/534.14
diff --git a/tests/devices.txt b/tests/devices.txt
@@ -3971,7 +3971,6 @@ Mozilla/5.0 (compatible; StaffsStudentImage; Windows NT 6.3; WOW64) AppleWebKit/
 Mozilla/5.0 (compatible; StaffsStudentImage; Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko
 Mozilla/5.0 (compatible; StaffsStudentImage; Windows NT 6.3; WOW64; Trident/7.0; StaffsStudentImage; rv:11.0) like Gecko
 Mozilla/5.0 (compatible;FW 2.0. 5170.p; MSIE 10.0; Windows NT 6.2; WOW64; Trident/6.0)
-Mozilla/5.0 (en-us) AppleWebKit/534.14 (KHTML, like Gecko; Google Wireless Transcoder) Chrome/9.0.597 Safari/534.14
 Mozilla/5.0 (en-us;)
 Mozilla/5.0 (hp-tablet; Linux; hpwOS/3.0.5; U; en-GB) AppleWebKit/534.6 (KHTML, like Gecko) wOSBrowser/234.83 Safari/534.6 TouchPad/1.0
 Mozilla/5.0 (hp-tablet; Linux; hpwOS/3.0.5; U; en-US) AppleWebKit/534.6 (KHTML, like Gecko) wOSBrowser/234.83 Safari/534.6 TouchPad/1.0