From 3b59555b204829c2e888e60083c9a07614b4b1bc Mon Sep 17 00:00:00 2001 From: Mark Beech Date: Wed, 22 Apr 2015 20:30:29 +0100 Subject: [PATCH 1/5] Added BUbiNG bot --- README.md | 3 +++ src/CrawlerDetect.php | 1 + tests/crawlers.txt | 3 ++- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 94ec081..f1a0e2e 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,9 @@ Failing that, just create an issue with the user agent you have found, and we'll If you would like to use this with Laravel 5, please see [Laravel-Crawler-Detect](https://github.com/JayBizzle/Laravel-Crawler-Detect) ### Changelog +**v1.0.3** + - Added ['BUbiNG'](http://law.di.unimi.it/BUbiNG.html) + **v1.0.2** - Added ['AbiLogicBot'](http://www.abilogic.com/bot.html) - Added ['Link Valet'](http://www.htmlhelp.com/tools/valet/) diff --git a/src/CrawlerDetect.php b/src/CrawlerDetect.php index 553f9ab..3f6e6a6 100644 --- a/src/CrawlerDetect.php +++ b/src/CrawlerDetect.php @@ -48,6 +48,7 @@ class CrawlerDetect { "boitho\.com-robot", "brainobot", "btbot", + "BUbiNG", "buzzbot", "careerbot", "CatchBot", diff --git a/tests/crawlers.txt b/tests/crawlers.txt index 6772ca7..bc76dba 100644 --- a/tests/crawlers.txt +++ b/tests/crawlers.txt @@ -539,4 +539,5 @@ Xenu Link Sleuth 1.2e Xenu Link Sleuth 1.2d Xenu Link Sleuth 1.2c Xenu Link Sleuth 1.2b -Xenu Link Sleuth/1.3.7 \ No newline at end of file +Xenu Link Sleuth/1.3.7 +BUbiNG (+http://law.di.unimi.it/BUbiNG.html) \ No newline at end of file From fafbdbe5debe299195e83ee08c7b2c07b8c61c8c Mon Sep 17 00:00:00 2001 From: Mark Beech Date: Wed, 22 Apr 2015 20:32:32 +0100 Subject: [PATCH 2/5] Added Qwantify bot --- README.md | 1 + src/CrawlerDetect.php | 1 + tests/crawlers.txt | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f1a0e2e..b064e96 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ If you would like to use this with Laravel 5, please see [Laravel-Crawler-Detect ### Changelog **v1.0.3** - Added ['BUbiNG'](http://law.di.unimi.it/BUbiNG.html) + - Added ['Qwantify'](https://www.qwant.com/) **v1.0.2** - Added ['AbiLogicBot'](http://www.abilogic.com/bot.html) diff --git a/src/CrawlerDetect.php b/src/CrawlerDetect.php index 3f6e6a6..3872faf 100644 --- a/src/CrawlerDetect.php +++ b/src/CrawlerDetect.php @@ -227,6 +227,7 @@ class CrawlerDetect { "Python-urllib", "Qseero", "QuerySeekerSpider", + "Qwantify", "Radian6", "RAMPyBot", "REL Link Checker", diff --git a/tests/crawlers.txt b/tests/crawlers.txt index bc76dba..6d9cd59 100644 --- a/tests/crawlers.txt +++ b/tests/crawlers.txt @@ -540,4 +540,5 @@ Xenu Link Sleuth 1.2d Xenu Link Sleuth 1.2c Xenu Link Sleuth 1.2b Xenu Link Sleuth/1.3.7 -BUbiNG (+http://law.di.unimi.it/BUbiNG.html) \ No newline at end of file +BUbiNG (+http://law.di.unimi.it/BUbiNG.html) +Mozilla/5.0 (compatible; Qwantify/2.0n; +https://www.qwant.com/)/* \ No newline at end of file From a413edbba9030f9af1319b95426798948170f238 Mon Sep 17 00:00:00 2001 From: Mark Beech Date: Wed, 22 Apr 2015 20:34:11 +0100 Subject: [PATCH 3/5] Added archive.org_bot --- README.md | 1 + src/CrawlerDetect.php | 1 + tests/crawlers.txt | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b064e96..b5cf157 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ If you would like to use this with Laravel 5, please see [Laravel-Crawler-Detect **v1.0.3** - Added ['BUbiNG'](http://law.di.unimi.it/BUbiNG.html) - Added ['Qwantify'](https://www.qwant.com/) + - Added ['archive.org_bot'](http://www.archive.org/details/archive.org_bot) **v1.0.2** - Added ['AbiLogicBot'](http://www.abilogic.com/bot.html) diff --git a/src/CrawlerDetect.php b/src/CrawlerDetect.php index 3872faf..3a12167 100644 --- a/src/CrawlerDetect.php +++ b/src/CrawlerDetect.php @@ -29,6 +29,7 @@ class CrawlerDetect { "arabot", "Arachmo", "archive-com", + "archive.org_bot", "B-l-i-t-z-B-O-T", "backlinkcrawler", "baiduspider", diff --git a/tests/crawlers.txt b/tests/crawlers.txt index 6d9cd59..7a3bb13 100644 --- a/tests/crawlers.txt +++ b/tests/crawlers.txt @@ -541,4 +541,5 @@ Xenu Link Sleuth 1.2c Xenu Link Sleuth 1.2b Xenu Link Sleuth/1.3.7 BUbiNG (+http://law.di.unimi.it/BUbiNG.html) -Mozilla/5.0 (compatible; Qwantify/2.0n; +https://www.qwant.com/)/* \ No newline at end of file +Mozilla/5.0 (compatible; Qwantify/2.0n; +https://www.qwant.com/)/* +Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot) \ No newline at end of file From 54285c43c4b6bf3ff9b657569802a60c21e1ec65 Mon Sep 17 00:00:00 2001 From: Mark Beech Date: Wed, 22 Apr 2015 20:37:02 +0100 Subject: [PATCH 4/5] Added Applebot --- README.md | 1 + src/CrawlerDetect.php | 1 + tests/crawlers.txt | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b5cf157..25f869a 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ If you would like to use this with Laravel 5, please see [Laravel-Crawler-Detect - Added ['BUbiNG'](http://law.di.unimi.it/BUbiNG.html) - Added ['Qwantify'](https://www.qwant.com/) - Added ['archive.org_bot'](http://www.archive.org/details/archive.org_bot) + - Added ['Applebot'](http://www.apple.com/go/applebot) **v1.0.2** - Added ['AbiLogicBot'](http://www.abilogic.com/bot.html) diff --git a/src/CrawlerDetect.php b/src/CrawlerDetect.php index 3a12167..8dfba77 100644 --- a/src/CrawlerDetect.php +++ b/src/CrawlerDetect.php @@ -26,6 +26,7 @@ class CrawlerDetect { "AISearchBot", "antibot", "AnyApexBot", + "Applebot", "arabot", "Arachmo", "archive-com", diff --git a/tests/crawlers.txt b/tests/crawlers.txt index 7a3bb13..e45c015 100644 --- a/tests/crawlers.txt +++ b/tests/crawlers.txt @@ -542,4 +542,5 @@ Xenu Link Sleuth 1.2b Xenu Link Sleuth/1.3.7 BUbiNG (+http://law.di.unimi.it/BUbiNG.html) Mozilla/5.0 (compatible; Qwantify/2.0n; +https://www.qwant.com/)/* -Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot) \ No newline at end of file +Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot) +Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1) \ No newline at end of file From c8e40d49afbe2b41fa9b42015ba9d078350ad114 Mon Sep 17 00:00:00 2001 From: Mark Beech Date: Wed, 22 Apr 2015 20:38:41 +0100 Subject: [PATCH 5/5] Added TweetmemeBot --- README.md | 1 + src/CrawlerDetect.php | 1 + tests/crawlers.txt | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 25f869a..48af668 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ If you would like to use this with Laravel 5, please see [Laravel-Crawler-Detect - Added ['Qwantify'](https://www.qwant.com/) - Added ['archive.org_bot'](http://www.archive.org/details/archive.org_bot) - Added ['Applebot'](http://www.apple.com/go/applebot) + - Added ['TweetmemeBot'](http://datasift.com/bot.html) **v1.0.2** - Added ['AbiLogicBot'](http://www.abilogic.com/bot.html) diff --git a/src/CrawlerDetect.php b/src/CrawlerDetect.php index 8dfba77..8afabcf 100644 --- a/src/CrawlerDetect.php +++ b/src/CrawlerDetect.php @@ -284,6 +284,7 @@ class CrawlerDetect { "truwoGPS", "turnitinbot", "TweetedTimes Bot", + "TweetmemeBot", "twengabot", "Twitterbot", "uMBot", diff --git a/tests/crawlers.txt b/tests/crawlers.txt index e45c015..50eee45 100644 --- a/tests/crawlers.txt +++ b/tests/crawlers.txt @@ -543,4 +543,5 @@ Xenu Link Sleuth/1.3.7 BUbiNG (+http://law.di.unimi.it/BUbiNG.html) Mozilla/5.0 (compatible; Qwantify/2.0n; +https://www.qwant.com/)/* Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot) -Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1) \ No newline at end of file +Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1) +Mozilla/5.0 (TweetmemeBot/4.0; +http://datasift.com/bot.html) Gecko/20100101 Firefox/31.0 \ No newline at end of file