diff --git a/raw/Exclusions.json b/raw/Exclusions.json index e7e0141..bf4128b 100644 --- a/raw/Exclusions.json +++ b/raw/Exclusions.json @@ -1 +1 @@ -["Safari.[\\d\\.]*","Firefox.[\\d\\.]*"," Chrome.[\\d\\.]*","Chromium.[\\d\\.]*","MSIE.[\\d\\.]","Opera\\\/[\\d\\.]*","Mozilla.[\\d\\.]*","AppleWebKit.[\\d\\.]*","Trident.[\\d\\.]*","Windows NT.[\\d\\.]*","Android [\\d\\.]*","Macintosh.","Ubuntu","Linux","[ ]Intel","Mac OS X [\\d_]*","(like )?Gecko(.[\\d\\.]*)?","KHTML,","CriOS.[\\d\\.]*","CPU iPhone OS ([0-9_])* like Mac OS X","CPU OS ([0-9_])* like Mac OS X","iPod","compatible","x86_..","i686","x64","X11","rv:[\\d\\.]*","Version.[\\d\\.]*","WOW64","Win64","Dalvik.[\\d\\.]*"," \\.NET CLR [\\d\\.]*","Presto.[\\d\\.]*","Media Center PC","BlackBerry","Build","Opera Mini\\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]*\\\/\\d{1,2}\\.","Opera"," \\.NET[\\d\\.]*","cubot","; M bot","; CRONO","; B bot","; IDbot","; ID bot","; POWER BOT","OCTOPUS-CORE","htc_botdugls","super\\\/\\d+\\\/Android\\\/\\d+"] \ No newline at end of file +["Safari.[\\d\\.]*","Firefox.[\\d\\.]*"," Chrome.[\\d\\.]*","Chromium.[\\d\\.]*","MSIE.[\\d\\.]","Opera\\\/[\\d\\.]*","Mozilla.[\\d\\.]*","AppleWebKit.[\\d\\.]*","Trident.[\\d\\.]*","Windows NT.[\\d\\.]*","Android [\\d\\.]*","Macintosh.","Ubuntu","Linux","[ ]Intel","Mac OS X [\\d_]*","(like )?Gecko(.[\\d\\.]*)?","KHTML,","CriOS.[\\d\\.]*","CPU iPhone OS ([0-9_])* like Mac OS X","CPU OS ([0-9_])* like Mac OS X","iPod","compatible","x86_..","i686","x64","X11","rv:[\\d\\.]*","Version.[\\d\\.]*","WOW64","Win64","Dalvik.[\\d\\.]*"," \\.NET CLR [\\d\\.]*","Presto.[\\d\\.]*","Media Center PC","BlackBerry","Build","Opera Mini\\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]*\\\/\\d{1,2}\\.","Opera"," \\.NET[\\d\\.]*","cubot","; M bot","; CRONO","; B bot","; IDbot","; ID bot","; POWER BOT","OCTOPUS-CORE","htc_botdugls","super\\\/\\d+\\\/Android\\\/\\d+","\"Yandex\"","YandexModule2"] \ No newline at end of file diff --git a/raw/Exclusions.txt b/raw/Exclusions.txt index a44a99c..52026e0 100644 --- a/raw/Exclusions.txt +++ b/raw/Exclusions.txt @@ -47,4 +47,6 @@ cubot ; POWER BOT OCTOPUS-CORE htc_botdugls -super\/\d+\/Android\/\d+ \ No newline at end of file +super\/\d+\/Android\/\d+ +"Yandex" +YandexModule2 \ No newline at end of file diff --git a/src/Fixtures/Exclusions.php b/src/Fixtures/Exclusions.php index 6274557..bfc3ede 100644 --- a/src/Fixtures/Exclusions.php +++ b/src/Fixtures/Exclusions.php @@ -70,5 +70,7 @@ class Exclusions extends AbstractProvider 'OCTOPUS-CORE', 'htc_botdugls', 'super\/\d+\/Android\/\d+', + '"Yandex"', + 'YandexModule2', ); } diff --git a/tests/UserAgentTest.php b/tests/UserAgentTest.php index 21bf029..8b331e8 100644 --- a/tests/UserAgentTest.php +++ b/tests/UserAgentTest.php @@ -21,7 +21,7 @@ final class UserAgentTest extends TestCase public function user_agents_are_bots() { $this->CrawlerDetect = new CrawlerDetect(); - $lines = file(__DIR__.'/crawlers.txt', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES); + $lines = file(__DIR__ . '/data/user_agent/crawlers.txt', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES); foreach ($lines as $line) { $test = $this->CrawlerDetect->isCrawler($line); @@ -33,7 +33,31 @@ public function user_agents_are_bots() public function user_agents_are_devices() { $this->CrawlerDetect = new CrawlerDetect(); - $lines = file(__DIR__.'/devices.txt', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES); + $lines = file(__DIR__ . '/data/user_agent/devices.txt', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES); + + foreach ($lines as $line) { + $test = $this->CrawlerDetect->isCrawler($line); + $this->assertFalse($test, $line); + } + } + + /** @test */ + public function sec_ch_ua_are_bots() + { + $this->CrawlerDetect = new CrawlerDetect(); + $lines = file(__DIR__ . '/data/sec_ch_ua/crawlers.txt', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES); + + foreach ($lines as $line) { + $test = $this->CrawlerDetect->isCrawler($line); + $this->assertTrue($test, $line); + } + } + + /** @test */ + public function sec_ch_ua_are_devices() + { + $this->CrawlerDetect = new CrawlerDetect(); + $lines = file(__DIR__ . '/data/sec_ch_ua/devices.txt', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES); foreach ($lines as $line) { $test = $this->CrawlerDetect->isCrawler($line); diff --git a/tests/data/sec_ch_ua/crawlers.txt b/tests/data/sec_ch_ua/crawlers.txt new file mode 100644 index 0000000..b49d152 --- /dev/null +++ b/tests/data/sec_ch_ua/crawlers.txt @@ -0,0 +1,3 @@ +"HeadlessChrome";v="129", "Not=A?Brand";v="8", "Chromium";v="129" +"HeadlessChrome";v="113", "Chromium";v="113", "Not-A.Brand";v="24" +"Chromium";v="128", "Not;A=Brand";v="24", "HeadlessChrome";v="128" \ No newline at end of file diff --git a/tests/data/sec_ch_ua/devices.txt b/tests/data/sec_ch_ua/devices.txt new file mode 100644 index 0000000..2a36854 --- /dev/null +++ b/tests/data/sec_ch_ua/devices.txt @@ -0,0 +1,7 @@ +"Google Chrome";v="117", "Not;A=Brand";v="8", "Chromium";v="117" +"(Not(A:Brand";v="8", "Chromium";v="98" +" Not A;Brand";v="99", "Chromium";v="102", "Google Chrome";v="102" +" Not A;Brand";v="99", "Chromium";v="96", "Microsoft Edge";v="96" +"Opera";v="81", " Not;A Brand";v="99", "Chromium";v="95" +"Not/A)Brand";v="8", "Chromium";v="126", "YaBrowser";v="24.7", "Yowser";v="2.5" +"Chromium";v="124", "Yandex";v="24", "Not-A.Brand";v="99" \ No newline at end of file diff --git a/tests/crawlers.txt b/tests/data/user_agent/crawlers.txt similarity index 99% rename from tests/crawlers.txt rename to tests/data/user_agent/crawlers.txt index 92e3c06..72a77f8 100644 --- a/tests/crawlers.txt +++ b/tests/data/user_agent/crawlers.txt @@ -1116,7 +1116,6 @@ ShowyouBot (http://showyou.com/crawler) Yoleo Consumer v0.2 Mozilla/5.0 (compatible; IstellaBot/1.10.2 +http://www.tiscali.it/) Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server; +http://ws.daum.net/aboutWebSearch.html) Daumoa/3.0 -Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 YaBrowser/15.12.0.6151 Safari/537.36 Yandex.Translate Mozilla/5.0 (compatible; TeeRaidBot; +https://teeraid.com/bot/) Mozilla/5.0 (compatible; Scopia Crawler 1.1; +http://www.scopia.co) CopperEgg/RevealUptime/DublinIE(aws) @@ -3674,4 +3673,6 @@ facebookcatalog/1.0 Verity/1.1 (gumgum.com/verity; verity-support@gumgum.com) Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GoogleOther) Chrome/117.0.5938.132 Safari/537.36 Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.5938.132 Mobile Safari/537.36 (compatible; GoogleOther) -Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.6261.94 Mobile Safari/537.36 (compatible; GoogleOther) \ No newline at end of file +Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.6261.94 Mobile Safari/537.36 (compatible; GoogleOther) +Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots) +Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots) \ No newline at end of file diff --git a/tests/devices.txt b/tests/data/user_agent/devices.txt similarity index 99% rename from tests/devices.txt rename to tests/data/user_agent/devices.txt index 5d07f58..89ec1be 100644 --- a/tests/devices.txt +++ b/tests/data/user_agent/devices.txt @@ -165632,4 +165632,6 @@ Mozilla/5.0 (iPhone; CPU iPhone OS 13_3_1 like Mac OS X) AppleWebKit/605.1.15 (K Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148/mvmkbtaqqdhshwcbsygjnuqcqmpndbrt Mozilla/5.0 (Linux; Android 5.0; SM-N9100 Build/LRX21V) > AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 > Chrome/37.0.0.0 Mobile Safari/537.36 > MicroMessenger/6.0.2.56_r958800.520 NetType/WIFI Mozilla/5.0 (Linux; U; Android 5.0; en-US; SM-N900S;iPhone: Build/LRX21V) -Mozilla/5.0 (Linux; Android 9; OCTOPUS-CORE T8 Build/PPR1.181005.003) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Mobile Safari/537.36 YaApp_Android/9.50/apad YaSearchBrowser/9.50 \ No newline at end of file +Mozilla/5.0 (Linux; Android 9; OCTOPUS-CORE T8 Build/PPR1.181005.003) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Mobile Safari/537.36 YaApp_Android/9.50/apad YaSearchBrowser/9.50 +Mozilla/5.0 (Linux; Android 9; YandexModule2-00001 Build/PI; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/113.0.5672.163 YaBrowser/24.1.2.86 (lite) TV Safari/537.36 +Mozilla/5.0 (Linux; Android 9; YandexModule2-00001 Build/PI; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/113.0.5672.163 Mobile Safari/537.36 \ No newline at end of file