Fix HHVM tests and add more user agents and tests (#201)

* Add Abounti user agent

* Add CloudEndure #200

* Add 5 more user agents

* Try fixing HHVM failure travis-ci/travis-ci#7712

.travis.yml

@@ -9,6 +9,7 @@ matrix:
     - php: 7.0
     - php: 7.1
     - php: hhvm
+      dist: trusty
     - php: nightly
   allow_failures:
     - php: nightly

src/Fixtures/Crawlers.php

@@ -51,6 +51,7 @@ class Crawlers extends AbstractProvider
         'a3logics\.in',
         'A6-Indexer',
         'a\.pr-cy\.ru',
+        'Abonti\/',
         'Aboundex',
         'aboutthedomain',
         'Accoona-AI-Agent',
@@ -70,6 +71,7 @@ class Crawlers extends AbstractProvider
         'alertra',
         'alexa site audit',
         'Alibaba\.Security\.Heimdall',
+        'allloadin\.com',
         'alyze\.info',
         'amagit',
         'AndroidDownloadManager',
@@ -142,6 +144,7 @@ class Crawlers extends AbstractProvider
         'CJNetworkQuality',
         'clips\.ua\.ac\.be',
         'Cloud mapping experiment',
+        'CloudEndure',
         'CloudFlare-AlwaysOnline',
         'Cloudinary\/[0-9]',
         'cmcm\.com',
@@ -190,6 +193,7 @@ class Crawlers extends AbstractProvider
         'eCairn-Grabber',
         'ECCP',
         'echocrawl',
+        'eContext\/',
         'ElectricMonk',
         'elefent',
         'EMail Exractor',
@@ -375,7 +379,6 @@ class Crawlers extends AbstractProvider
         'Kml-Google',
         'knows\.is',
         'kouio',
-        'KrOWLer',
         'kulturarw3',
         'KumKie',
         'L\.webis',
@@ -485,6 +488,7 @@ class Crawlers extends AbstractProvider
         'Optimizer',
         'Orbiter',
         'OrgProbe\/[0-9]',
+        'Owler',
         'ow\.ly',
         'ownCloud News',
         'OxfordCloudService\/[0-9]',
@@ -622,6 +626,7 @@ class Crawlers extends AbstractProvider
         'Snoopy',
         'sogou web',
         'SortSite',
+        'sovereign\.ai',
         'spaziodati',
         'Specificfeeds',
         'speedy',
@@ -661,6 +666,7 @@ class Crawlers extends AbstractProvider
         'ThumbSniper',
         'TinEye',
         'Tiny Tiny RSS',
+        'TLSProbe\/',
         'topster',
         'touche.com',
         'Traackr.com',

tests/crawlers.txt

@@ -3083,4 +3083,31 @@ Kaspersky Lab CFR link resolver [email protected]
 Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453 Safari/537.36 (compatible; Google-HotelAdsVerifier/2.0)
 b0t
 "echocrawl 2.0"
-masscan/1.0
+masscan/1.0
+Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B411 Safari/600.1.4 (compatible; YandexBot/3.0; +http://yandex.com/bots)
+Mozilla/5.0 (compatible; YandexAccessibilityBot/3.0; +http://yandex.com/bots)
+Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B411 Safari/600.1.4 (compatible; YandexMobileBot/3.0; +http://yandex.com/bots)
+Mozilla/5.0 (compatible; YandexDirectDyn/1.0; +http://yandex.com/bots
+Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)
+Mozilla/5.0 (compatible; YandexVideo/3.0; +http://yandex.com/bots)
+Mozilla/5.0 (compatible; YandexMedia/3.0; +http://yandex.com/bots)
+Mozilla/5.0 (compatible; YandexBlogs/0.99; robot; +http://yandex.com/bots)
+Mozilla/5.0 (compatible; YandexFavicons/1.0; +http://yandex.com/bots)
+Mozilla/5.0 (compatible; YandexWebmaster/2.0; +http://yandex.com/bots)
+Mozilla/5.0 (compatible; YandexPagechecker/1.0; +http://yandex.com/bots)
+Mozilla/5.0 (compatible; YandexImageResizer/2.0; +http://yandex.com/bots)
+Mozilla/5.0 (compatible; YaDirectFetcher/1.0; Dyatel; +http://yandex.com/bots)
+Mozilla/5.0 (compatible; YandexCalendar/1.0; +http://yandex.com/bots)
+Mozilla/5.0 (compatible; YandexSitelinks; Dyatel; +http://yandex.com/bots)
+Mozilla/5.0 (compatible; YandexMetrika/3.0; +http://yandex.com/bots)
+Mozilla/5.0 (compatible; YandexAntivirus/2.0; +http://yandex.com/bots)
+Mozilla/5.0 (compatible; YandexVertis/3.0; +http://yandex.com/bots)
+Mozilla/5.0 (compatible; YandexBot/3.0; MirrorDetector; +http://yandex.com/bots)
+jetmon/1.0 (Jetpack Site Uptime Monitor by WordPress.com)
+Mozilla/5.0 (compatible; Abonti/0.8 - http://www.abonti.com)  
+CloudEndure Scanner ([email protected])
+AFB/3.0 (+http://allloadin.com)
+TLSProbe/1.0 (+https://scan.trustnet.venafi.com/)
+Sens.ai http://sovereign.ai/contact
+eContext/1.0 (eContext Classification Engine)
+Owler