From de631df3de359ef758ccc1e287e1e66a74337e5b Mon Sep 17 00:00:00 2001 From: Mark Beech Date: Wed, 10 Oct 2018 20:58:09 +0100 Subject: [PATCH] Run export --- raw/Crawlers.json | 2 +- raw/Crawlers.txt | 109 ++++++++++++++++++++++++++++------------------ 2 files changed, 67 insertions(+), 44 deletions(-) diff --git a/raw/Crawlers.json b/raw/Crawlers.json index b1e1bf3..c3ea77f 100644 --- a/raw/Crawlers.json +++ b/raw/Crawlers.json @@ -1 +1 @@ -[".*Java.*outbrain"," YLT","008\\\/","192\\.comAgent","2ip\\.ru","404checker","404enemy","80legs","^b0t$","^bluefish ","^Calypso v\\\/","^COMODO DCV","^DangDang","^DavClnt","^FDM ","^git\\\/","^Goose\\\/","^Grabber","^HTTPClient\\\/","^Java\\\/","^Jeode\\\/","^Jetty\\\/","^Mail\\\/","^Mget","^Microsoft URL Control","^NG\\\/[0-9\\.]","^NING\\\/","^PHP\\\/[0-9]","^RMA\\\/","^Ruby|Ruby\\\/[0-9]","^VSE\\\/[0-9]","^WordPress\\.com","^XRL\\\/[0-9]","^ZmEu","a3logics\\.in","A6-Indexer","a\\.pr-cy\\.ru","Abonti","Aboundex","aboutthedomain","Accoona-AI-Agent","acoon","acrylicapps\\.com\\\/pulp","Acunetix","AdAuth\\\/","adbeat","AddThis","ADmantX","adressendeutschland","adscanner\\\/","Advanced Email Extractor v","agentslug","AHC","aihit","aiohttp\\\/","Airmail","akka-http\\\/","akula\\\/","alertra","alexa site audit","Alibaba\\.Security\\.Heimdall","Alligator","allloadin\\.com","AllSubmitter","alyze\\.info","amagit","Anarchie","AndroidDownloadManager","Anemone","AngleSharp\\\/","annotate_google","Ant\\.com","Anturis Agent","AnyEvent-HTTP\\\/","Apache Droid","Apache OpenOffice","Apache-HttpAsyncClient\\\/","Apache-HttpClient\\\/","ApacheBench\\\/","Apexoo","APIs-Google","AportWorm\\\/[0-9]","AppBeat\\\/[0-9]","AppEngine-Google","AppStoreScraperZ","Arachmo","arachnode","Arachnophilia","aria2","Arukereso","asafaweb.com","AskQuickly","ASPSeek","Asterias","Astute","asynchttp","Attach","autocite","Autonomy","axios\\\/","B-l-i-t-z-B-O-T","Backlink-Ceck","backlink-check","BackStreet","BackWeb","Bad-Neighborhood","Badass","baidu\\.com","Bandit","BatchFTP","Battleztar\\ Bazinga","baypup\\\/[0-9]","baypup\\\/colbert","BazQux","BBBike","BCKLINKS","BDFetch","BegunAdvertising\\\/","Bidtellect\\\/","BigBozz","Bigfoot","biglotron","BingLocalSearch","BingPreview","binlar","biNu image cacher","Bitacle","biz_Directory","Black\\ Hole","Blackboard Safeassign","BlackWidow","BlockNote\\.Net","Bloglines\\\/","Bloglovin","BlogPulseLive","BlogSearch","Blogtrottr","BlowFish","Boardreader","boitho\\.com-dc","BPImageWalker","Braintree-Webhooks","Branch Metrics API","Branch-Passthrough","Brandprotect","Brandwatch","Brodie\\\/","Browsershots","BUbiNG","Buck\\\/","Buddy","BuiltWith","Bullseye","BunnySlippers","Burf Search","Butterfly\\\/","BuzzSumo","CAAM\\\/[0-9]","CakePHP","Calculon","CapsuleChecker","CaretNail","catexplorador","cb crawl","CC Metadata Scaper","Cegbfeieh","Cerberian Drtrs","CERT\\.at-Statistics-Survey","cg-eye","changedetection","ChangesMeter\\\/","Charlotte","CheckHost","checkprivacy","CherryPicker","ChinaClaw","Chirp\\\/[0-9]","chkme\\.com","Chlooe","CirrusExplorer\\\/","CISPA Vulnerability Notification","Citoid","CJNetworkQuality","Clarsentia","clips\\.ua\\.ac\\.be","Cloud\\ mapping","CloudEndure","CloudFlare-AlwaysOnline","Cloudinary\\\/[0-9]","cmcm\\.com","coccoc","cognitiveseo","colly -","CommaFeed","Commons-HttpClient","Comodo SSL Checker","contactbigdatafr","contentkingapp","convera","CookieReports\\.com","copyright sheriff","CopyRightCheck","Copyscape","Cosmos4j\\.feedback","Covario-IDS","CrawlForMe\\\/[0-9]","Crescent","cron-job\\.org","Crowsnest","CSHttp","curb","Curious George","curl","cuwhois\\\/[0-9]","cybo\\.com","DareBoost","DatabaseDriverMysqli","DataCha0s","Datanyze","DataparkSearch","dataprovider","DataXu","Daum(oa)?[ \\\/][0-9]","DemandbasePublisherAnalyzer\\\/","Demon","DeuSu","developers\\.google\\.com\\\/\\+\\\/web\\\/snippet\\\/","Devil","Digg","Digincore","DigitalPebble","Dirbuster","Dispatch\\\/","DittoSpyder","dlvr","DMBrowser","DNS-Tools Header-Analyzer","DNSPod-reporting","docoloc","Dolphin http client\\\/","DomainAppender","Donuts Content Explorer","dotMailer content retrieval","dotSemantic","downforeveryoneorjustme","Download\\ Wonder","downnotifier\\.com","DowntimeDetector","Dragonfly File Reader","Drip","drupact","Drupal \\(\\+http:\\\/\\\/drupal\\.org\\\/\\)","DTS\\ Agent","dubaiindex","EARTHCOM","Easy-Thumb","EasyDL","Ebingbong","ec2linkfinder","eCairn-Grabber","eCatch","ECCP","echocrawl","eContext\\\/","Ecxi","EirGrabber","ElectricMonk","elefent","EMail Exractor","Email%20Extractor%20Lite","EMail\\ Wolf","EmailWolf","Embed PHP Library","Embedly","endo\\\/","europarchive\\.org","evc-batch","EventMachine HttpClient","Everwall Link Expander","Evidon","Evrinid","ExactSearch","ExaleadCloudview","Excel\\\/","Exif Viewer","ExperianCrawlUK","Exploratodo","Express WebPictures","ExtractorPro","Extreme\\ Picture\\ Finder","EyeNetIE","ezooms","facebookexternalhit","facebookplatform","fairshare","Faraday v","fasthttp","Faveeo","Favicon downloader","FavOrg","Feed Wrangler","Feedable\\\/","Feedbin","FeedBooster","FeedBucket","FeedBunch\\\/[0-9]","FeedBurner","FeedChecker","Feedly","Feedreader","FeedshowOnline","Feedspot","Feedwind\\\/[0-9]","FeedZcollector","feeltiptop","Fetch API","Fetch\\\/[0-9]","Fever\\\/[0-9]","FHscan","Fimap","findlink","findthatfile","FlashGet","FlipboardBrowserProxy","FlipboardProxy","FlipboardRSS","Flock\\\/","fluffy","Flunky","flynxapp","forensiq","FoundSeoTool\\\/[0-9]","free thumbnails","Freeuploader","FreeWebMonitoring SiteChecker","Funnelback","G-i-g-a-b-o-t","g00g1e\\.net","GAChecker","ganarvisitas\\\/[0-9]","geek-tools","Genderanalyzer","Genieo","GentleSource","Getintent","GetLinkInfo","getprismatic\\.com","GetRight","GetURLInfo\\\/[0-9]","GetWeb","Ghost Inspector","GigablastOpenSource","GIS-LABS","github-camo","github\\.com\\\/","Go [\\d\\.]* package http","Go http package","Go!Zilla","Go-Ahead-Got-It","Go-http-client","gobyus","gofetch","GomezAgent","gooblog","Goodzer\\\/[0-9]","GoogleCloudMonitoring","Google favicon","Google Keyword Suggestion","Google Keyword Tool","Google Page Speed Insights","Google PP Default","Google Search Console","Google Web Preview","Google-Adwords","Google-Apps-Script","Google-Calendar-Importer","Google-HotelAdsVerifier","Google-HTTP-Java-Client","Google-Publisher-Plugin","Google-SearchByImage","Google-Site-Verification","Google-Structured-Data-Testing-Tool","Google-Youtube-Links","google_partner_monitoring","GoogleDocs","GoogleHC\\\/","GoogleProducer","Gookey","GoScraper","GoSpotCheck","GoSquared-Status-Checker","gosquared-thumbnailer","Gotit","GoZilla","grabify","GrabNet","Grafula","Grammarly","GrapeFX","grokkit","grouphigh","grub-client","gSOAP\\\/","GT::WWW","GTmetrix","GuzzleHttp","gvfs\\\/","HAA(A)?RTLAND http client","Haansoft","hackney\\\/","Hatena","Havij","hawkReader","HEADMasterSEO","HeartRails_Capture","help@dataminr\\.com","heritrix","historious\\\/","hledejLevne\\.cz\\\/[0-9]","Hloader","HMView","Holmes","HonesoSearchEngine\\\/","HootSuite Image proxy","Hootsuite-WebFeed\\\/[0-9]","hosterstats","HostTracker","ht:\\\/\\\/check","htdig","HTMLparser","htmlyse\\.com","http-get","HTTP-Header-Abfrage","http-kit","http-request\\\/","HTTP-Tiny","HTTP::Lite","http\\.rb\\\/","HTTP_Compression_Test","http_request2","http_requester","HttpComponents","httphr","HTTPMon","httpscheck","httpssites_power","httpunit","HttpUrlConnection","httrack","huaweisymantec","HubPages.*crawlingpolicy","HubSpot ","Humanlinks","HyperZbozi.cz Feeder","i2kconnect\\\/","Iblog","ichiro","Id-search","IdeelaborPlagiaat","IDG Twitter Links Resolver","IDwhois\\\/[0-9]","Iframely","igdeSpyder","IlTrovatore","ImageVisu\\\/","Image\\ Fetch","Image\\ Sucker","ImageEngine\\\/","Imagga","imgsizer","InAGist","inbound\\.li parser","InDesign%20CC","Indy\\ Library","InetURL","infegy","infohelfer","InfoTekies","InfoWizards Reciprocal Link System PRO","inpwrd\\.com","instabid","Instapaper","Integrity","integromedb","Intelliseek","InterGET","Internet\\ Ninja","internet_archive","InternetSeer","internetVista monitor","intraVnews","IODC","IOI","iplabel","ips-agent","IPS\\\/[0-9]","IPWorks HTTP\\\/S Component","iqdb\\\/","Iria","Irokez","isitup\\.org","iskanie","iZSearch","janforman","Jaunt\\\/","Jbrofuzz","Jersey\\\/","JetCar","Jigsaw","Jobboerse","JobFeed discovery","Jobg8 URL Monitor","jobo","Jobrapido","Jobsearch1\\.5","JoinVision Generic","JolokiaPwn","Joomla","Jorgee","JS-Kit","JustView","Kaspersky Lab CFR link resolver","KeepRight OpenStreetMap Checker","Kelny\\\/","Kerrigan\\\/","KeyCDN","Keyword Extractor","Keyword\\ Density","Keywords Research","KickFire","KimonoLabs\\\/","Kml-Google","knows\\.is","KOCMOHABT","kouio","kube-probe","kulturarw3","KumKie","L\\.webis","Larbin","Lavf\\\/","LayeredExtractor","LeechFTP","LeechGet","letsencrypt","Lftp","LibVLC","LibWeb","Libwhisker","libwww","Licorne Image Snapshot","Liferea\\\/","Lightspeedsystems","Likse","link checker","Link Valet","link_thumbnailer","LinkAlarm\\\/","linkCheck","linkdex","LinkExaminer","linkfluence","linkpeek","LinkPreviewGenerator","LinkScan","LinksManager","LinkTiger","LinkWalker","Lipperhey","Litemage_walker","livedoor ScreenShot","LoadImpactRload","localsearch-web","LongURL API","looksystems\\.net","ltx71","lua-resty-http","lwp-request","lwp-trivial","LWP::Simple","lycos","LYT\\.SR","mabontland","Mag-Net","MagpieRSS","Mail.Ru","MailChimp","Majestic12","makecontact\\\/","Mandrill","MapperCmd","marketinggrader","MarkMonitor","MarkWatch","Mass\\ Downloader","masscan\\\/[0-9]","Mata\\ Hari","Mediapartners-Google","mediawords","MegaIndex\\.ru","Melvil Rawi\\\/","MergeFlow-PageReader","Metaspinner","MetaURI","MFC_Tear_Sample","Microsearch","Microsoft Office ","Microsoft Outlook","Microsoft Windows Network Diagnostics","Microsoft-WebDAV-MiniRedir","Microsoft\\ Data\\ Access","MIDown\\ tool","MIIxpc","Mindjet","Miniature.io\\\/","Miniflux","Mister\\ PiX","mixdata dot com","mixed-content-scan","Mixmax-LinkPreview","mixnode","Mnogosearch","mogimogi","Mojeek","Mojolicious \\(Perl\\)","monitis","Monit\\\/","Monitority\\\/[0-9]","montastic","MonTools","Moreover","Morfeus\\ Fucking\\ Scanner","Morning Paper","MovableType","mowser","Mrcgiguy","MS\\ Web\\ Services\\ Client\\ Protocol","MSFrontPage","mShots","MuckRack\\\/","muhstik-scan","MVAClient","MxToolbox\\\/","nagios","Najdi\\.si\\\/","Name\\ Intelligence","Nameprotect","Navroad","NearSite","Needle","Nessus","Net\\ Vampire","NetAnts","NETCRAFT","NetLyzer","NetMechanic","NetNewsWire","Netpursual","netresearch","NetShelter ContentScan","Netsparker","NetTrack","Netvibes","NetZIP","Neustar WPM","NeutrinoAPI","NewRelicPinger","NewsBlur .*Finder","NewsGator","newsme","newspaper\\\/","Nexgate Ruby Client","NG-Search","Nibbler","NICErsPRO","Nikto","nineconnections\\.com","NLNZ_IAHarvester","Nmap Scripting Engine","node-superagent","node-urllib\\\/","node\\.io","nominet\\.org\\.uk","Norton-Safeweb","Notifixious","notifyninja","nuhk","nutch","Nuzzel","nWormFeedFinder","Nymesis","NYU","Ocelli\\\/[0-9]","Octopus","oegp","og-scraper\\\/","Offline Explorer","Offline\\ Navigator","okhttp","Omea Reader","omgili","OMSC","Online Domain Tools","OpenCalaisSemanticProxy","Openfind","OpenLinkProfiler","Openstat\\\/","OpenVAS","Optimizer","Orbiter","OrgProbe\\\/[0-9]","orion-semantics","Outlook-Express","ow\\.ly","Owler","ownCloud News","OxfordCloudService\\\/[0-9]","Page Analyzer","Page Valet","page2rss","page\\ scorer","page_verifier","PageAnalyzer","PageGrabber","PagePeeker","PageScorer","Pagespeed\\\/[0-9]","Panopta","panscient","Papa\\ Foto","parsijoo","Pavuk","PayPal IPN","pcBrowser","Pcore-HTTP","PEAR HTTPRequest","Pearltrees","PECL::HTTP","peerindex","Peew","PeoplePal","Perlu -","PhantomJS Screenshoter","PhantomJS\\\/","Photon\\\/","phpcrawl","phpservermon","Pi-Monster","Picscout","Picsearch","PictureFinder","Pimonster","ping\\.blo\\.gs\\\/","Pingability","PingAdmin\\.Ru","Pingdom","Pingoscope","PingSpot","pinterest\\.com","Pixray","Pizilla","Plagger\\\/","PleaseCrawl","Ploetz \\+ Zeller","Plukkie","plumanalytics","PocketParser","Pockey","POE-Component-Client-HTTP","Pompos","Porkbun","Port Monitor","postano","PostmanRuntime\\\/","PostPost","postrank","PowerPoint\\\/","Priceonomics Analysis Engine","PrintFriendly\\.com","PritTorrent\\\/[0-9]","Prlog","probethenet","Project 25499","Promotion_Tools_www.searchenginepromotionhelp.com","prospectb2b","Protopage","ProWebWalker","proximic","PRTG Network Monitor","pshtt, https scanning","PTST ","PTST\\\/[0-9]+","Pulsepoint XT3 web scraper","Pump","Python-httplib2","python-requests","Python-urllib","Qirina Hurdler","QQDownload","QrafterPro","Qseero","Qualidator.com SiteAnalyzer","QueryN\\ Metasearch","Quora Link Preview","Qwantify","Radian6","RankActive","RankFlex","RankSonicSiteAuditor","raynette_httprequest","Readability","RealDownload","RealPlayer%20Downloader","RebelMouse","Recorder","RecurPost\\\/","redback\\\/","Redirect Checker Tool","ReederForMac","ReGet","RepoMonkey","request\\.js","ResponseCodeTest\\\/[0-9]","RestSharp","Riddler","Rival IQ","Robosourcer","Robozilla\\\/[0-9]","ROI Hunter","RPT-HTTPClient","RSSOwl","RssReader\\\/","safe-agent-scanner","SalesIntelligent","Saleslift","SauceNAO","SBIder","scalaj-http","scan\\.lol","ScanAlert","ScrapeBox Page Scanner","Scoop","scooter","ScoutJet","ScoutURLMonitor","Scrapy","Screaming","ScreenShotService\\\/[0-9]","Scrubby","Scrutiny\\\/","Search37\\\/","search\\.thunderstone","Searchestate","SearchExpress","SearchSight","Seeker","semanticdiscovery","semanticjuice","Semiocast HTTP client","Semrush","sentry\\\/","SEO Browser","Seo Servis","seo-nastroj.cz","Seobility","SEOCentro","SeoCheck","SEOkicks","Seomoz","SEOprofiler","SeopultContentAnalyzer","seoscanners","SEOstats","Server Density Service Monitoring","servernfo\\.com","SetCronJob\\\/","sexsearcher","Seznam","Shelob","Shodan","Shoppimon Analyzer","ShoppimonAgent\\\/[0-9]","ShopWiki","ShortLinkTranslate","shrinktheweb","Sideqik","SilverReader","SimplePie","SimplyFast","Siphon","SISTRIX","Site-Shot\\\/","Site24x7","Site\\ Sucker","SiteBar","Sitebeam","Sitebulb\\\/","SiteCondor","SiteExplorer","SiteGuardian","Siteimprove","SiteIndexed","Sitemap(s)? Generator","SiteMonitor","Siteshooter B0t","SiteSnagger","SiteSucker","SiteTruth","Sitevigil","sitexy\\.com","SkypeUriPreview","Slack\\\/","slider\\.com","slurp","SlySearch","SmartDownload","SMRF URL Expander","SMUrlExpander","Snarfer\\\/","Snake","Snappy","SniffRSS","sniptracker","Snoopy","SnowHaze Search","sogou web","SortSite","Sottopop","sovereign\\.ai","SpaceBison","Spammen","Spanner","spaziodati","SPDYCheck","Specificfeeds","speedy","SPEng","Spinn3r","spray-can","Sprinklr ","spyonweb","sqlmap","Sqlworm","Sqworm","SSL Labs","ssl-tools","StackRambler","Statastico\\\/","StatusCake","Steeler","Stratagems Kumo","Stroke.cz","StudioFACA","suchen","Sucuri","summify","Super Monitoring","SuperHTTP","Surphace Scout","Suzuran","SwiteScraper","Symfony BrowserKit","Symfony2 BrowserKit","SynHttpClient-Built","Sysomos","sysscan","Szukacz","T0PHackTeam","tAkeOut","Tarantula\\\/","Taringa UGC","Teleport","Telesoft","Telesphoreo","Telesphorep","Tenon\\.io","teoma","terrainformatica\\.com","Test Certificate Info","Tetrahedron\\\/[0-9]","The Drop Reaper","The Expert HTML Source Viewer","The Knowledge AI","The\\ Intraformant","theinternetrules","TheNomad","theoldreader\\.com","Thinklab","Thumbshots","ThumbSniper","TinEye","Tiny Tiny RSS","TLSProbe\\\/","Toata","topster","touche.com","Traackr.com","TrapitAgent","Trendiction","Trendsmap Resolver","trendspottr\\.com","truwoGPS","TulipChain","Turingos","Turnitin","tweetedtimes\\.com","Tweetminster","Tweezler\\\/","twibble","Twice","Twikle","Twingly","Twisted PageGetter","Typhoeus","ubermetrics-technologies","uclassify","uCrawlr\\\/","UdmSearch","unirest-java","UniversalFeedParser","Unshorten\\.It","Untiny","UnwindFetchor","updated","updown\\.io daemon","Upflow","Uptimia","URL Verifier","URLChecker","URLitor.com","urlresolver","Urlstat","UrlTrends Ranking Updater","URLy\\ Warning","URLy\\.Warning","Vacuum","Vagabondo","VB\\ Project","vBSEO","VCI","via ggpht\\.com GoogleImageProxy","VidibleScraper","Virusdie","visionutils","vkShare","VoidEYE","Voil","voltron","voyager\\\/","VSAgent\\\/[0-9]","VSB-TUO\\\/[0-9]","Vulnbusters Meter","VYU2","w3af\\.org","W3C-checklink","W3C-mobileOK","W3C_I18n-Checker","W3C_Unicorn","Wallpapers\\\/[0-9]+","WallpapersHD","wangling","Wappalyzer","WatchMouse","WbSrch\\\/","web-capture\\.net","Web-Monitoring","Web-sniffer","Web\\ Auto","Web\\ Collage","Web\\ Enhancer","Web\\ Fetch","Web\\ Fuck","Web\\ Pix","Web\\ Sauger","Web\\ Sucker","Webalta","Webauskunft","WebAuto","WebCapture","WebClient\\\/","webcollage","WebCookies","WebCopier","WebCorp","WebDoc","WebEnhancer","WebFetch","WebFuck","WebGo\\ IS","WebImageCollector","WebImages","WebIndex","webkit2png","WebLeacher","webmastercoffee","webmon ","WebPix","WebReaper","WebSauger","webscreenie","Webshag","Webshot","Website Analyzer\\\/","Website\\ Quester","WebsiteExtractor","websitepulse agent","websitepulse[+ ]checker","WebsiteQuester","Websnapr\\\/","Webster","WebStripper","WebSucker","Webthumb\\\/[0-9]","WebThumbnail","WebWhacker","WebZIP","WeCrawlForThePeace","WeLikeLinks","WEPA","WeSEE","wf84","Wfuzz\\\/","wget","WhatsApp","WhatsMyIP","WhatWeb","WhereGoes\\?","Whibse","WhoRunsCoinHive","Whynder Magnet","Windows-RSS-Platform","WinHttpRequest","WinPodder","wkhtmlto","wmtips","Woko","Word\\\/","WordPress\\\/","WordupinfoSearch","wotbox","WP Engine Install Performance API","wpif","wprecon\\.com survey","WPScan","wscheck","Wtrace","WWW-Collector-E","WWW-Mechanize","WWW::Mechanize","www\\.monitor\\.us","WWWOFFLE","x09Mozilla","x22Mozilla","XaxisSemanticsClassifier","Xenu Link Sleuth","XING-contenttabreceiver\\\/[0-9]","XmlSitemapGenerator","xpymep([0-9]?)\\.exe","Y!J-(ASR|BSC)","Yaanb","yacy","Yahoo Ad monitoring","Yahoo Link Preview","YahooCacheSystem","YahooYSMcm","YandeG","Yandex(?!Search)","yanga","yeti","Yo-yo","Yoleo Consumer","yoogliFetchAgent","YottaaMonitor","Your-Website-Sucks\\\/[0-9]","yourls\\.org","Zade","Zao","Zauba","Zemanta Aggregator","Zend\\\\Http\\\\Client","Zend_Http_Client","Zermelo","Zeus","zgrab","ZnajdzFoto","Zombie\\.js","ZyBorg","SpamExperts","[a-z0-9\\-_]*(bot|crawler|archiver|transcoder|spider|uptime|validator|fetcher)"] \ No newline at end of file +[".*Java.*outbrain"," YLT","^b0t$","^bluefish ","^Calypso v\\\/","^COMODO DCV","^DangDang","^DavClnt","^FDM ","^git\\\/","^Goose\\\/","^Grabber","^HTTPClient\\\/","^Java\\\/","^Jeode\\\/","^Jetty\\\/","^Mail\\\/","^Mget","^Microsoft URL Control","^NG\\\/[0-9\\.]","^NING\\\/","^PHP\\\/[0-9]","^RMA\\\/","^Ruby|Ruby\\\/[0-9]","^VSE\\\/[0-9]","^WordPress\\.com","^XRL\\\/[0-9]","^ZmEu","008\\\/","13TABS","192\\.comAgent","2ip\\.ru","404checker","404enemy","7Siters","80legs","a\\.pr-cy\\.ru","a3logics\\.in","A6-Indexer","Abonti","Aboundex","aboutthedomain","Accoona-AI-Agent","acoon","acrylicapps\\.com\\\/pulp","Acunetix","AdAuth\\\/","adbeat","AddThis","ADmantX","adressendeutschland","adscanner\\\/","Advanced Email Extractor v","agentslug","AHC","aihit","aiohttp\\\/","Airmail","Akamai_Site_Analyzer","akka-http\\\/","akula\\\/","alertra","alexa site audit","Alibaba\\.Security\\.Heimdall","Alligator","allloadin\\.com","AllSubmitter","alyze\\.info","amagit","Anarchie","AndroidDownloadManager","Anemone","AngleSharp\\\/","annotate_google","Ant\\.com","Anturis Agent","AnyEvent-HTTP\\\/","Apache Droid","Apache OpenOffice","Apache-HttpAsyncClient\\\/","Apache-HttpClient\\\/","ApacheBench\\\/","Apexoo","APIs-Google","AportWorm\\\/[0-9]","AppBeat\\\/[0-9]","AppEngine-Google","AppStoreScraperZ","Aprc\\\/[0-9]","Arachmo","arachnode","Arachnophilia","aria2","Arukereso","asafaweb.com","AskQuickly","ASPSeek","Asterias","Astute","asynchttp","Attach","autocite","Autonomy","axios\\\/","B-l-i-t-z-B-O-T","Backlink-Ceck","backlink-check","BackStreet","BackWeb","Bad-Neighborhood","Badass","baidu\\.com","Bandit","basicstate","BatchFTP","Battleztar\\ Bazinga","baypup\\\/[0-9]","baypup\\\/colbert","BazQux","BBBike","BCKLINKS","BDFetch","BegunAdvertising\\\/","Bidtellect\\\/","BigBozz","Bigfoot","biglotron","BingLocalSearch","BingPreview","binlar","biNu image cacher","Bitacle","biz_Directory","Black\\ Hole","Blackboard Safeassign","BlackWidow","BlockNote\\.Net","Bloglines\\\/","Bloglovin","BlogPulseLive","BlogSearch","Blogtrottr","BlowFish","Boardreader","boitho\\.com-dc","BPImageWalker","Braintree-Webhooks","Branch Metrics API","Branch-Passthrough","Brandprotect","BrandVerity\\\/[0-9]","Brandwatch","Brodie\\\/","Browsershots","BUbiNG","Buck\\\/","Buddy","BuiltWith","Bullseye","BunnySlippers","Burf Search","Butterfly\\\/","BuzzSumo","CAAM\\\/[0-9]","CakePHP","Calculon","CapsuleChecker","CaretNail","catexplorador","CC Metadata Scaper","Cegbfeieh","censys","Cerberian Drtrs","CERT\\.at-Statistics-Survey","cg-eye","changedetection","ChangesMeter\\\/","Charlotte","CheckHost","checkprivacy","CherryPicker","ChinaClaw","Chirp\\\/[0-9]","chkme\\.com","Chlooe","Chromaxa","CirrusExplorer\\\/","CISPA Vulnerability Notification","Citoid","CJNetworkQuality","Clarsentia","clips\\.ua\\.ac\\.be","Cloud\\ mapping","CloudEndure","CloudFlare-AlwaysOnline","Cloudinary\\\/[0-9]","cmcm\\.com","coccoc","cognitiveseo","colly -","CommaFeed","Commons-HttpClient","Comodo SSL Checker","contactbigdatafr","contentkingapp","convera","CookieReports\\.com","copyright sheriff","CopyRightCheck","Copyscape","Cosmos4j\\.feedback","Covario-IDS","Crescent","Crowsnest","CSHttp","curb","Curious George","curl","cuwhois\\\/[0-9]","cybo\\.com","DareBoost","DatabaseDriverMysqli","DataCha0s","Datanyze","DataparkSearch","dataprovider","DataXu","Daum(oa)?[ \\\/][0-9]","DemandbasePublisherAnalyzer\\\/","Demon","DeuSu","developers\\.google\\.com\\\/\\+\\\/web\\\/snippet\\\/","Devil","Digg","Digincore","DigitalPebble","Dirbuster","Dispatch\\\/","DittoSpyder","dlvr","DMBrowser","DNS-Tools Header-Analyzer","DNSPod-reporting","docoloc","Dolphin http client\\\/","DomainAppender","Donuts Content Explorer","dotMailer content retrieval","dotSemantic","downforeveryoneorjustme","Download\\ Wonder","downnotifier\\.com","DowntimeDetector","Dragonfly File Reader","Drip","drupact","Drupal \\(\\+http:\\\/\\\/drupal\\.org\\\/\\)","DTS\\ Agent","dubaiindex","EARTHCOM","Easy-Thumb","EasyDL","Ebingbong","ec2linkfinder","eCairn-Grabber","eCatch","ECCP","eContext\\\/","Ecxi","EirGrabber","ElectricMonk","elefent","EMail Exractor","EMail\\ Wolf","Email%20Extractor%20Lite","EmailWolf","Embed PHP Library","Embedly","endo\\\/","europarchive\\.org","evc-batch","EventMachine HttpClient","Everwall Link Expander","Evidon","Evrinid","ExactSearch","ExaleadCloudview","Excel\\\/","exif","Exploratodo","Express WebPictures","ExtractorPro","Extreme\\ Picture\\ Finder","EyeNetIE","ezooms","facebookexternalhit","facebookplatform","fairshare","Faraday v","fasthttp","Faveeo","Favicon downloader","faviconkit","FavOrg","Feed Wrangler","Feedable\\\/","Feedbin","FeedBooster","FeedBucket","FeedBunch\\\/[0-9]","FeedBurner","FeedChecker","Feedly","Feedreader","FeedshowOnline","Feedspot","Feedwind\\\/[0-9]","FeedZcollector","feeltiptop","Fetch API","Fetch\\\/[0-9]","Fever\\\/[0-9]","FHscan","Fimap","findlink","findthatfile","FlashGet","FlipboardBrowserProxy","FlipboardProxy","FlipboardRSS","Flock\\\/","fluffy","Flunky","flynxapp","forensiq","FoundSeoTool\\\/[0-9]","free thumbnails","Freeuploader","FreeWebMonitoring SiteChecker","Funnelback","G-i-g-a-b-o-t","g00g1e\\.net","GAChecker","ganarvisitas\\\/[0-9]","geek-tools","Genderanalyzer","Genieo","GentleSource","Getintent","GetLinkInfo","getprismatic\\.com","GetRight","getroot","GetURLInfo\\\/[0-9]","GetWeb","Ghost Inspector","GigablastOpenSource","GIS-LABS","github-camo","github\\.com\\\/","Go [\\d\\.]* package http","Go http package","Go-Ahead-Got-It","Go-http-client","Go!Zilla","gobyus","gofetch","GomezAgent","gooblog","Goodzer\\\/[0-9]","Google AppsViewer","Google favicon","Google Keyword Suggestion","Google Keyword Tool","Google Page Speed Insights","Google PP Default","Google Search Console","Google Web Preview","google_partner_monitoring","Google-Adwords","Google-Apps-Script","Google-Calendar-Importer","Google-HotelAdsVerifier","Google-HTTP-Java-Client","Google-Publisher-Plugin","Google-SearchByImage","Google-Site-Verification","Google-Structured-Data-Testing-Tool","Google-Youtube-Links","GoogleCloudMonitoring","GoogleDocs","GoogleHC\\\/","GoogleProducer","GoogleSites","Gookey","GoScraper","GoSpotCheck","GoSquared-Status-Checker","gosquared-thumbnailer","Gotit","GoZilla","grabify","GrabNet","Grafula","Grammarly","GrapeFX","grokkit","grouphigh","grub-client","gSOAP\\\/","GT::WWW","GTmetrix","GuzzleHttp","gvfs\\\/","HAA(A)?RTLAND http client","Haansoft","hackney\\\/","Hatena","Havij","hawkReader","HEADMasterSEO","HeartRails_Capture","help@dataminr\\.com","heritrix","historious\\\/","hkedcity","hledejLevne\\.cz\\\/[0-9]","Hloader","HMView","Holmes","HonesoSearchEngine\\\/","HootSuite Image proxy","Hootsuite-WebFeed\\\/[0-9]","hosterstats","HostTracker","ht:\\\/\\\/check","htdig","HTMLparser","htmlyse\\.com","HTTP_Compression_Test","http_request2","http_requester","http-get","HTTP-Header-Abfrage","http-kit","http-request\\\/","HTTP-Tiny","HTTP::Lite","http\\.rb\\\/","HttpComponents","httphr","HTTPMon","httpscheck","httpssites_power","httpunit","HttpUrlConnection","httrack","huaweisymantec","HubSpot ","Humanlinks","HyperZbozi.cz Feeder","i2kconnect\\\/","Iblog","ichiro","Id-search","IdeelaborPlagiaat","IDG Twitter Links Resolver","IDwhois\\\/[0-9]","Iframely","igdeSpyder","IlTrovatore","Image\\ Fetch","Image\\ Sucker","ImageEngine\\\/","ImageVisu\\\/","Imagga","imagineeasy","imgsizer","InAGist","inbound\\.li parser","InDesign%20CC","Indy\\ Library","InetURL","infegy","infohelfer","InfoTekies","InfoWizards Reciprocal Link System PRO","inpwrd\\.com","instabid","Instapaper","Integrity","integromedb","Intelliseek","InterGET","internet_archive","Internet\\ Ninja","InternetSeer","internetVista monitor","intraVnews","IODC","IOI","iplabel","ips-agent","IPS\\\/[0-9]","IPWorks HTTP\\\/S Component","iqdb\\\/","Iria","Irokez","isitup\\.org","iskanie","isUp\\.li","iThemes Sync\\\/[0-9]","iZSearch","JAHHO","janforman","Jaunt\\\/","Jbrofuzz","Jersey\\\/","JetCar","Jigsaw","Jobboerse","JobFeed discovery","Jobg8 URL Monitor","jobo","Jobrapido","Jobsearch1\\.5","JoinVision Generic","JolokiaPwn","Joomla","Jorgee","JS-Kit","JustView","Kaspersky Lab CFR link resolver","KeepRight OpenStreetMap Checker","Kelny\\\/","Kerrigan\\\/","KeyCDN","Keyword Extractor","Keyword\\ Density","Keywords Research","KickFire","KimonoLabs\\\/","Kml-Google","knows\\.is","KOCMOHABT","kouio","kube-probe","kulturarw3","KumKie","L\\.webis","Larbin","Lavf\\\/","LayeredExtractor","LeechFTP","LeechGet","letsencrypt","Lftp","LibVLC","LibWeb","Libwhisker","libwww","Licorne","Liferea\\\/","Lightspeedsystems","Likse","link checker","Link Valet","link_thumbnailer","LinkAlarm\\\/","linkCheck","linkdex","LinkExaminer","linkfluence","linkpeek","LinkPreviewGenerator","LinkScan","LinksManager","LinkTiger","LinkWalker","Lipperhey","Litemage_walker","livedoor ScreenShot","LoadImpactRload","localsearch-web","LongURL API","looksystems\\.net","ltx71","lua-resty-http","lwp-request","lwp-trivial","LWP::Simple","lycos","LYT\\.SR","mabontland","Mag-Net","MagpieRSS","Mail.Ru","MailChimp","Majestic12","makecontact\\\/","Mandrill","MapperCmd","marketinggrader","MarkMonitor","MarkWatch","Mass\\ Downloader","masscan\\\/[0-9]","Mata\\ Hari","Mediapartners-Google","mediawords","MegaIndex\\.ru","Melvil Rawi\\\/","MergeFlow-PageReader","Metaspinner","MetaURI","MFC_Tear_Sample","Microsearch","Microsoft Office ","Microsoft Outlook","Microsoft Windows Network Diagnostics","Microsoft-WebDAV-MiniRedir","Microsoft\\ Data\\ Access","MIDown\\ tool","MIIxpc","Mindjet","Miniature.io\\\/","Miniflux","Mister\\ PiX","mixdata dot com","mixed-content-scan","Mixmax-LinkPreview","mixnode","Mnogosearch","mogimogi","Mojeek","Mojolicious \\(Perl\\)","Monit\\\/","monitis","Monitority\\\/[0-9]","montastic","MonTools","Moreover","Morfeus\\ Fucking\\ Scanner","Morning Paper","MovableType","mowser","Mrcgiguy","MS\\ Web\\ Services\\ Client\\ Protocol","MSFrontPage","mShots","MuckRack\\\/","muhstik-scan","MVAClient","MxToolbox\\\/","nagios","Najdi\\.si\\\/","Name\\ Intelligence","Nameprotect","Navroad","NearSite","Needle","Nessus","Net\\ Vampire","NetAnts","NETCRAFT","NetLyzer","NetMechanic","NetNewsWire","Netpursual","netresearch","NetShelter ContentScan","Netsparker","NetTrack","Netvibes","NetZIP","Neustar WPM","NeutrinoAPI","NewRelicPinger","NewsBlur .*Finder","NewsGator","newsme","newspaper\\\/","Nexgate Ruby Client","NG-Search","Nibbler","NICErsPRO","Nikto","nineconnections\\.com","NLNZ_IAHarvester","Nmap Scripting Engine","node-superagent","node-urllib\\\/","node\\.io","Nodemeter","NodePing","nominet\\.org\\.uk","Norton-Safeweb","Notifixious","notifyninja","nuhk","nutch","Nuzzel","nWormFeedFinder","Nymesis","NYU","Ocelli\\\/[0-9]","Octopus","oegp","Offline Explorer","Offline\\ Navigator","og-scraper\\\/","okhttp","Omea Reader","omgili","OMSC","Online Domain Tools","OpenCalaisSemanticProxy","Openfind","OpenLinkProfiler","Openstat\\\/","OpenVAS","Optimizer","Orbiter","OrgProbe\\\/[0-9]","orion-semantics","Outlook-Express","ow\\.ly","Owler","ownCloud News","OxfordCloudService\\\/[0-9]","Page Analyzer","Page Valet","page_verifier","page\\ scorer","page2rss","PageAnalyzer","PageGrabber","PagePeeker","PageScorer","Pagespeed\\\/[0-9]","Panopta","panscient","Papa\\ Foto","parsijoo","Pavuk","PayPal IPN","pcBrowser","Pcore-HTTP","PEAR HTTPRequest","Pearltrees","PECL::HTTP","peerindex","Peew","PeoplePal","Perlu -","PhantomJS Screenshoter","PhantomJS\\\/","Photon\\\/","phpservermon","Pi-Monster","Picscout","Picsearch","PictureFinder","Pimonster","ping\\.blo\\.gs\\\/","Pingability","PingAdmin\\.Ru","Pingdom","Pingoscope","PingSpot","pinterest\\.com","Pixray","Pizilla","Plagger\\\/","Ploetz \\+ Zeller","Plukkie","plumanalytics","PocketImageCache","PocketParser","Pockey","POE-Component-Client-HTTP","Pompos","Porkbun","Port Monitor","postano","PostmanRuntime\\\/","PostPost","postrank","PowerPoint\\\/","Priceonomics Analysis Engine","PrintFriendly\\.com","PritTorrent\\\/[0-9]","Prlog","probethenet","Project 25499","Promotion_Tools_www.searchenginepromotionhelp.com","prospectb2b","Protopage","ProWebWalker","proximic","PRTG Network Monitor","pshtt, https scanning","PTST ","PTST\\\/[0-9]+","Pulsepoint XT3 web scraper","Pump","Python-httplib2","python-requests","Python-urllib","Qirina Hurdler","QQDownload","QrafterPro","Qseero","Qualidator.com SiteAnalyzer","QueryN\\ Metasearch","queuedriver","Quora Link Preview","Qwantify","Radian6","RankActive","RankFlex","RankSonicSiteAuditor","raynette_httprequest","Re-re Studio","Readability","RealDownload","RealPlayer%20Downloader","RebelMouse","Recorder","RecurPost\\\/","redback\\\/","Redirect Checker Tool","ReederForMac","ReGet","RepoMonkey","request\\.js","ResponseCodeTest\\\/[0-9]","RestSharp","Riddler","Rival IQ","Robosourcer","Robozilla\\\/[0-9]","ROI Hunter","RPT-HTTPClient","rss reader","RSSOwl","RssReader\\\/","safe-agent-scanner","SalesIntelligent","Saleslift","SauceNAO","SBIder","scalaj-http","scan\\.lol","ScanAlert","Scoop","scooter","ScoutJet","ScoutURLMonitor","ScrapeBox Page Scanner","Scrapy","Screaming","ScreenShotService\\\/[0-9]","Scrubby","Scrutiny\\\/","search\\.thunderstone","Search37\\\/","Searchestate","SearchExpress","SearchSight","Seeker","semanticdiscovery","semanticjuice","Semiocast HTTP client","Semrush","sentry\\\/","SEO Browser","Seo Servis","seo-nastroj.cz","seo4ajax","Seobility","SEOCentro","SeoCheck","SEOkicks","Seomoz","SEOprofiler","SeopultContentAnalyzer","seoscanners","SEOstats","Server Density Service Monitoring","servernfo\\.com","sexsearcher","Seznam","Shelob","Shodan","Shoppimon Analyzer","ShoppimonAgent\\\/[0-9]","ShopWiki","ShortLinkTranslate","shrinktheweb","Sideqik","SilverReader","SimplePie","SimplyFast","Siphon","SISTRIX","Site-Shot\\\/","Site\\ Sucker","Site24x7","SiteBar","Sitebeam","Sitebulb\\\/","SiteCondor","SiteExplorer","SiteGuardian","Siteimprove","SiteIndexed","Sitemap(s)? Generator","SitemapGenerator","SiteMonitor","Siteshooter B0t","SiteSnagger","SiteSucker","SiteTruth","Sitevigil","sitexy\\.com","SkypeUriPreview","Slack\\\/","slider\\.com","slurp","SlySearch","SmartDownload","SMRF URL Expander","SMUrlExpander","Snake","Snappy","SnapSearch","Snarfer\\\/","SniffRSS","sniptracker","Snoopy","SnowHaze Search","sogou web","SortSite","Sottopop","sovereign\\.ai","SpaceBison","SpamExperts","Spammen","Spanner","spaziodati","SPDYCheck","Specificfeeds","speedy","SPEng","Spinn3r","spray-can","Sprinklr ","spyonweb","sqlmap","Sqlworm","Sqworm","SSL Labs","ssl-tools","StackRambler","Statastico\\\/","StatusCake","Steeler","Stratagems Kumo","Stroke.cz","StudioFACA","StumbleUpon","suchen","Sucuri","summify","Super Monitoring","SuperHTTP","Surphace Scout","Suzuran","SwiteScraper","Symfony BrowserKit","Symfony2 BrowserKit","SynHttpClient-Built","Sysomos","sysscan","Szukacz","T0PHackTeam","tAkeOut","Tarantula\\\/","Taringa UGC","TarmotGezgin","Teleport","Telesoft","Telesphoreo","Telesphorep","Tenon\\.io","teoma","terrainformatica\\.com","Test Certificate Info","Tetrahedron\\\/[0-9]","The Drop Reaper","The Expert HTML Source Viewer","The Knowledge AI","The\\ Intraformant","theinternetrules","TheNomad","theoldreader\\.com","Thinklab","Thumbshots","ThumbSniper","TinEye","Tiny Tiny RSS","TLSProbe\\\/","Toata","topster","touche.com","Traackr.com","TrapitAgent","Trendiction","Trendsmap Resolver","trendspottr\\.com","truwoGPS","TulipChain","Turingos","Turnitin","tweetedtimes\\.com","Tweetminster","Tweezler\\\/","twibble","Twice","Twikle","Twingly","Twisted PageGetter","Typhoeus","ubermetrics-technologies","uclassify","UdmSearch","unirest-java","UniversalFeedParser","Unshorten\\.It","Untiny","UnwindFetchor","updated","updown\\.io daemon","Upflow","Uptimia","URL Verifier","URLChecker","URLitor.com","urlresolver","Urlstat","UrlTrends Ranking Updater","URLy\\ Warning","URLy\\.Warning","Vacuum","Vagabondo","VB\\ Project","vBSEO","VCI","via ggpht\\.com GoogleImageProxy","VidibleScraper","Virusdie","visionutils","vkShare","VoidEYE","Voil","voltron","voyager\\\/","VSAgent\\\/[0-9]","VSB-TUO\\\/[0-9]","Vulnbusters Meter","VYU2","w3af\\.org","W3C_I18n-Checker","W3C_Unicorn","W3C-checklink","W3C-mobileOK","WAC-OFU","Wallpapers\\\/[0-9]+","WallpapersHD","wangling","Wappalyzer","WatchMouse","WbSrch\\\/","WDT\\.io","web-capture\\.net","Web-Monitoring","Web-sniffer","Web\\ Auto","Web\\ Collage","Web\\ Enhancer","Web\\ Fetch","Web\\ Fuck","Web\\ Pix","Web\\ Sauger","Web\\ Sucker","Webalta","Webauskunft","WebAuto","WebCapture","WebClient\\\/","webcollage","WebCookies","WebCopier","WebCorp","WebDataStats\\\/[0-9]","WebDoc","WebEnhancer","WebFetch","WebFuck","WebGo\\ IS","WebImageCollector","WebImages","WebIndex","webkit2png","WebLeacher","webmastercoffee","webmon ","WebPix","WebReaper","WebSauger","webscreenie","Webshag","Webshot","Website Analyzer\\\/","Website\\ Quester","WebsiteExtractor","websitepulse agent","websitepulse[+ ]checker","WebsiteQuester","Websnapr\\\/","Webster","WebStripper","WebSucker","Webthumb\\\/[0-9]","WebThumbnail","WebWhacker","WebZIP","WeLikeLinks","WEPA","WeSEE","wf84","Wfuzz\\\/","wget","WhatsApp","WhatsMyIP","WhatWeb","WhereGoes\\?","Whibse","WhoRunsCoinHive","Whynder Magnet","Windows-RSS-Platform","WinHttpRequest","WinPodder","wkhtmlto","wmtips","Woko","Word\\\/","WordPress\\\/","WordupinfoSearch","wotbox","WP Engine Install Performance API","wpif","wprecon\\.com survey","WPScan","wscheck","Wtrace","WWW-Collector-E","WWW-Mechanize","WWW::Document","WWW::Mechanize","www\\.monitor\\.us","WWWOFFLE","x09Mozilla","x22Mozilla","XaxisSemanticsClassifier","Xenu Link Sleuth","XING-contenttabreceiver\\\/[0-9]","xpymep([0-9]?)\\.exe","Y!J-(ASR|BSC)","Y\\!J-BRW","Yaanb","yacy","Yahoo Ad monitoring","Yahoo Link Preview","YahooCacheSystem","YahooYSMcm","YandeG","Yandex(?!Search)","yanga","yeti","Yo-yo","Yoleo Consumer","yoogliFetchAgent","YottaaMonitor","Your-Website-Sucks\\\/[0-9]","yourls\\.org","YoYs\\.net","YP\\.PL","Zade","Zao","Zauba","Zemanta Aggregator","Zend_Http_Client","Zend\\\\Http\\\\Client","Zermelo","Zeus","zgrab","ZnajdzFoto","Zombie\\.js","ZyBorg","[a-z0-9\\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron)"] \ No newline at end of file diff --git a/raw/Crawlers.txt b/raw/Crawlers.txt index 5022bd9..f3697fc 100644 --- a/raw/Crawlers.txt +++ b/raw/Crawlers.txt @@ -1,11 +1,5 @@ .*Java.*outbrain YLT -008\/ -192\.comAgent -2ip\.ru -404checker -404enemy -80legs ^b0t$ ^bluefish ^Calypso v\/ @@ -32,9 +26,17 @@ ^WordPress\.com ^XRL\/[0-9] ^ZmEu +008\/ +13TABS +192\.comAgent +2ip\.ru +404checker +404enemy +7Siters +80legs +a\.pr-cy\.ru a3logics\.in A6-Indexer -a\.pr-cy\.ru Abonti Aboundex aboutthedomain @@ -54,6 +56,7 @@ AHC aihit aiohttp\/ Airmail +Akamai_Site_Analyzer akka-http\/ akula\/ alertra @@ -83,6 +86,7 @@ AportWorm\/[0-9] AppBeat\/[0-9] AppEngine-Google AppStoreScraperZ +Aprc\/[0-9] Arachmo arachnode Arachnophilia @@ -107,6 +111,7 @@ Bad-Neighborhood Badass baidu\.com Bandit +basicstate BatchFTP Battleztar\ Bazinga baypup\/[0-9] @@ -143,6 +148,7 @@ Braintree-Webhooks Branch Metrics API Branch-Passthrough Brandprotect +BrandVerity\/[0-9] Brandwatch Brodie\/ Browsershots @@ -161,9 +167,9 @@ Calculon CapsuleChecker CaretNail catexplorador -cb crawl CC Metadata Scaper Cegbfeieh +censys Cerberian Drtrs CERT\.at-Statistics-Survey cg-eye @@ -177,6 +183,7 @@ ChinaClaw Chirp\/[0-9] chkme\.com Chlooe +Chromaxa CirrusExplorer\/ CISPA Vulnerability Notification Citoid @@ -203,9 +210,7 @@ CopyRightCheck Copyscape Cosmos4j\.feedback Covario-IDS -CrawlForMe\/[0-9] Crescent -cron-job\.org Crowsnest CSHttp curb @@ -260,15 +265,14 @@ ec2linkfinder eCairn-Grabber eCatch ECCP -echocrawl eContext\/ Ecxi EirGrabber ElectricMonk elefent EMail Exractor -Email%20Extractor%20Lite EMail\ Wolf +Email%20Extractor%20Lite EmailWolf Embed PHP Library Embedly @@ -282,8 +286,7 @@ Evrinid ExactSearch ExaleadCloudview Excel\/ -Exif Viewer -ExperianCrawlUK +exif Exploratodo Express WebPictures ExtractorPro @@ -297,6 +300,7 @@ Faraday v fasthttp Faveeo Favicon downloader +faviconkit FavOrg Feed Wrangler Feedable\/ @@ -346,6 +350,7 @@ Getintent GetLinkInfo getprismatic\.com GetRight +getroot GetURLInfo\/[0-9] GetWeb Ghost Inspector @@ -355,15 +360,15 @@ github-camo github\.com\/ Go [\d\.]* package http Go http package -Go!Zilla Go-Ahead-Got-It Go-http-client +Go!Zilla gobyus gofetch GomezAgent gooblog Goodzer\/[0-9] -GoogleCloudMonitoring +Google AppsViewer Google favicon Google Keyword Suggestion Google Keyword Tool @@ -371,6 +376,7 @@ Google Page Speed Insights Google PP Default Google Search Console Google Web Preview +google_partner_monitoring Google-Adwords Google-Apps-Script Google-Calendar-Importer @@ -381,10 +387,11 @@ Google-SearchByImage Google-Site-Verification Google-Structured-Data-Testing-Tool Google-Youtube-Links -google_partner_monitoring +GoogleCloudMonitoring GoogleDocs GoogleHC\/ GoogleProducer +GoogleSites Gookey GoScraper GoSpotCheck @@ -416,6 +423,7 @@ HeartRails_Capture help@dataminr\.com heritrix historious\/ +hkedcity hledejLevne\.cz\/[0-9] Hloader HMView @@ -429,6 +437,9 @@ ht:\/\/check htdig HTMLparser htmlyse\.com +HTTP_Compression_Test +http_request2 +http_requester http-get HTTP-Header-Abfrage http-kit @@ -436,9 +447,6 @@ http-request\/ HTTP-Tiny HTTP::Lite http\.rb\/ -HTTP_Compression_Test -http_request2 -http_requester HttpComponents httphr HTTPMon @@ -448,7 +456,6 @@ httpunit HttpUrlConnection httrack huaweisymantec -HubPages.*crawlingpolicy HubSpot Humanlinks HyperZbozi.cz Feeder @@ -462,11 +469,12 @@ IDwhois\/[0-9] Iframely igdeSpyder IlTrovatore -ImageVisu\/ Image\ Fetch Image\ Sucker ImageEngine\/ +ImageVisu\/ Imagga +imagineeasy imgsizer InAGist inbound\.li parser @@ -484,8 +492,8 @@ Integrity integromedb Intelliseek InterGET -Internet\ Ninja internet_archive +Internet\ Ninja InternetSeer internetVista monitor intraVnews @@ -500,7 +508,10 @@ Iria Irokez isitup\.org iskanie +isUp\.li +iThemes Sync\/[0-9] iZSearch +JAHHO janforman Jaunt\/ Jbrofuzz @@ -548,7 +559,7 @@ LibVLC LibWeb Libwhisker libwww -Licorne Image Snapshot +Licorne Liferea\/ Lightspeedsystems Likse @@ -623,8 +634,8 @@ Mnogosearch mogimogi Mojeek Mojolicious \(Perl\) -monitis Monit\/ +monitis Monitority\/[0-9] montastic MonTools @@ -680,6 +691,8 @@ Nmap Scripting Engine node-superagent node-urllib\/ node\.io +Nodemeter +NodePing nominet\.org\.uk Norton-Safeweb Notifixious @@ -693,9 +706,9 @@ NYU Ocelli\/[0-9] Octopus oegp -og-scraper\/ Offline Explorer Offline\ Navigator +og-scraper\/ okhttp Omea Reader omgili @@ -717,9 +730,9 @@ ownCloud News OxfordCloudService\/[0-9] Page Analyzer Page Valet -page2rss -page\ scorer page_verifier +page\ scorer +page2rss PageAnalyzer PageGrabber PagePeeker @@ -743,7 +756,6 @@ Perlu - PhantomJS Screenshoter PhantomJS\/ Photon\/ -phpcrawl phpservermon Pi-Monster Picscout @@ -760,10 +772,10 @@ pinterest\.com Pixray Pizilla Plagger\/ -PleaseCrawl Ploetz \+ Zeller Plukkie plumanalytics +PocketImageCache PocketParser Pockey POE-Component-Client-HTTP @@ -801,6 +813,7 @@ QrafterPro Qseero Qualidator.com SiteAnalyzer QueryN\ Metasearch +queuedriver Quora Link Preview Qwantify Radian6 @@ -808,6 +821,7 @@ RankActive RankFlex RankSonicSiteAuditor raynette_httprequest +Re-re Studio Readability RealDownload RealPlayer%20Downloader @@ -828,6 +842,7 @@ Robosourcer Robozilla\/[0-9] ROI Hunter RPT-HTTPClient +rss reader RSSOwl RssReader\/ safe-agent-scanner @@ -838,18 +853,18 @@ SBIder scalaj-http scan\.lol ScanAlert -ScrapeBox Page Scanner Scoop scooter ScoutJet ScoutURLMonitor +ScrapeBox Page Scanner Scrapy Screaming ScreenShotService\/[0-9] Scrubby Scrutiny\/ -Search37\/ search\.thunderstone +Search37\/ Searchestate SearchExpress SearchSight @@ -862,6 +877,7 @@ sentry\/ SEO Browser Seo Servis seo-nastroj.cz +seo4ajax Seobility SEOCentro SeoCheck @@ -873,7 +889,6 @@ seoscanners SEOstats Server Density Service Monitoring servernfo\.com -SetCronJob\/ sexsearcher Seznam Shelob @@ -890,8 +905,8 @@ SimplyFast Siphon SISTRIX Site-Shot\/ -Site24x7 Site\ Sucker +Site24x7 SiteBar Sitebeam Sitebulb\/ @@ -901,6 +916,7 @@ SiteGuardian Siteimprove SiteIndexed Sitemap(s)? Generator +SitemapGenerator SiteMonitor Siteshooter B0t SiteSnagger @@ -916,9 +932,10 @@ SlySearch SmartDownload SMRF URL Expander SMUrlExpander -Snarfer\/ Snake Snappy +SnapSearch +Snarfer\/ SniffRSS sniptracker Snoopy @@ -928,6 +945,7 @@ SortSite Sottopop sovereign\.ai SpaceBison +SpamExperts Spammen Spanner spaziodati @@ -951,6 +969,7 @@ Steeler Stratagems Kumo Stroke.cz StudioFACA +StumbleUpon suchen Sucuri summify @@ -969,6 +988,7 @@ T0PHackTeam tAkeOut Tarantula\/ Taringa UGC +TarmotGezgin Teleport Telesoft Telesphoreo @@ -1014,7 +1034,6 @@ Twisted PageGetter Typhoeus ubermetrics-technologies uclassify -uCrawlr\/ UdmSearch unirest-java UniversalFeedParser @@ -1052,16 +1071,18 @@ VSB-TUO\/[0-9] Vulnbusters Meter VYU2 w3af\.org -W3C-checklink -W3C-mobileOK W3C_I18n-Checker W3C_Unicorn +W3C-checklink +W3C-mobileOK +WAC-OFU Wallpapers\/[0-9]+ WallpapersHD wangling Wappalyzer WatchMouse WbSrch\/ +WDT\.io web-capture\.net Web-Monitoring Web-sniffer @@ -1082,6 +1103,7 @@ webcollage WebCookies WebCopier WebCorp +WebDataStats\/[0-9] WebDoc WebEnhancer WebFetch @@ -1114,7 +1136,6 @@ Webthumb\/[0-9] WebThumbnail WebWhacker WebZIP -WeCrawlForThePeace WeLikeLinks WEPA WeSEE @@ -1146,6 +1167,7 @@ wscheck Wtrace WWW-Collector-E WWW-Mechanize +WWW::Document WWW::Mechanize www\.monitor\.us WWWOFFLE @@ -1154,9 +1176,9 @@ x22Mozilla XaxisSemanticsClassifier Xenu Link Sleuth XING-contenttabreceiver\/[0-9] -XmlSitemapGenerator xpymep([0-9]?)\.exe Y!J-(ASR|BSC) +Y\!J-BRW Yaanb yacy Yahoo Ad monitoring @@ -1173,17 +1195,18 @@ yoogliFetchAgent YottaaMonitor Your-Website-Sucks\/[0-9] yourls\.org +YoYs\.net +YP\.PL Zade Zao Zauba Zemanta Aggregator -Zend\\Http\\Client Zend_Http_Client +Zend\\Http\\Client Zermelo Zeus zgrab ZnajdzFoto Zombie\.js ZyBorg -SpamExperts -[a-z0-9\-_]*(bot|crawler|archiver|transcoder|spider|uptime|validator|fetcher) \ No newline at end of file +[a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron) \ No newline at end of file