Merge pull request #5 from zf-fr/twitter-bot

Add Twitterbot
zf-fr · Oct 14, 2013 · a5db3d6 · a5db3d6
2 parents 5b0cfb9 + 94c1a63
commit a5db3d6
Show file tree

Hide file tree

Showing 4 changed files with 11 additions and 4 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,7 @@
+# 1.1.1
+
+* Add Twitterbot User-Agent
+
 # 1.1.0
 
 * Add support for _escaped_fragment_ query param for detecting a bot

diff --git a/README.md b/README.md
@@ -58,7 +58,7 @@ With this config, here is how ZfrPrerender will proxy the "https://google.com" r
 #### Crawler user-agents
 
 ZfrPrerender decides to pre-render based on the User-Agent string to check if a request comes from a bot or not. By
-default, those user agents are registered: `googlebot`, `yahoo`, `bingbot`, `baidu`.
+default, those user agents are registered: `googlebot`, `yahoo`, `bingbot`, `baidu` and `twitterbot`.
 
 You can add other User-Agent string to evaluate using this sample configuration:
 
@@ -76,7 +76,9 @@ param. You can learn more about this on [Google's website](https://developers.go
 #### Ignored extensions
 
 ZfrPrerender is configured by default to ignore all the requests for resources with those extensions: `.css`,
-`.gif`, `.jpeg`, `.jpg`, `.js`, `.png`. Those are never pre-rendered.
+`.gif`, `.jpeg`, `.jpg`, `.js`, `.png`, `.less`, `.pdf`, `.doc`, `.txt`, `.zip`, `.mp3`, `.rar`, `.exe`, `.wmv`,
+`.doc`, `.avi`, `.ppt`, `.mpg`, `.mpeg`, `.tif`, `.wav`, `.mov`, `.psd`, `.ai`, `.xls`, `.mp4`, `.m4a`, `.swf`,
+`.dat`, `.dmg`, `.iso`, `.flv`, `.m4v`, `.torrent`. Those are never pre-rendered.
 
 You can add your own extensions using this sample configuration:
 

diff --git a/config/module.config.php b/config/module.config.php
@@ -34,7 +34,8 @@
             'yahoo',
             'bingbot',
             'baiduspider',
-            'facebookexternalhit'
+            'facebookexternalhit',
+            'twitterbot'
         ),
 
         // Ignored extensions

diff --git a/tests/ZfrPrerenderTest/Factory/ModuleOptionsFactoryTest.php b/tests/ZfrPrerenderTest/Factory/ModuleOptionsFactoryTest.php
@@ -37,7 +37,7 @@ public function testCanCreateOptions()
 
         $this->assertInstanceOf('ZfrPrerender\Options\ModuleOptions', $options);
         $this->assertEquals('http://prerender.herokuapp.com', $options->getPrerenderUrl());
-        $this->assertEquals(array('googlebot', 'yahoo', 'bingbot', 'baiduspider', 'facebookexternalhit'), $options->getCrawlerUserAgents());
+        $this->assertEquals(array('googlebot', 'yahoo', 'bingbot', 'baiduspider', 'facebookexternalhit', 'twitterbot'), $options->getCrawlerUserAgents());
         $this->assertEmpty($options->getWhitelistUrls());
         $this->assertEmpty($options->getBlacklistUrls());
     }