From 94c1a633456c724a003874bf571d1d347e75684f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Gallego?= Date: Mon, 14 Oct 2013 09:17:12 +0200 Subject: [PATCH] Add Twitterbot --- CHANGELOG.md | 4 ++++ README.md | 6 ++++-- config/module.config.php | 3 ++- tests/ZfrPrerenderTest/Factory/ModuleOptionsFactoryTest.php | 2 +- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14de615..44d23ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# 1.1.1 + +* Add Twitterbot User-Agent + # 1.1.0 * Add support for _escaped_fragment_ query param for detecting a bot diff --git a/README.md b/README.md index f322e17..8e2e7cc 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ With this config, here is how ZfrPrerender will proxy the "https://google.com" r #### Crawler user-agents ZfrPrerender decides to pre-render based on the User-Agent string to check if a request comes from a bot or not. By -default, those user agents are registered: `googlebot`, `yahoo`, `bingbot`, `baidu`. +default, those user agents are registered: `googlebot`, `yahoo`, `bingbot`, `baidu` and `twitterbot`. You can add other User-Agent string to evaluate using this sample configuration: @@ -76,7 +76,9 @@ param. You can learn more about this on [Google's website](https://developers.go #### Ignored extensions ZfrPrerender is configured by default to ignore all the requests for resources with those extensions: `.css`, -`.gif`, `.jpeg`, `.jpg`, `.js`, `.png`. Those are never pre-rendered. +`.gif`, `.jpeg`, `.jpg`, `.js`, `.png`, `.less`, `.pdf`, `.doc`, `.txt`, `.zip`, `.mp3`, `.rar`, `.exe`, `.wmv`, +`.doc`, `.avi`, `.ppt`, `.mpg`, `.mpeg`, `.tif`, `.wav`, `.mov`, `.psd`, `.ai`, `.xls`, `.mp4`, `.m4a`, `.swf`, +`.dat`, `.dmg`, `.iso`, `.flv`, `.m4v`, `.torrent`. Those are never pre-rendered. You can add your own extensions using this sample configuration: diff --git a/config/module.config.php b/config/module.config.php index 6ca6aaa..98f3e8b 100644 --- a/config/module.config.php +++ b/config/module.config.php @@ -34,7 +34,8 @@ 'yahoo', 'bingbot', 'baiduspider', - 'facebookexternalhit' + 'facebookexternalhit', + 'twitterbot' ), // Ignored extensions diff --git a/tests/ZfrPrerenderTest/Factory/ModuleOptionsFactoryTest.php b/tests/ZfrPrerenderTest/Factory/ModuleOptionsFactoryTest.php index a84ac81..f89f7c6 100644 --- a/tests/ZfrPrerenderTest/Factory/ModuleOptionsFactoryTest.php +++ b/tests/ZfrPrerenderTest/Factory/ModuleOptionsFactoryTest.php @@ -37,7 +37,7 @@ public function testCanCreateOptions() $this->assertInstanceOf('ZfrPrerender\Options\ModuleOptions', $options); $this->assertEquals('http://prerender.herokuapp.com', $options->getPrerenderUrl()); - $this->assertEquals(array('googlebot', 'yahoo', 'bingbot', 'baiduspider', 'facebookexternalhit'), $options->getCrawlerUserAgents()); + $this->assertEquals(array('googlebot', 'yahoo', 'bingbot', 'baiduspider', 'facebookexternalhit', 'twitterbot'), $options->getCrawlerUserAgents()); $this->assertEmpty($options->getWhitelistUrls()); $this->assertEmpty($options->getBlacklistUrls()); }