Skip to content

Commit

Permalink
Assert that referer take into account blacklist
Browse files Browse the repository at this point in the history
  • Loading branch information
@bakura10
bakura10 committed Oct 7, 2013
1 parent a0ba24f commit 8453c39
Show file tree
Hide file tree
Showing 3 changed files with 66 additions and 7 deletions.
3 changes: 2 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,8 @@ that happen very early in the MVC process, before the routing is actually done.
#### Blacklist

Blacklist a single url path or multiple url paths. Compares using regex, so be specific when possible. If a blacklist
is supplied, all url's will be prerendered except ones containing a blacklist path.
is supplied, all url's will be pre-rendered except ones containing a blacklist part. Please note that if the referer
is part of the blacklist, it won't be pre-rendered too.

Here is a sample configuration that prerender all URLs *excepting* the ones that contains "/users/":

Expand Down
11 changes: 7 additions & 4 deletions src/ZfrPrerender/Mvc/PrerenderListener.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,10 +146,11 @@ public function shouldPrerenderPage(RequestInterface $request)
return false;
}

// Finally, return false if it is blacklisted
// Finally, return false if it is blacklisted (or the referer)
$referer = $request->getHeader('Referer') ? $request->getHeader('Referer')->getFieldValue() : null;
$blacklistUrls = $this->moduleOptions->getBlacklistUrls();

if (!empty($blacklistUrls) && $this->isBlacklisted($uri, $blacklistUrls)) {
if (!empty($blacklistUrls) && $this->isBlacklisted($uri, $referer, $blacklistUrls)) {
return false;
}

Expand Down Expand Up @@ -199,13 +200,15 @@ protected function isWhitelisted($uri, array $whitelistUrls)
* Check if the request is blacklisted
*
* @param string $uri
* @param string $referer
* @param array $blacklistUrls
* @return bool
*/
protected function isBlacklisted($uri, array $blacklistUrls)
protected function isBlacklisted($uri, $referer, array $blacklistUrls)
{
foreach ($blacklistUrls as $blacklistUrl) {
$match = preg_match('`' . $blacklistUrl . '`i', $uri);
$pattern = '`' . $blacklistUrl . '`i';
$match = preg_match($pattern, $uri) + preg_match($pattern, $referer);

if ($match > 0) {
return true;
Expand Down
59 changes: 57 additions & 2 deletions tests/ZfrPrerenderTest/Mvc/PrerenderListenerTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ public function shouldRenderProvider()
array(
'user_agent' => '',
'uri' => 'http://www.example.com',
'referer' => 'http://google.com',
'ignored_extensions' => array(),
'whitelist' => array(),
'blacklist' => array(),
Expand All @@ -49,6 +50,7 @@ public function shouldRenderProvider()
array(
'user_agent' => 'Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25',
'uri' => 'http://www.example.com',
'referer' => 'http://google.com',
'ignored_extensions' => array(),
'whitelist' => array(),
'blacklist' => array(),
Expand All @@ -58,6 +60,7 @@ public function shouldRenderProvider()
array(
'user_agent' => 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
'uri' => 'http://www.example.com',
'referer' => 'http://google.com',
'ignored_extensions' => array(),
'whitelist' => array(),
'blacklist' => array(),
Expand All @@ -67,6 +70,7 @@ public function shouldRenderProvider()
array(
'user_agent' => 'Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)',
'uri' => 'http://www.example.com',
'referer' => 'http://google.com',
'ignored_extensions' => array(),
'whitelist' => array(),
'blacklist' => array(),
Expand All @@ -76,6 +80,7 @@ public function shouldRenderProvider()
array(
'user_agent' => 'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)',
'uri' => 'http://www.example.com',
'referer' => 'http://google.com',
'ignored_extensions' => array(),
'whitelist' => array(),
'blacklist' => array(),
Expand All @@ -85,6 +90,7 @@ public function shouldRenderProvider()
array(
'user_agent' => 'Baiduspider+(+http://www.baidu.com/search/spider.htm)',
'uri' => 'http://www.example.com',
'referer' => 'http://google.com',
'ignored_extensions' => array(),
'whitelist' => array(),
'blacklist' => array(),
Expand All @@ -94,6 +100,7 @@ public function shouldRenderProvider()
array(
'user_agent' => 'Baiduspider+(+http://www.baidu.com/search/spider.htm)',
'uri' => 'http://www.example.com/screen.css',
'referer' => 'http://google.com',
'ignored_extensions' => array('.jpg', '.css'),
'whitelist' => array(),
'blacklist' => array(),
Expand All @@ -103,6 +110,7 @@ public function shouldRenderProvider()
array(
'user_agent' => 'Baiduspider+(+http://www.baidu.com/search/spider.htm)',
'uri' => 'http://www.example.com',
'referer' => 'http://google.com',
'ignored_extensions' => array(),
'whitelist' => array('example.com'),
'blacklist' => array(),
Expand All @@ -112,6 +120,7 @@ public function shouldRenderProvider()
array(
'user_agent' => 'Baiduspider+(+http://www.baidu.com/search/spider.htm)',
'uri' => 'http://www.example.com/users/michael',
'referer' => 'http://google.com',
'ignored_extensions' => array(),
'whitelist' => array('/users/.*'),
'blacklist' => array(),
Expand All @@ -121,6 +130,7 @@ public function shouldRenderProvider()
array(
'user_agent' => 'Baiduspider+(+http://www.baidu.com/search/spider.htm)',
'uri' => 'http://www.example.com/foo',
'referer' => 'http://google.com',
'ignored_extensions' => array(),
'whitelist' => array('/bar'),
'blacklist' => array(),
Expand All @@ -130,6 +140,7 @@ public function shouldRenderProvider()
array(
'user_agent' => 'Baiduspider+(+http://www.baidu.com/search/spider.htm)',
'uri' => 'http://www.example.com/foo',
'referer' => 'http://google.com',
'ignored_extensions' => array(),
'whitelist' => array(),
'blacklist' => array('/foo'),
Expand All @@ -139,15 +150,57 @@ public function shouldRenderProvider()
array(
'user_agent' => 'Baiduspider+(+http://www.baidu.com/search/spider.htm)',
'uri' => 'http://www.example.com/users/julia',
'referer' => 'http://google.com',
'ignored_extensions' => array(),
'whitelist' => array(),
'blacklist' => array('/users/*'),
'should_prerender' => false
),
// Test a bot crawler that is not blacklisted
array(
'user_agent' => 'Baiduspider+(+http://www.baidu.com/search/spider.htm)',
'uri' => 'http://www.example.com/bar',
'referer' => 'http://google.com',
'ignored_extensions' => array(),
'whitelist' => array(),
'blacklist' => array('/foo'),
'should_prerender' => true
),
// Test a bot crawler and a referer that is blacklisted
array(
'user_agent' => 'Baiduspider+(+http://www.baidu.com/search/spider.htm)',
'uri' => 'http://www.example.com/foo',
'referer' => '/search',
'ignored_extensions' => array(),
'whitelist' => array(),
'blacklist' => array('/search'),
'should_prerender' => false
),
// Test a bot crawler and a referer that is not blacklisted
array(
'user_agent' => 'Baiduspider+(+http://www.baidu.com/search/spider.htm)',
'uri' => 'http://www.example.com/foo',
'referer' => '/search',
'ignored_extensions' => array(),
'whitelist' => array(),
'blacklist' => array(),
'should_prerender' => true
),
// Test a bot crawler and a referer that is not blacklisted by a regex
array(
'user_agent' => 'Baiduspider+(+http://www.baidu.com/search/spider.htm)',
'uri' => 'http://www.example.com/foo',
'referer' => '/profile/search',
'ignored_extensions' => array(),
'whitelist' => array(),
'blacklist' => array('^/search', 'help'),
'should_prerender' => true
),
// Test a bot crawler that combines whitelist and blacklist (1)
array(
'user_agent' => 'Baiduspider+(+http://www.baidu.com/search/spider.htm)',
'uri' => 'http://www.example.com/users/julia',
'referer' => 'http://google.com',
'ignored_extensions' => array(),
'whitelist' => array('/users/*'),
'blacklist' => array('/users/julia'),
Expand All @@ -157,6 +210,7 @@ public function shouldRenderProvider()
array(
'user_agent' => 'Baiduspider+(+http://www.baidu.com/search/spider.htm)',
'uri' => 'http://www.example.com/users/julia',
'referer' => 'http://google.com',
'ignored_extensions' => array(),
'whitelist' => array('/users/*'),
'blacklist' => array('/users/michael'),
Expand All @@ -168,7 +222,7 @@ public function shouldRenderProvider()
/**
* @dataProvider shouldRenderProvider
*/
public function testShouldRender($userAgent, $uri, $ignoredExtensions, $whitelist, $blacklist, $result)
public function testShouldRender($userAgent, $uri, $referer, $ignoredExtensions, $whitelist, $blacklist, $result)
{
$moduleOptions = ServiceManagerFactory::getServiceManager()->get('ZfrPrerender\Options\ModuleOptions');
$moduleOptions->setIgnoredExtensions($ignoredExtensions);
Expand All @@ -177,7 +231,8 @@ public function testShouldRender($userAgent, $uri, $ignoredExtensions, $whitelis

$request = new HttpRequest();
$request->setUri($uri);
$request->getHeaders()->addHeaderLine('User-Agent', $userAgent);
$request->getHeaders()->addHeaderLine('User-Agent', $userAgent)
->addHeaderLine('Referer', $referer);

$listener = new PrerenderListener($moduleOptions);

Expand Down

0 comments on commit 8453c39

Please sign in to comment.