Skip to content

Commit 1236f29

Browse files
miguelbalpardamiguelbalparda
committed
Merge pull request #1185 from nexcess/devel
Devel
2 parents bf24144 + 000362f commit 1236f29

File tree

9 files changed

+112
-32
lines changed

9 files changed

+112
-32
lines changed

‎app/code/community/Nexcessnet/Turpentine/Helper/Cron.php

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,24 @@ public function getCrawlerDebugEnabled() {
145145
return Mage::getStoreConfig('turpentine_varnish/general/crawler_debug');
146146
}
147147

148+
/**
149+
* Get number of urls to crawl per batch
150+
*
151+
* @return int
152+
*/
153+
public function getCrawlerBatchSize() {
154+
return Mage::getStoreConfig('turpentine_varnish/general/crawler_batchsize');
155+
}
156+
157+
/**
158+
* Get time in seconds to wait between url batches
159+
*
160+
* @return int
161+
*/
162+
public function getCrawlerWaitPeriod() {
163+
return Mage::getStoreConfig('turpentine_varnish/general/crawler_batchwait');
164+
}
165+
148166
/**
149167
* Get the list of all URLs
150168
*

‎app/code/community/Nexcessnet/Turpentine/Model/Core/Session.php

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,4 +50,23 @@ public function real_getFormKey()
5050
}
5151
return $this->getData('_form_key');
5252
}
53+
54+
/**
55+
* Creates new Form key
56+
*/
57+
public function renewFormKey()
58+
{
59+
$this->setData('_form_key', Mage::helper('core')->getRandomString(16));
60+
}
61+
62+
/**
63+
* Validates Form key
64+
*
65+
* @param string|null $formKey
66+
* @return bool
67+
*/
68+
public function validateFormKey($formKey)
69+
{
70+
return ($formKey === $this->getFormKey());
71+
}
5372
}

‎app/code/community/Nexcessnet/Turpentine/Model/Observer/Ban.php

Lines changed: 30 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -298,37 +298,39 @@ public function banCacheType($eventObject) {
298298
* @return bool
299299
*/
300300
public function banProductReview($eventObject) {
301-
$patterns = array();
302-
/* @var $review \Mage_Review_Model_Review*/
303-
$review = $eventObject->getObject();
304-
305-
/* @var $productCollection \Mage_Review_Model_Resource_Review_Product_Collection*/
306-
$productCollection = $review->getProductCollection();
307-
308-
$products = $productCollection->addEntityFilter((int) $review->getEntityPkValue())->getItems();
301+
if (Mage::helper('turpentine/varnish')->getVarnishEnabled()) {
302+
$patterns = array();
303+
/* @var $review \Mage_Review_Model_Review*/
304+
$review = $eventObject->getObject();
305+
306+
/* @var $productCollection \Mage_Review_Model_Resource_Review_Product_Collection*/
307+
$productCollection = $review->getProductCollection();
308+
309+
$products = $productCollection->addEntityFilter((int) $review->getEntityPkValue())->getItems();
309310

310-
$productIds = array_unique(array_map(
311-
create_function('$p', 'return $p->getEntityId();'),
312-
$products ));
313-
$patterns[] = sprintf('/review/product/list/id/(?:%s)/category/',
314-
implode('|', array_unique($productIds)));
315-
$patterns[] = sprintf('/review/product/view/id/%d/',
316-
$review->getEntityId());
317-
$productPatterns = array();
318-
foreach ($products as $p) {
319-
$urlKey = $p->getUrlModel()->formatUrlKey($p->getName());
320-
if ($urlKey) {
321-
$productPatterns[] = $urlKey;
311+
$productIds = array_unique(array_map(
312+
create_function('$p', 'return $p->getEntityId();'),
313+
$products ));
314+
$patterns[] = sprintf('/review/product/list/id/(?:%s)/category/',
315+
implode('|', array_unique($productIds)));
316+
$patterns[] = sprintf('/review/product/view/id/%d/',
317+
$review->getEntityId());
318+
$productPatterns = array();
319+
foreach ($products as $p) {
320+
$urlKey = $p->getUrlModel()->formatUrlKey($p->getName());
321+
if ($urlKey) {
322+
$productPatterns[] = $urlKey;
323+
}
322324
}
323-
}
324-
if ( ! empty($productPatterns)) {
325-
$productPatterns = array_unique($productPatterns);
326-
$patterns[] = sprintf('(?:%s)', implode('|', $productPatterns));
327-
}
328-
$urlPattern = implode('|', $patterns);
325+
if ( ! empty($productPatterns)) {
326+
$productPatterns = array_unique($productPatterns);
327+
$patterns[] = sprintf('(?:%s)', implode('|', $productPatterns));
328+
}
329+
$urlPattern = implode('|', $patterns);
329330

330-
$result = $this->_getVarnishAdmin()->flushUrl($urlPattern);
331-
return $this->_checkResult($result);
331+
$result = $this->_getVarnishAdmin()->flushUrl($urlPattern);
332+
return $this->_checkResult($result);
333+
}
332334
}
333335

334336
/**

‎app/code/community/Nexcessnet/Turpentine/Model/Observer/Cron.php

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,11 @@ public function crawlUrls($eventObject) {
4949
if ($maxRunTime === 0) {
5050
$maxRunTime = self::MAX_CRAWL_TIME;
5151
}
52+
53+
$batchSize = $helper->getCrawlerBatchSize();
54+
$timeout = $helper->getCrawlerWaitPeriod();
55+
$crawlCount = 0;
56+
5257
// just in case we have a silly short max_execution_time
5358
$maxRunTime = abs($maxRunTime - self::EXEC_TIME_BUFFER);
5459
while (($helper->getRunTime() < $maxRunTime) &&
@@ -57,6 +62,16 @@ public function crawlUrls($eventObject) {
5762
Mage::helper('turpentine/debug')->logWarn(
5863
'Failed to crawl URL: %s', $url );
5964
}
65+
66+
if ($crawlCount > 0
67+
&& $timeout > 0
68+
&& $batchSize > 0
69+
&& $crawlCount % $batchSize == 0
70+
) {
71+
Mage::helper('turpentine/debug')->logDebug('Crawled '.$crawlCount.' urls, sleeping for '.$timeout.' seconds');
72+
sleep($timeout);
73+
}
74+
$crawlCount++;
6075
}
6176
}
6277
}

‎app/code/community/Nexcessnet/Turpentine/etc/config.xml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@
3535
<fix_product_toolbar>0</fix_product_toolbar>
3636
<crawler_enable>0</crawler_enable>
3737
<crawler_debug>0</crawler_debug>
38+
<crawler_batchsize>0</crawler_batchsize>
39+
<crawler_batchwait>0</crawler_batchwait>
3840
</general>
3941
<logging>
4042
<use_custom_log_file>0</use_custom_log_file>
@@ -59,7 +61,7 @@
5961
<frontend_timeout>300</frontend_timeout>
6062
<admin_timeout>21600</admin_timeout>
6163
<crawlers>127.0.0.1</crawlers>
62-
<crawler_user_agents><![CDATA[ApacheBench/.*,.*Googlebot.*,JoeDog/.*Siege.*,magespeedtest\.com,Nexcessnet_Turpentine/.*]]></crawler_user_agents>
64+
<crawler_user_agents><![CDATA[ApacheBench/.*,.*Googlebot.*,JoeDog/.*Siege.*,magespeedtest\.com,Nexcessnet_Turpentine/.*,.*PTST.*]]></crawler_user_agents>
6365
</backend>
6466
<normalization>
6567
<encoding>1</encoding>

‎app/code/community/Nexcessnet/Turpentine/etc/system.xml

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,30 @@
132132
<show_in_website>0</show_in_website>
133133
<show_in_store>0</show_in_store>
134134
</crawler_debug>
135+
<crawler_batchsize translate="label" module="turpentine">
136+
<label>Crawler Batch Size</label>
137+
<comment>Number of URLs to crawl per batch, when 0 requests will not be batched</comment>
138+
<frontend_type>text</frontend_type>
139+
<sort_order>90</sort_order>
140+
<show_in_default>1</show_in_default>
141+
<show_in_website>0</show_in_website>
142+
<show_in_store>0</show_in_store>
143+
<depends>
144+
<crawler_enable>1</crawler_enable>
145+
</depends>
146+
</crawler_batchsize>
147+
<crawler_batchwait translate="label" module="turpentine">
148+
<label>Crawler Batch Wait</label>
149+
<comment>Time in seconds to wait between batches</comment>
150+
<frontend_type>text</frontend_type>
151+
<sort_order>100</sort_order>
152+
<show_in_default>1</show_in_default>
153+
<show_in_website>0</show_in_website>
154+
<show_in_store>0</show_in_store>
155+
<depends>
156+
<crawler_enable>1</crawler_enable>
157+
</depends>
158+
</crawler_batchwait>
135159
</fields>
136160
</general>
137161
<logging translate="label" module="turpentine">

‎app/code/community/Nexcessnet/Turpentine/misc/version-4.vcl

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ sub vcl_recv {
116116
if (!{{enable_caching}} || req.http.Authorization ||
117117
req.method !~ "^(GET|HEAD|OPTIONS)$" ||
118118
req.http.Cookie ~ "varnish_bypass={{secret_handshake}}") {
119-
return (pipe);
119+
return (pass);
120120
}
121121

122122
if({{send_unmodified_url}}) {

‎build/build_package.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -280,7 +280,7 @@ def _build_dependencies_tag(self, dependencies_tag, connect_dom):
280280
min_tag.text = connect_dom.findtext('depends_php_min')
281281
max_tag = ElementTree.SubElement(php_tag, 'max')
282282
max_tag.text = connect_dom.findtext('depends_php_max')
283-
self._logger.debug('Finished adding dependancies')
283+
self._logger.debug('Finished adding dependencies')
284284
return dependencies_tag
285285

286286
def _get_module_dom(self, ext_name):

‎build/mage-package.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
</email>
2727
</authors>
2828
<depends_php_min>5.2.13</depends_php_min>
29-
<depends_php_max>6.0.0</depends_php_max>
29+
<depends_php_max>7.1.0</depends_php_max>
3030
<depends>
3131
<package>
3232
<name>

0 commit comments

Comments
 (0)