From cf3f078e0062504d8837c3013afd0bdbb37ce17e Mon Sep 17 00:00:00 2001 From: David WATTIER Date: Thu, 16 Mar 2017 19:28:06 +0100 Subject: [PATCH 1/4] #8815 * send multiple small purge request instead of one big request to avoid reaching the varnish http_req_hdr_len threshold. --- .../Observer/InvalidateVarnishObserver.php | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/app/code/Magento/CacheInvalidate/Observer/InvalidateVarnishObserver.php b/app/code/Magento/CacheInvalidate/Observer/InvalidateVarnishObserver.php index 3527d4f6cdf4e..acfd321ea0950 100644 --- a/app/code/Magento/CacheInvalidate/Observer/InvalidateVarnishObserver.php +++ b/app/code/Magento/CacheInvalidate/Observer/InvalidateVarnishObserver.php @@ -21,6 +21,18 @@ class InvalidateVarnishObserver implements ObserverInterface */ protected $purgeCache; + /** + * Batch size of the purge request. + * + * Based on default Varnish 4 http_req_hdr_len size minus a 512 bytes margin for method, + * header name, line feeds etc. + * + * @see http://www.varnish-cache.org/docs/4.0/reference/varnishd.html#http-req-hdr-len + * + * @var int + */ + private $requestSize = 7680; + /** * Invalidation tags resolver * @@ -44,6 +56,8 @@ public function __construct( * If Varnish caching is enabled it collects array of tags * of incoming object and asks to clean cache. * + * For scaling reason the purge request is chopped down to fix batch size. + * * @param \Magento\Framework\Event\Observer $observer * @return void */ @@ -56,10 +70,21 @@ public function execute(\Magento\Framework\Event\Observer $observer) if ($this->config->getType() == \Magento\PageCache\Model\Config::VARNISH && $this->config->isEnabled()) { $bareTags = $this->getTagResolver()->getTags($object); + $tagsBatchSize = 0; $tags = []; - $pattern = "((^|,)%s(,|$))"; + $pattern = '((^|,)%s(,|$))'; foreach ($bareTags as $tag) { - $tags[] = sprintf($pattern, $tag); + $formattedTag = sprintf($pattern, $tag); + + // Send request if batch size is reached and add the implode with pipe to the computation + if ($tagsBatchSize + strlen($formattedTag) > $this->requestSize - count($tags) - 1) { + $this->purgeCache->sendPurgeRequest(implode('|', array_unique($tags))); + $tags = []; + $tagsBatchSize = 0; + } + + $tagsBatchSize += strlen($formattedTag); + $tags[] = $formattedTag; } if (!empty($tags)) { $this->purgeCache->sendPurgeRequest(implode('|', array_unique($tags))); From 677356e1014be1d6577c4ea041f9b535fb800c4e Mon Sep 17 00:00:00 2001 From: slopukhov Date: Thu, 11 Oct 2018 15:29:11 +0300 Subject: [PATCH 2/4] MAGETWO-95275: Varnish "Connection reset by peer" error when large catalog is reindexed on schedule --- .../CacheInvalidate/Model/PurgeCache.php | 68 +++++++++++++++++-- .../Observer/InvalidateVarnishObserver.php | 29 +------- 2 files changed, 66 insertions(+), 31 deletions(-) diff --git a/app/code/Magento/CacheInvalidate/Model/PurgeCache.php b/app/code/Magento/CacheInvalidate/Model/PurgeCache.php index 8acf170d43cfb..57e29d11495a2 100644 --- a/app/code/Magento/CacheInvalidate/Model/PurgeCache.php +++ b/app/code/Magento/CacheInvalidate/Model/PurgeCache.php @@ -26,6 +26,18 @@ class PurgeCache */ private $logger; + /** + * Batch size of the purge request. + * + * Based on default Varnish 4 http_req_hdr_len size minus a 512 bytes margin for method, + * header name, line feeds etc. + * + * @see https://varnish-cache.org/docs/4.1/reference/varnishd.html + * + * @var int + */ + private $requestSize = 7680; + /** * Constructor * @@ -52,10 +64,59 @@ public function __construct( */ public function sendPurgeRequest($tagsPattern) { + $successful = true; $socketAdapter = $this->socketAdapterFactory->create(); $servers = $this->cacheServer->getUris(); - $headers = [self::HEADER_X_MAGENTO_TAGS_PATTERN => $tagsPattern]; $socketAdapter->setOptions(['timeout' => 10]); + + $formattedTagsChunks = $this->splitTags($tagsPattern); + foreach ($formattedTagsChunks as $formattedTagsChunk) { + if (!$this->sendPurgeRequestToServers($socketAdapter, $servers, $formattedTagsChunk)) { + $successful = false; + } + } + + return $successful; + } + + /** + * Split tags by batches + * + * @param string $tagsPattern + * @return \Generator + */ + private function splitTags($tagsPattern) + { + $tagsBatchSize = 0; + $formattedTagsChunk = []; + $formattedTags = explode('|', $tagsPattern); + foreach ($formattedTags as $formattedTag) { + if ($tagsBatchSize + strlen($formattedTag) > $this->requestSize - count($formattedTagsChunk) - 1) { + yield implode('|', array_unique($formattedTagsChunk)); + $formattedTagsChunk = []; + $tagsBatchSize = 0; + } + + $tagsBatchSize += strlen($formattedTag); + $formattedTagsChunk[] = $formattedTag; + } + if (!empty($formattedTagsChunk)) { + yield implode('|', array_unique($formattedTagsChunk)); + } + } + + /** + * Send curl purge request to servers + * to invalidate cache by tags pattern + * + * @param \Zend\Http\Client\Adapter\Socket $socketAdapter + * @param \Zend\Uri\Uri[] $servers + * @param string $formattedTagsChunk + * @return bool Return true if successful; otherwise return false + */ + private function sendPurgeRequestToServers($socketAdapter, $servers, $formattedTagsChunk) + { + $headers = [self::HEADER_X_MAGENTO_TAGS_PATTERN => $formattedTagsChunk]; foreach ($servers as $server) { $headers['Host'] = $server->getHost(); try { @@ -69,12 +130,11 @@ public function sendPurgeRequest($tagsPattern) $socketAdapter->read(); $socketAdapter->close(); } catch (\Exception $e) { - $this->logger->critical($e->getMessage(), compact('server', 'tagsPattern')); + $this->logger->critical($e->getMessage(), compact('server', 'formattedTagsChunk')); return false; } } - - $this->logger->execute(compact('servers', 'tagsPattern')); + $this->logger->execute(compact('servers', 'formattedTagsChunk')); return true; } } diff --git a/app/code/Magento/CacheInvalidate/Observer/InvalidateVarnishObserver.php b/app/code/Magento/CacheInvalidate/Observer/InvalidateVarnishObserver.php index acfd321ea0950..3527d4f6cdf4e 100644 --- a/app/code/Magento/CacheInvalidate/Observer/InvalidateVarnishObserver.php +++ b/app/code/Magento/CacheInvalidate/Observer/InvalidateVarnishObserver.php @@ -21,18 +21,6 @@ class InvalidateVarnishObserver implements ObserverInterface */ protected $purgeCache; - /** - * Batch size of the purge request. - * - * Based on default Varnish 4 http_req_hdr_len size minus a 512 bytes margin for method, - * header name, line feeds etc. - * - * @see http://www.varnish-cache.org/docs/4.0/reference/varnishd.html#http-req-hdr-len - * - * @var int - */ - private $requestSize = 7680; - /** * Invalidation tags resolver * @@ -56,8 +44,6 @@ public function __construct( * If Varnish caching is enabled it collects array of tags * of incoming object and asks to clean cache. * - * For scaling reason the purge request is chopped down to fix batch size. - * * @param \Magento\Framework\Event\Observer $observer * @return void */ @@ -70,21 +56,10 @@ public function execute(\Magento\Framework\Event\Observer $observer) if ($this->config->getType() == \Magento\PageCache\Model\Config::VARNISH && $this->config->isEnabled()) { $bareTags = $this->getTagResolver()->getTags($object); - $tagsBatchSize = 0; $tags = []; - $pattern = '((^|,)%s(,|$))'; + $pattern = "((^|,)%s(,|$))"; foreach ($bareTags as $tag) { - $formattedTag = sprintf($pattern, $tag); - - // Send request if batch size is reached and add the implode with pipe to the computation - if ($tagsBatchSize + strlen($formattedTag) > $this->requestSize - count($tags) - 1) { - $this->purgeCache->sendPurgeRequest(implode('|', array_unique($tags))); - $tags = []; - $tagsBatchSize = 0; - } - - $tagsBatchSize += strlen($formattedTag); - $tags[] = $formattedTag; + $tags[] = sprintf($pattern, $tag); } if (!empty($tags)) { $this->purgeCache->sendPurgeRequest(implode('|', array_unique($tags))); From 6380421a356b7f85fde10c7596f515467ab1f138 Mon Sep 17 00:00:00 2001 From: Vitaliy Honcharenko Date: Tue, 2 Oct 2018 16:09:57 +0300 Subject: [PATCH 3/4] MAGETWO-95435: [2.3] Profile changes for performance-toolkit --- setup/performance-toolkit/config/description.xml | 12 ++++++------ .../performance-toolkit/profiles/ce/extra_large.xml | 6 +++--- setup/performance-toolkit/profiles/ce/large.xml | 6 +++--- setup/performance-toolkit/profiles/ce/medium.xml | 6 +++--- .../performance-toolkit/profiles/ce/medium_msite.xml | 6 +++--- setup/performance-toolkit/profiles/ce/small.xml | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/setup/performance-toolkit/config/description.xml b/setup/performance-toolkit/config/description.xml index ae7e1e343b1da..06fa6a2618f68 100644 --- a/setup/performance-toolkit/config/description.xml +++ b/setup/performance-toolkit/config/description.xml @@ -7,16 +7,16 @@ --> - 4 - 10 + 7 + 7 - 10 - 15 + 12 + 12 - 5 - 7 + 6 + 6 diff --git a/setup/performance-toolkit/profiles/ce/extra_large.xml b/setup/performance-toolkit/profiles/ce/extra_large.xml index 44b3512090b12..390bf7fb12003 100644 --- a/setup/performance-toolkit/profiles/ce/extra_large.xml +++ b/setup/performance-toolkit/profiles/ce/extra_large.xml @@ -40,9 +40,9 @@ 20 2 - 200 - 50 - 2 + 100 + 30 + 15 true 2 diff --git a/setup/performance-toolkit/profiles/ce/large.xml b/setup/performance-toolkit/profiles/ce/large.xml index 606c854d27adc..ed91b22930af5 100644 --- a/setup/performance-toolkit/profiles/ce/large.xml +++ b/setup/performance-toolkit/profiles/ce/large.xml @@ -40,9 +40,9 @@ 20 2 - 200 - 50 - 2 + 50 + 20 + 15 true 2 diff --git a/setup/performance-toolkit/profiles/ce/medium.xml b/setup/performance-toolkit/profiles/ce/medium.xml index 4f735ae4ac2d4..f01eabb7898f3 100644 --- a/setup/performance-toolkit/profiles/ce/medium.xml +++ b/setup/performance-toolkit/profiles/ce/medium.xml @@ -40,9 +40,9 @@ 20 2 - 100 - 50 - 2 + 30 + 10 + 8 true 2 diff --git a/setup/performance-toolkit/profiles/ce/medium_msite.xml b/setup/performance-toolkit/profiles/ce/medium_msite.xml index 24d51d170fbc2..a57fcad0779fe 100644 --- a/setup/performance-toolkit/profiles/ce/medium_msite.xml +++ b/setup/performance-toolkit/profiles/ce/medium_msite.xml @@ -46,9 +46,9 @@ 20 2 - 100 - 50 - 2 + 30 + 10 + 8 true 2 diff --git a/setup/performance-toolkit/profiles/ce/small.xml b/setup/performance-toolkit/profiles/ce/small.xml index 270828c2a2c9d..60ae901d8f5e0 100644 --- a/setup/performance-toolkit/profiles/ce/small.xml +++ b/setup/performance-toolkit/profiles/ce/small.xml @@ -41,8 +41,8 @@ 2 10 - 10 - 2 + 5 + 5 true 2 From af3de710de5d11d12548b8342f6978cc435ac9c0 Mon Sep 17 00:00:00 2001 From: slopukhov Date: Wed, 17 Oct 2018 09:01:06 +0300 Subject: [PATCH 4/4] MAGETWO-95275: Varnish "Connection reset by peer" error when large catalog is reindexed on schedule --- app/code/Magento/CacheInvalidate/Model/PurgeCache.php | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/app/code/Magento/CacheInvalidate/Model/PurgeCache.php b/app/code/Magento/CacheInvalidate/Model/PurgeCache.php index 57e29d11495a2..727e18280d76f 100644 --- a/app/code/Magento/CacheInvalidate/Model/PurgeCache.php +++ b/app/code/Magento/CacheInvalidate/Model/PurgeCache.php @@ -7,6 +7,9 @@ use Magento\Framework\Cache\InvalidateLogger; +/** + * Class PurgeCache + */ class PurgeCache { const HEADER_X_MAGENTO_TAGS_PATTERN = 'X-Magento-Tags-Pattern'; @@ -56,8 +59,7 @@ public function __construct( } /** - * Send curl purge request - * to invalidate cache by tags pattern + * Send curl purge request to invalidate cache by tags pattern * * @param string $tagsPattern * @return bool Return true if successful; otherwise return false @@ -106,8 +108,7 @@ private function splitTags($tagsPattern) } /** - * Send curl purge request to servers - * to invalidate cache by tags pattern + * Send curl purge request to servers to invalidate cache by tags pattern * * @param \Zend\Http\Client\Adapter\Socket $socketAdapter * @param \Zend\Uri\Uri[] $servers