From 9938de465d33258363e7d57bcd0a3e9811b9b471 Mon Sep 17 00:00:00 2001 From: Piotr Kwiecinski Date: Thu, 13 Apr 2017 21:50:11 +0100 Subject: [PATCH 1/2] Varnish Vcl generator command --- .../PageCache/Api/VclGeneratorInterface.php | 21 ++ .../Api/VclTemplateLocatorInterface.php | 26 ++ .../Console/Command/GenerateVclCommand.php | 275 ++++++++++++++++++ .../Exception/UnsupportedVarnishVersion.php | 14 + .../PageCache/Model/Varnish/VclGenerator.php | 221 ++++++++++++++ .../Model/Varnish/VclTemplateLocator.php | 103 +++++++ app/code/Magento/PageCache/etc/di.xml | 9 + 7 files changed, 669 insertions(+) create mode 100644 app/code/Magento/PageCache/Api/VclGeneratorInterface.php create mode 100644 app/code/Magento/PageCache/Api/VclTemplateLocatorInterface.php create mode 100644 app/code/Magento/PageCache/Console/Command/GenerateVclCommand.php create mode 100644 app/code/Magento/PageCache/Exception/UnsupportedVarnishVersion.php create mode 100644 app/code/Magento/PageCache/Model/Varnish/VclGenerator.php create mode 100644 app/code/Magento/PageCache/Model/Varnish/VclTemplateLocator.php diff --git a/app/code/Magento/PageCache/Api/VclGeneratorInterface.php b/app/code/Magento/PageCache/Api/VclGeneratorInterface.php new file mode 100644 index 0000000000000..20c23bb784711 --- /dev/null +++ b/app/code/Magento/PageCache/Api/VclGeneratorInterface.php @@ -0,0 +1,21 @@ + 'accessList', + self::BACKEND_PORT_OPTION => 'backendPort', + self::BACKEND_HOST_OPTION => 'backendHost', + self::GRACE_PERIOD_OPTION => 'gracePeriod', + ]; + + /** + * @var ScopeConfigInterface + */ + private $scopeConfig; + + /** + * @var Json + */ + private $serializer; + + /** + * @inheritdoc + */ + protected function configure() + { + $this->setName('varnish:vcl:generate') + ->setDescription('Generates Varnish VCL and echos it to the command line') + ->setDefinition($this->getOptionList()); + } + + /** + * @param VclGeneratorInterfaceFactory $vclGeneratorFactory + * @param WriteFactory $writeFactory + * @param ScopeConfigInterface $scopeConfig + * @param Json $serializer + */ + public function __construct( + VclGeneratorInterfaceFactory $vclGeneratorFactory, + WriteFactory $writeFactory, + ScopeConfigInterface $scopeConfig, + Json $serializer + ) { + parent::__construct(); + $this->writeFactory = $writeFactory; + $this->vclGeneratorFactory = $vclGeneratorFactory; + $this->scopeConfig = $scopeConfig; + $this->serializer = $serializer; + } + + /** + * @inheritdoc + */ + protected function execute(InputInterface $input, OutputInterface $output) + { + $errors = $this->validate($input); + if ($errors) { + foreach ($errors as $error) { + $output->writeln(''.$error.''); + + return Cli::RETURN_FAILURE; + } + } + + try { + $outputFile = $input->getOption(self::OUTPUT_FILE_OPTION); + $varnishVersion = $input->getOption(self::EXPORT_VERSION_OPTION); + $vclParameters = array_merge($this->inputToVclParameters($input), [ + 'sslOffloadedHeader' => $this->getSslOffloadedHeader(), + 'designExceptions' => $this->getDesignExceptions(), + ]); + $vclGenerator = $this->vclGeneratorFactory->create($vclParameters); + $vcl = $vclGenerator->generateVcl($varnishVersion); + + if ($outputFile) { + $writer = $this->writeFactory->create($outputFile, DriverPool::FILE, 'w+'); + $writer->write($vcl); + $writer->close(); + } else { + $output->writeln($vcl); + } + + return Cli::RETURN_SUCCESS; + } catch (\Exception $e) { + $output->writeln(''.$e->getMessage().''); + if ($output->getVerbosity() >= OutputInterface::VERBOSITY_VERBOSE) { + $output->writeln($e->getTraceAsString()); + } + + return Cli::RETURN_FAILURE; + } + } + + /** + * Get list of options for the command + * + * @return InputOption[] + */ + private function getOptionList() + { + return [ + new InputOption( + self::ACCESS_LIST_OPTION, + null, + InputOption::VALUE_REQUIRED | InputOption::VALUE_IS_ARRAY, + 'IPs access list that can purge Varnish', + ['localhost'] + ), + new InputOption( + self::BACKEND_HOST_OPTION, + null, + InputOption::VALUE_REQUIRED, + 'Backend host for configuration', + 'localhost' + ), + new InputOption( + self::BACKEND_PORT_OPTION, + null, + InputOption::VALUE_REQUIRED, + 'Backend post for configuration', + 8080 + ), + new InputOption( + self::EXPORT_VERSION_OPTION, + null, + InputOption::VALUE_REQUIRED, + 'Varnish configuration version to export', + VclTemplateLocator::VARNISH_SUPPORTED_VERSION_4 + ), + new InputOption( + self::GRACE_PERIOD_OPTION, + null, + InputOption::VALUE_REQUIRED, + 'Grace period in seconds', + 300 + ), + new InputOption( + self::OUTPUT_FILE_OPTION, + null, + InputOption::VALUE_REQUIRED, + 'Save output to target file' + ), + ]; + } + + /** + * @param InputInterface $input + * @return array + */ + private function inputToVclParameters(InputInterface $input) + { + $parameters = []; + + foreach ($this->inputToVclMap as $inputKey => $vclKey) { + $parameters[$vclKey] = $input->getOption($inputKey); + } + + return $parameters; + } + + /** + * Input validation + * + * @param InputInterface $input + * @return array + */ + private function validate(InputInterface $input) + { + $errors = []; + + if ($input->hasOption(self::BACKEND_PORT_OPTION) + && ($input->getOption(self::BACKEND_PORT_OPTION) < 0 + || $input->getOption(self::BACKEND_PORT_OPTION) > 65535) + ) { + $errors[] = 'Invalid backend port value'; + } + + if ($input->hasOption(self::GRACE_PERIOD_OPTION) + && $input->getOption(self::GRACE_PERIOD_OPTION) < 0 + ) { + $errors[] = 'Grace period can\'t be lower than 0'; + } + + return $errors; + } + + /** + * Get ssl Offloaded header + * + * @return mixed + */ + private function getSslOffloadedHeader() + { + return $this->scopeConfig->getValue(Request::XML_PATH_OFFLOADER_HEADER); + } + + /** + * Get design exceptions + * + * @return array + */ + private function getDesignExceptions() + { + $expressions = $this->scopeConfig->getValue( + Config::XML_VARNISH_PAGECACHE_DESIGN_THEME_REGEX, + ScopeInterface::SCOPE_STORE + ); + + return $expressions ? $this->serializer->unserialize($expressions) : []; + } +} \ No newline at end of file diff --git a/app/code/Magento/PageCache/Exception/UnsupportedVarnishVersion.php b/app/code/Magento/PageCache/Exception/UnsupportedVarnishVersion.php new file mode 100644 index 0000000000000..19635671d2fea --- /dev/null +++ b/app/code/Magento/PageCache/Exception/UnsupportedVarnishVersion.php @@ -0,0 +1,14 @@ +backendHost = $backendHost; + $this->backendPort = $backendPort; + $this->accessList = $accessList; + $this->gracePeriod = $gracePeriod; + $this->vclTemplateLocator = $vclTemplateLocator; + $this->sslOffloadedHeader = $sslOffloadedHeader; + $this->designExceptions = $designExceptions; + } + + /** + * Return generated varnish.vcl configuration file + * + * @param int $version + * @return string + * @api + */ + public function generateVcl($version) + { + $template = $this->vclTemplateLocator->getTemplate($version); + return strtr($template, $this->getReplacements()); + } + + /** + * Prepare data for VCL config + * + * @return array + */ + private function getReplacements() + { + return [ + '/* {{ host }} */' => $this->getBackendHost(), + '/* {{ port }} */' => $this->getBackendPort(), + '/* {{ ips }} */' => $this->getTransformedAccessList(), + '/* {{ design_exceptions_code }} */' => $this->getRegexForDesignExceptions(), + // http headers get transformed by php `X-Forwarded-Proto: https` + // becomes $SERVER['HTTP_X_FORWARDED_PROTO'] = 'https' + // Apache and Nginx drop all headers with underlines by default. + '/* {{ ssl_offloaded_header }} */' => str_replace('_', '-', $this->getSslOffloadedHeader()), + '/* {{ grace_period }} */' => $this->getGracePeriod(), + ]; + } + + /** + * Get regexs for design exceptions + * Different browser user-agents may use different themes + * Varnish supports regex with internal modifiers only so + * we have to convert "/pattern/iU" into "(?Ui)pattern" + * + * @return string + */ + private function getRegexForDesignExceptions() + { + $result = ''; + $tpl = "%s (req.http.user-agent ~ \"%s\") {\n"." hash_data(\"%s\");\n"." }"; + + $expressions = $this->getDesignExceptions(); + + if ($expressions) { + $rules = array_values($expressions); + foreach ($rules as $i => $rule) { + if (preg_match('/^[\W]{1}(.*)[\W]{1}(\w+)?$/', $rule['regexp'], $matches)) { + if (!empty($matches[2])) { + $pattern = sprintf("(?%s)%s", $matches[2], $matches[1]); + } else { + $pattern = $matches[1]; + } + $if = $i == 0 ? 'if' : ' elsif'; + $result .= sprintf($tpl, $if, $pattern, $rule['value']); + } + } + } + + return $result; + } + + /** + * Get IPs access list that can purge Varnish configuration for config file generation + * and transform it to appropriate view + * + * acl purge{ + * "127.0.0.1"; + * "127.0.0.2"; + * + * @return string + */ + private function getTransformedAccessList() + { + $tpl = " \"%s\";"; + $result = array_reduce($this->getAccessList(), function ($ips, $ip) use ($tpl) { + return $ips.sprintf($tpl, trim($ip))."\n"; + }, ''); + + return $result; + } + + /** + * Get access list + * + * @return array + */ + private function getAccessList() + { + return $this->accessList; + } + + /** + * Get backend host + * + * @return string + */ + private function getBackendHost() + { + return $this->backendHost; + } + + /** + * Get backend post + * + * @return int + */ + private function getBackendPort() + { + return $this->backendPort; + } + + /** + * Get grace period + * + * @return int + */ + private function getGracePeriod() + { + return $this->gracePeriod; + } + + /** + * Get SSL Offloaded Header + * + * @return string + */ + private function getSslOffloadedHeader() + { + return $this->sslOffloadedHeader; + } + + /** + * @return array + */ + private function getDesignExceptions() + { + return $this->designExceptions; + } +} \ No newline at end of file diff --git a/app/code/Magento/PageCache/Model/Varnish/VclTemplateLocator.php b/app/code/Magento/PageCache/Model/Varnish/VclTemplateLocator.php new file mode 100644 index 0000000000000..1d37c23deafc1 --- /dev/null +++ b/app/code/Magento/PageCache/Model/Varnish/VclTemplateLocator.php @@ -0,0 +1,103 @@ + self::VARNISH_4_CONFIGURATION_PATH, + self::VARNISH_SUPPORTED_VERSION_5 => self::VARNISH_5_CONFIGURATION_PATH, + ]; + + /** + * @var Reader + */ + private $reader; + + /** + * @var ReadFactory + */ + private $readFactory; + + /** + * @var ScopeConfigInterface + */ + private $scopeConfig; + + /** + * VclTemplateLocator constructor. + * + * @param Reader $reader + * @param ReadFactory $readFactory + * @param ScopeConfigInterface $scopeConfig + */ + public function __construct(Reader $reader, ReadFactory $readFactory, ScopeConfigInterface $scopeConfig) + { + $this->reader = $reader; + $this->readFactory = $readFactory; + $this->scopeConfig = $scopeConfig; + } + + /** + * {@inheritdoc} + */ + public function getTemplate($version) + { + $moduleEtcPath = $this->reader->getModuleDir(Dir::MODULE_ETC_DIR, 'Magento_PageCache'); + $configFilePath = $moduleEtcPath . '/' . $this->scopeConfig->getValue($this->getVclTemplatePath($version)); + $directoryRead = $this->readFactory->create($moduleEtcPath); + $configFilePath = $directoryRead->getRelativePath($configFilePath); + $template = $directoryRead->readFile($configFilePath); + return $template; + } + + /** + * Get Vcl template path + * + * @param int $version Varnish version + * @return string + * @throws UnsupportedVarnishVersion + */ + private function getVclTemplatePath($version) + { + if (!isset($this->supportedVarnishVersions[$version])) { + throw new UnsupportedVarnishVersion(__('Unsupported varnish version')); + } + + return $this->supportedVarnishVersions[$version]; + } +} \ No newline at end of file diff --git a/app/code/Magento/PageCache/etc/di.xml b/app/code/Magento/PageCache/etc/di.xml index 567baa493be50..f890619ac8342 100644 --- a/app/code/Magento/PageCache/etc/di.xml +++ b/app/code/Magento/PageCache/etc/di.xml @@ -25,4 +25,13 @@ + + + + Magento\PageCache\Console\Command\GenerateVclCommand + + + + + From b95d230d627827cb7724dd55f9f3f83d8f4bb92f Mon Sep 17 00:00:00 2001 From: Piotr Kwiecinski Date: Tue, 18 Apr 2017 00:33:04 +0100 Subject: [PATCH 2/2] Use vcl generator logic in admin ui, deprecate getVclFile --- app/code/Magento/PageCache/Model/Config.php | 34 +++++++++++++++++---- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/app/code/Magento/PageCache/Model/Config.php b/app/code/Magento/PageCache/Model/Config.php index bb9e7af1357ab..9b169acc3c911 100644 --- a/app/code/Magento/PageCache/Model/Config.php +++ b/app/code/Magento/PageCache/Model/Config.php @@ -9,6 +9,7 @@ use Magento\Framework\Filesystem; use Magento\Framework\Module\Dir; use Magento\Framework\Serialize\Serializer\Json; +use Magento\PageCache\Api\VclGeneratorInterfaceFactory; /** * Model is responsible for replacing default vcl template @@ -81,12 +82,17 @@ class Config * @var Json */ private $serializer; + /** + * @var VclGeneratorInterfaceFactory + */ + private $vclGeneratorFactory; /** * @param Filesystem\Directory\ReadFactory $readFactory * @param \Magento\Framework\App\Config\ScopeConfigInterface $scopeConfig * @param \Magento\Framework\App\Cache\StateInterface $cacheState * @param Dir\Reader $reader + * @param VclGeneratorInterfaceFactory $vclGeneratorFactory * @param Json|null $serializer */ public function __construct( @@ -94,6 +100,7 @@ public function __construct( \Magento\Framework\App\Config\ScopeConfigInterface $scopeConfig, \Magento\Framework\App\Cache\StateInterface $cacheState, \Magento\Framework\Module\Dir\Reader $reader, + VclGeneratorInterfaceFactory $vclGeneratorFactory, Json $serializer = null ) { $this->readFactory = $readFactory; @@ -101,6 +108,7 @@ public function __construct( $this->_cacheState = $cacheState; $this->reader = $reader; $this->serializer = $serializer ?: ObjectManager::getInstance()->get(Json::class); + $this->vclGeneratorFactory = $vclGeneratorFactory; } /** @@ -130,16 +138,30 @@ public function getTtl() * * @param string $vclTemplatePath * @return string + * @deprecated * @api */ public function getVclFile($vclTemplatePath) { - $moduleEtcPath = $this->reader->getModuleDir(Dir::MODULE_ETC_DIR, 'Magento_PageCache'); - $configFilePath = $moduleEtcPath . '/' . $this->_scopeConfig->getValue($vclTemplatePath); - $directoryRead = $this->readFactory->create($moduleEtcPath); - $configFilePath = $directoryRead->getRelativePath($configFilePath); - $data = $directoryRead->readFile($configFilePath); - return strtr($data, $this->_getReplacements()); + $accessList = $this->_scopeConfig->getValue(self::XML_VARNISH_PAGECACHE_ACCESS_LIST); + $designExceptions = $this->_scopeConfig->getValue( + self::XML_VARNISH_PAGECACHE_DESIGN_THEME_REGEX, + \Magento\Store\Model\ScopeInterface::SCOPE_STORE + ); + + $version = $vclTemplatePath === self::VARNISH_5_CONFIGURATION_PATH ? 5 : 4; + $sslOffloadedHeader = $this->_scopeConfig->getValue( + \Magento\Framework\HTTP\PhpEnvironment\Request::XML_PATH_OFFLOADER_HEADER + ); + $vclGenerator = $this->vclGeneratorFactory->create([ + 'backendHost' => $this->_scopeConfig->getValue(self::XML_VARNISH_PAGECACHE_BACKEND_HOST), + 'backendPort' => $this->_scopeConfig->getValue(self::XML_VARNISH_PAGECACHE_BACKEND_PORT), + 'accessList' => $accessList ? explode(',',$accessList) : [], + 'designExceptions' => $designExceptions ? $this->serializer->unserialize($designExceptions) : [], + 'sslOffloadedHeader' => $sslOffloadedHeader, + 'gracePeriod' => $this->_scopeConfig->getValue(self::XML_VARNISH_PAGECACHE_GRACE_PERIOD) + ]); + return $vclGenerator->generateVcl($version); } /**