Skip to content

Commit

Permalink
Merge pull request #389 from magento-performance/MAGETWO-52660
Browse files Browse the repository at this point in the history 
MAGETWO-52660: [Performance] Improve performance of static assets deployment
  • Loading branch information
@kandy
kandy authored Sep 15, 2016
2 parents 152dbdc + ddbdfff commit 7800e37
Show file tree
Hide file tree
Showing 39 changed files with 2,506 additions and 651 deletions.
155 changes: 41 additions & 114 deletions app/code/Magento/Deploy/Console/Command/DeployStaticContentCommand.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,10 @@
use Magento\Framework\App\ObjectManagerFactory;
use Magento\Framework\ObjectManagerInterface;
use Magento\Framework\Validator\Locale;
use Magento\Framework\Console\Cli;
use Magento\Deploy\Model\ProcessManager;
use Magento\Deploy\Model\Process;
use Magento\Framework\Exception\LocalizedException;
use Magento\Framework\App\State;
use Magento\Deploy\Console\Command\DeployStaticOptionsInterface as Options;
use Magento\Deploy\Model\DeployManager;

/**
* Deploy static content command
Expand Down Expand Up @@ -121,109 +119,116 @@ protected function configure()
Options::FORCE_RUN,
'-f',
InputOption::VALUE_NONE,
'If specified, then run files will be deployed in any mode.'
'Deploy files in any mode.'
),
new InputOption(
Options::NO_JAVASCRIPT,
null,
InputOption::VALUE_NONE,
'If specified, no JavaScript will be deployed.'
'Do not deploy JavaScript files'
),
new InputOption(
Options::NO_CSS,
null,
InputOption::VALUE_NONE,
'If specified, no CSS will be deployed.'
'Do not deploy CSS files.'
),
new InputOption(
Options::NO_LESS,
null,
InputOption::VALUE_NONE,
'If specified, no LESS will be deployed.'
'Do not deploy LESS files.'
),
new InputOption(
Options::NO_IMAGES,
null,
InputOption::VALUE_NONE,
'If specified, no images will be deployed.'
'Do not deploy images.'
),
new InputOption(
Options::NO_FONTS,
null,
InputOption::VALUE_NONE,
'If specified, no font files will be deployed.'
'Do not deploy font files.'
),
new InputOption(
Options::NO_HTML,
null,
InputOption::VALUE_NONE,
'If specified, no html files will be deployed.'
'Do not deploy HTML files.'
),
new InputOption(
Options::NO_MISC,
null,
InputOption::VALUE_NONE,
'If specified, no miscellaneous files will be deployed.'
'Do not deploy other types of files (.md, .jbf, .csv, etc...).'
),
new InputOption(
Options::NO_HTML_MINIFY,
null,
InputOption::VALUE_NONE,
'If specified, html will not be minified.'
'Do not minify HTML files.'
),
new InputOption(
Options::THEME,
'-t',
InputOption::VALUE_IS_ARRAY | InputOption::VALUE_OPTIONAL,
'If specified, just specific theme(s) will be actually deployed.',
'Generate static view files for only the specified themes.',
['all']
),
new InputOption(
Options::EXCLUDE_THEME,
null,
InputOption::VALUE_IS_ARRAY | InputOption::VALUE_OPTIONAL,
'If specified, exclude specific theme(s) from deployment.',
'Do not generate files for the specified themes.',
['none']
),
new InputOption(
Options::LANGUAGE,
'-l',
InputOption::VALUE_IS_ARRAY | InputOption::VALUE_OPTIONAL,
'List of languages you want the tool populate files for.',
'Generate files only for the specified languages.',
['all']
),
new InputOption(
Options::EXCLUDE_LANGUAGE,
null,
InputOption::VALUE_IS_ARRAY | InputOption::VALUE_OPTIONAL,
'List of langiages you do not want the tool populate files for.',
'Do not generate files for the specified languages.',
['none']
),
new InputOption(
Options::AREA,
'-a',
InputOption::VALUE_IS_ARRAY | InputOption::VALUE_OPTIONAL,
'List of areas you want the tool populate files for.',
'Generate files only for the specified areas.',
['all']
),
new InputOption(
Options::EXCLUDE_AREA,
null,
InputOption::VALUE_IS_ARRAY | InputOption::VALUE_OPTIONAL,
'List of areas you do not want the tool populate files for.',
'Do not generate files for the specified areas.',
['none']
),
new InputOption(
Options::JOBS_AMOUNT,
'-j',
InputOption::VALUE_OPTIONAL,
'Amount of jobs to which script can be paralleled.',
'Enable parallel processing using the specified number of jobs.',
self::DEFAULT_JOBS_AMOUNT
),
new InputOption(
Options::SYMLINK_LOCALE,
null,
InputOption::VALUE_NONE,
'Create symlinks for the files of those locales, which are passed for deployment, '
. 'but have no customizations'
),

This comment has been minimized.

Sign in to view

Copy link
@hostep

hostep Sep 15, 2016

Contributor

Woohoow! 😄
new InputArgument(
self::LANGUAGES_ARGUMENT,
InputArgument::IS_ARRAY,
'List of languages you want the tool populate files for.'
'Space-separated list of ISO-636 language codes for which to output static view files.'
),
]);

Expand Down Expand Up @@ -374,8 +379,11 @@ protected function execute(InputInterface $input, OutputInterface $output)
if (!$input->getOption(Options::FORCE_RUN) && $this->getAppState()->getMode() !== State::MODE_PRODUCTION) {
throw new LocalizedException(
__(
"Deploy static content is applicable only for production mode.\n"
. "Please use command 'bin/magento deploy:mode:set production' for set up production mode."
'NOTE: Manual static content deployment is not required in "default" and "developer" modes.'
. PHP_EOL . 'In "default" and "developer" modes static contents are being deployed '
. 'automatically on demand.'
. PHP_EOL . 'If you still want to deploy in these modes, use -f option: '
. "'bin/magento setup:static-content:deploy -f'"
)
);
}
Expand All @@ -390,20 +398,24 @@ protected function execute(InputInterface $input, OutputInterface $output)
$output->writeln("Requested areas: " . implode(', ', array_keys($deployableAreaThemeMap)));
$output->writeln("Requested themes: " . implode(', ', $requestedThemes));

$deployer = $this->objectManager->create(
\Magento\Deploy\Model\Deployer::class,
/** @var $deployManager DeployManager */
$deployManager = $this->objectManager->create(
DeployManager::class,
[
'filesUtil' => $filesUtil,
'output' => $output,
'options' => $this->input->getOptions(),
]
);

if ($this->isCanBeParalleled()) {
return $this->runProcessesInParallel($deployer, $deployableAreaThemeMap, $deployableLanguages);
} else {
return $this->deploy($deployer, $deployableLanguages, $deployableAreaThemeMap);
foreach ($deployableAreaThemeMap as $area => $themes) {
foreach ($deployableLanguages as $locale) {
foreach ($themes as $themePath) {
$deployManager->addPack($area, $themePath, $locale);
}
}
}

return $deployManager->deploy();
}

/**
Expand Down Expand Up @@ -464,89 +476,4 @@ private function prepareDeployableEntities($filesUtil)

return [$deployableLanguages, $deployableAreaThemeMap, $requestedThemes];
}

/**
* @param \Magento\Deploy\Model\Deployer $deployer
* @param array $deployableLanguages
* @param array $deployableAreaThemeMap
* @return int
*/
private function deploy($deployer, $deployableLanguages, $deployableAreaThemeMap)
{
return $deployer->deploy(
$this->objectManagerFactory,
$deployableLanguages,
$deployableAreaThemeMap
);
}

/**
* @param \Magento\Deploy\Model\Deployer $deployer
* @param array $deployableAreaThemeMap
* @param array $deployableLanguages
* @return int
* @SuppressWarnings(PHPMD.CyclomaticComplexity)
*/
private function runProcessesInParallel($deployer, $deployableAreaThemeMap, $deployableLanguages)
{
/** @var ProcessManager $processManager */
$processManager = $this->objectManager->create(ProcessManager::class);
$processNumber = 0;
$processQueue = [];
foreach ($deployableAreaThemeMap as $area => &$themes) {
foreach ($themes as $theme) {
foreach ($deployableLanguages as $lang) {
$deployerFunc = function (Process $process) use ($area, $theme, $lang, $deployer) {
return $this->deploy($deployer, [$lang], [$area => [$theme]]);
};
if ($processNumber >= $this->getProcessesAmount()) {
$processQueue[] = $deployerFunc;
} else {
$processManager->fork($deployerFunc);
}
$processNumber++;
}
}
}
$returnStatus = null;
while (count($processManager->getProcesses()) > 0) {
foreach ($processManager->getProcesses() as $process) {
if ($process->isCompleted()) {
$processManager->delete($process);
$returnStatus |= $process->getStatus();
if ($queuedProcess = array_shift($processQueue)) {
$processManager->fork($queuedProcess);
}
if (count($processManager->getProcesses()) >= $this->getProcessesAmount()) {
break 1;
}
}
}
usleep(5000);
}

return $returnStatus === Cli::RETURN_SUCCESS ?: Cli::RETURN_FAILURE;
}

/**
* @return bool
*/
private function isCanBeParalleled()
{
return function_exists('pcntl_fork') && $this->getProcessesAmount() > 1;
}

/**
* @return int
*/
private function getProcessesAmount()
{
$jobs = (int)$this->input->getOption(Options::JOBS_AMOUNT);
if ($jobs < 1) {
throw new \InvalidArgumentException(
Options::JOBS_AMOUNT . ' argument has invalid value. It must be greater than 0'
);
}
return $jobs;
}
}
5 changes: 5 additions & 0 deletions app/code/Magento/Deploy/Console/Command/DeployStaticOptionsInterface.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,4 +92,9 @@ interface DeployStaticOptionsInterface
* Force run of static deploy
*/
const FORCE_RUN = 'force';

/**
* Symlink locale if it not customized
*/
const SYMLINK_LOCALE = 'symlink-locale';
}
23 changes: 23 additions & 0 deletions app/code/Magento/Deploy/Model/Deploy/DeployInterface.php
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
<?php
/**
* Copyright © 2016 Magento. All rights reserved.
* See COPYING.txt for license details.
*/

namespace Magento\Deploy\Model\Deploy;

interface DeployInterface
{
/**
* Base locale option without customizations
*/
const DEPLOY_BASE_LOCALE = 'deploy_base_locale';

/**
* @param string $area
* @param string $themePath
* @param string $locale
* @return int
*/
public function deploy($area, $themePath, $locale);
}
Loading

0 comments on commit 7800e37

Please sign in to comment.