MAGETWO-52660: Improve performance of static assets deployment

Author: mslabko

app/code/Magento/Deploy/Model/Deploy/DeployInterface.php

@@ -8,6 +8,11 @@
 
 interface DeployInterface
 {
+    /**
+     * Base locale option without customizations
+     */
+    const DEPLOY_BASE_LOCALE = 'deploy_base_locale';
+
     /**
      * @param string $area
      * @param string $themePath

app/code/Magento/Deploy/Model/Deploy/LocaleDeploy.php

@@ -25,24 +25,34 @@
  */
 class LocaleDeploy implements DeployInterface
 {
-    /** @var int */
+    /**
+     * @var int
+     */
     private $count = 0;
 
-    /** @var int */
+    /**
+     * @var int
+     */
     private $errorCount = 0;
 
     /**
      * @var OutputInterface
      */
     private $output;
 
-    /** @var \Magento\Framework\View\Asset\Repository */
+    /**
+     * @var \Magento\Framework\View\Asset\Repository
+     */
     private $assetRepo;
 
-    /** @var Publisher */
+    /**
+     * @var Publisher
+     */
     private $assetPublisher;
 
-    /** @var \Magento\Framework\View\Asset\Bundle\Manager */
+    /**
+     * @var \Magento\Framework\View\Asset\Bundle\Manager
+     */
     private $bundleManager;
 
     /**

app/code/Magento/Deploy/Model/Deploy/LocaleQuickDeploy.php

@@ -73,13 +73,13 @@ public function deploy($area, $themePath, $locale)
 
         $this->output->writeln("=== {$area} -> {$themePath} -> {$locale} ===");
 
-        if (!isset($this->options[DeployManager::DEPLOY_BASE_LOCALE])) {
+        if (!isset($this->options[self::DEPLOY_BASE_LOCALE])) {
             throw new \InvalidArgumentException('Deploy base locale must be set for Quick Deploy');
         }
         $processedFiles = 0;
         $errorAmount = 0;
 
-        $baseLocale = $this->options[DeployManager::DEPLOY_BASE_LOCALE];
+        $baseLocale = $this->options[self::DEPLOY_BASE_LOCALE];
         $newLocalePath = $this->getLocalePath($area, $themePath, $locale);
         $baseLocalePath = $this->getLocalePath($area, $themePath, $baseLocale);
         $baseRequireJsPath = RequireJsConfig::DIR_NAME . DIRECTORY_SEPARATOR . $baseLocalePath;

app/code/Magento/Deploy/Model/DeployManager.php

@@ -10,15 +10,10 @@
 use Symfony\Component\Console\Output\OutputInterface;
 use Magento\Deploy\Console\Command\DeployStaticOptionsInterface as Options;
 use Magento\Deploy\Model\Deploy\TemplateMinifier;
-use \Magento\Framework\App\State;
+use Magento\Framework\App\State;
 
 class DeployManager
 {
-    /**
-     * Base locale without customizations
-     */
-    const DEPLOY_BASE_LOCALE = 'deploy_base_locale';
-
     /**
      * @var array
      */
@@ -45,12 +40,12 @@ class DeployManager
     private $deployStrategyProviderFactory;
 
     /**
-     * @var ProcessQueueManager
+     * @var ProcessQueueManagerFactory
      */
-    private $processQueueManager;
+    private $processQueueManagerFactory;
 
     /**
-     * @var \Magento\Deploy\Model\TemplateMinifier
+     * @var TemplateMinifier
      */
     private $templateMinifier;
 
@@ -68,7 +63,7 @@ class DeployManager
      * @param OutputInterface $output
      * @param StorageInterface $versionStorage
      * @param DeployStrategyProviderFactory $deployStrategyProviderFactory
-     * @param ProcessQueueManager $processQueueManager
+     * @param ProcessQueueManagerFactory $processQueueManagerFactory
      * @param TemplateMinifier $templateMinifier
      * @param State $state
      * @param array $options
@@ -77,7 +72,7 @@ public function __construct(
         OutputInterface $output,
         StorageInterface $versionStorage,
         DeployStrategyProviderFactory $deployStrategyProviderFactory,
-        ProcessQueueManager $processQueueManager,
+        ProcessQueueManagerFactory $processQueueManagerFactory,
         TemplateMinifier $templateMinifier,
         State $state,
         array $options
@@ -86,9 +81,10 @@ public function __construct(
         $this->options = $options;
         $this->versionStorage = $versionStorage;
         $this->deployStrategyProviderFactory = $deployStrategyProviderFactory;
-        $this->processQueueManager = $processQueueManager;
+        $this->processQueueManagerFactory = $processQueueManagerFactory;
         $this->templateMinifier = $templateMinifier;
         $this->state = $state;
+        $this->idDryRun = !empty($this->options[Options::DRY_RUN]);
     }
 
     /**
@@ -110,7 +106,6 @@ public function addPack($area, $themePath, $locale)
      */
     public function deploy()
     {
-        $this->idDryRun = isset($this->options[Options::DRY_RUN]) && $this->options[Options::DRY_RUN];
         if ($this->idDryRun) {
             $this->output->writeln('Dry run. Nothing will be recorded to the target directory.');
         }
@@ -123,7 +118,7 @@ public function deploy()
         if ($this->isCanBeParalleled()) {
             $result = $this->runInParallel($strategyProvider);
         } else {
-            $result = null;
+            $result = 0;
             foreach ($this->packages as $package) {
                 $locales = array_keys($package);
                 list($area, $themePath) = current($package);
@@ -162,7 +157,9 @@ private function minifyTemplates()
      */
     private function runInParallel(DeployStrategyProvider $strategyProvider)
     {
-        $this->processQueueManager->setMaxProcessesAmount($this->getProcessesAmount());
+        $processQueueManager = $this->processQueueManagerFactory->create(
+            ['maxProcesses' => $this->getProcessesAmount()]
+        );
         foreach ($this->packages as $package) {
             $locales = array_keys($package);
             list($area, $themePath) = current($package);
@@ -179,10 +176,10 @@ private function runInParallel(DeployStrategyProvider $strategyProvider)
                 }
 
             }
-            $this->processQueueManager->addTaskToQueue($baseStrategy, $dependentStrategy);
+            $processQueueManager->addTaskToQueue($baseStrategy, $dependentStrategy);
         }
 
-        return $this->processQueueManager->process();
+        return $processQueueManager->process();
     }
 
     /**

app/code/Magento/Deploy/Model/DeployStrategyFactory.php

@@ -33,6 +33,7 @@ public function __construct(ObjectManagerInterface $objectManager)
      * @param string $type
      * @param array $arguments
      * @return DeployInterface
+     * @throws \InvalidArgumentException
      */
     public function create($type, array $arguments = [])
     {

app/code/Magento/Deploy/Model/DeployStrategyProvider.php

@@ -8,6 +8,7 @@
 
 use Magento\Deploy\Model\Deploy\DeployInterface;
 use Magento\Framework\Component\ComponentRegistrar;
+use Magento\Framework\Module\Dir;
 use Magento\Framework\View\Design\Fallback\Rule\RuleInterface;
 use Magento\Framework\View\DesignInterface;
 use Magento\Framework\View\Design\Fallback\RulePool;
@@ -122,7 +123,7 @@ private function getLocaleDirectories($params)
         $dirs = $this->getFallbackRule()->getPatternDirs($params);
 
         return array_filter($dirs, function ($dir) {
-            return strpos($dir, 'i18n');
+            return strpos($dir, Dir::MODULE_I18N_DIR);
         });
     }
 
@@ -181,7 +182,7 @@ private function getDeployStrategy($type, $baseLocale = null)
     {
         $options = $this->options;
         if ($baseLocale) {
-            $options[DeployManager::DEPLOY_BASE_LOCALE] = $baseLocale;
+            $options[DeployInterface::DEPLOY_BASE_LOCALE] = $baseLocale;
         }
 
         return $this->deployStrategyFactory->create(

app/code/Magento/Deploy/Model/Deployer.php

@@ -73,6 +73,18 @@ public function __construct(
         }
     }
 
+    /**
+     * @return \Magento\Deploy\Model\DeployManagerFactory
+     */
+    private function getDeployManagerFactory()
+    {
+        if (null === $this->deployManagerFactory) {
+            $this->deployManagerFactory = ObjectManager::getInstance()->get(DeployManagerFactory::class);
+        }
+
+        return $this->deployManagerFactory;
+    }
+
     /**
      * Populate all static view files for specified root path and list of languages
      *
@@ -85,11 +97,8 @@ public function __construct(
      */
     public function deploy(ObjectManagerFactory $omFactory, array $locales, array $deployableAreaThemeMap = [])
     {
-        if (null === $this->deployManagerFactory) {
-            $this->deployManagerFactory = ObjectManager::getInstance()->get(DeployManagerFactory::class);
-        }
         /** @var DeployManager $deployerManager */
-        $deployerManager = $this->deployManagerFactory->create(
+        $deployerManager = $this->getDeployManagerFactory()->create(
             ['options' => $this->options, 'output' => $this->output]
         );
 

app/code/Magento/Deploy/Model/ProcessManager.php

@@ -11,6 +11,20 @@ class ProcessManager
     /** @var Process[] */
     private $processes = [];
 
+    /**
+     * @var ProcessFactory
+     */
+    private $processFactory;
+
+    /**
+     * ProcessManager constructor.
+     * @param ProcessFactory $processFactory
+     */
+    public function __construct(ProcessFactory $processFactory)
+    {
+        $this->processFactory = $processFactory;
+    }
+
     /**
      * Forks the currently running process.
      *
@@ -66,11 +80,11 @@ public function delete(Process $process)
      */
     private function createProcess(callable $handler)
     {
-        return new Process($handler);
+        return $this->processFactory->create(['handler' => $handler]);
     }
 
     /**
-     * Protect against zombie process
+     * Protect against zombie process§
      * @return void
      * @SuppressWarnings(PHPMD.UnusedLocalVariable)
      */

app/code/Magento/Deploy/Model/ProcessQueueManager.php

@@ -10,6 +10,11 @@
 
 class ProcessQueueManager
 {
+    /**
+     * Default max amount of processes
+     */
+    const DEFAULT_MAX_PROCESSES_AMOUNT = 4;
+
     /**
      * @var ProcessTask[]
      */
@@ -23,7 +28,7 @@ class ProcessQueueManager
     /**
      * @var int
      */
-    private $maxProcesses = 4;
+    private $maxProcesses;
 
     /**
      * @var ProcessManager
@@ -36,21 +41,25 @@ class ProcessQueueManager
     private $resourceConnection;
 
     /**
-     * @param ProcessManager $processManager
-     * @param ResourceConnection $resourceConnection
+     * @var ProcessTaskFactory
      */
-    public function __construct(ProcessManager $processManager, ResourceConnection $resourceConnection)
-    {
-        $this->processManager = $processManager;
-        $this->resourceConnection = $resourceConnection;
-    }
+    private $processTaskFactory;
 
     /**
+     * @param ProcessManager $processManager
+     * @param ResourceConnection $resourceConnection
+     * @param ProcessTaskFactory $processTaskFactory
      * @param int $maxProcesses
-     * @return void
      */
-    public function setMaxProcessesAmount($maxProcesses)
-    {
+    public function __construct(
+        ProcessManager $processManager,
+        ResourceConnection $resourceConnection,
+        ProcessTaskFactory $processTaskFactory,
+        $maxProcesses = self::DEFAULT_MAX_PROCESSES_AMOUNT
+    ) {
+        $this->processManager = $processManager;
+        $this->resourceConnection = $resourceConnection;
+        $this->processTaskFactory = $processTaskFactory;
         $this->maxProcesses = $maxProcesses;
     }
 
@@ -78,7 +87,7 @@ public function process()
         $processQueue = [];
         $this->internalQueueProcess($this->tasksQueue, $processQueue);
 
-        $returnStatus = null;
+        $returnStatus = 0;
         while (count($this->processManager->getProcesses()) > 0) {
             foreach ($this->processManager->getProcesses() as $process) {
                 if ($process->isCompleted()) {
@@ -131,7 +140,7 @@ private function internalQueueProcess($taskQueue, &$processQueue)
      */
     private function createTask($handler, $dependentTasks = [])
     {
-        return new ProcessTask($handler, $dependentTasks);
+        return $this->processTaskFactory->create(['handler' => $handler, 'dependentTasks' => $dependentTasks]);
     }
 
     /**

app/code/Magento/Deploy/Model/ProcessTask.php

@@ -27,7 +27,7 @@ class ProcessTask
      * @param callable $handler
      * @param array $dependentTasks
      */
-    public function __construct($handler, $dependentTasks = [])
+    public function __construct($handler, array $dependentTasks = [])
     {
         $this->taskId = uniqid('', true);
         $this->handler = $handler;