MAGECLOUD-3382: Increase default SCD_THREADS to 3 (#447)

oshmyheliuk · shiftedreality · commit 9cb4b55c9ba5 · 2019-04-22T14:34:25.000-05:00
diff --git a/dist/.magento.env.yaml b/dist/.magento.env.yaml
@@ -559,15 +559,6 @@
 #            variables:                                                                                               #
 #              ENV_APPLICATION: "MAGENTO_CLOUD_APPLICATION"                                                           #
 #######################################################################################################################
-# ENV_MODE - Environment variable used to get application mode.                                                       #
-# Magento Version: Magento 2.1.4 and later                                                                            #
-# Default value: - "MAGENTO_CLOUD_MODE"                                                                               #
-# Section: variables                                                                                                  #
-# Example:                                                                                                            #
-#          system:                                                                                                    #
-#            variables:                                                                                               #
-#              ENV_MODE: "MAGENTO_CLOUD_MODE"                                                                         #
-#######################################################################################################################
 # ENV_ENVIRONMENT - Environment variable used to get application environment.                                         #
 # Magento Version: Magento 2.1.4 and later                                                                            #
 # Default value: - "MAGENTO_CLOUD_ENVIRONMENT"                                                                        #
diff --git a/src/App/Container.php b/src/App/Container.php
@@ -28,6 +28,7 @@
 use Magento\MagentoCloud\Process\PostDeploy as PostDeployProcess;
 use Magento\MagentoCloud\Process\ProcessComposite;
 use Magento\MagentoCloud\Process\ProcessInterface;
+use Magento\MagentoCloud\Process\SetProductionMode;
 
 /**
  * @inheritdoc
@@ -175,6 +176,7 @@ function () {
                 return $this->container->makeWith(ProcessComposite::class, [
                     'processes' => [
                         $this->container->make(BuildProcess\PreBuild::class),
+                        $this->container->make(SetProductionMode::class),
                         $this->container->make(\Magento\MagentoCloud\Process\ValidateConfiguration::class, [
                             'validators' => [
                                 ValidatorInterface::LEVEL_CRITICAL => [
@@ -351,7 +353,7 @@ function () {
                         $this->container->make(DeployProcess\PreDeploy\CleanRedisCache::class),
                         $this->container->make(DeployProcess\PreDeploy\CleanFileCache::class),
                         $this->container->make(DeployProcess\PreDeploy\RestoreWritableDirectories::class),
-                        $this->container->make(DeployProcess\PreDeploy\SetProductionMode::class),
+                        $this->container->make(SetProductionMode::class),
                     ],
                 ]);
             });
diff --git a/src/Config/Environment.php b/src/Config/Environment.php
@@ -26,11 +26,6 @@ class Environment
      */
     const GIT_MASTER_BRANCH_RE = '/^(master|production|staging)(?:-[a-z0-9]+)?$/i';
 
-    /**
-     * @deprecated Threads environment variables must be used.
-     */
-    const CLOUD_MODE_ENTERPRISE = 'enterprise';
-
     const VAL_ENABLED = 'enabled';
     const VAL_DISABLED = 'disabled';
 
diff --git a/src/Config/Schema.php b/src/Config/Schema.php
@@ -129,8 +129,8 @@ public function getSchema()
                     StageConfigInterface::STAGE_DEPLOY
                 ],
                 self::SCHEMA_DEFAULT_VALUE => [
-                    StageConfigInterface::STAGE_BUILD => 1,
-                    StageConfigInterface::STAGE_DEPLOY => 1,
+                    StageConfigInterface::STAGE_BUILD => StageConfigInterface::VAR_SCD_THREADS_DEFAULT_VALUE,
+                    StageConfigInterface::STAGE_DEPLOY => StageConfigInterface::VAR_SCD_THREADS_DEFAULT_VALUE,
                 ],
             ],
             StageConfigInterface::VAR_SCD_EXCLUDE_THEMES => [
@@ -217,15 +217,6 @@ public function getSchema()
                     SystemConfigInterface::SYSTEM_VARIABLES => 'MAGENTO_CLOUD_APPLICATION',
                 ],
             ],
-            SystemConfigInterface::VAR_ENV_MODE => [
-                self::SCHEMA_TYPE => ['string'],
-                self::SCHEMA_SYSTEM => [
-                    SystemConfigInterface::SYSTEM_VARIABLES
-                ],
-                self::SCHEMA_DEFAULT_VALUE => [
-                    SystemConfigInterface::SYSTEM_VARIABLES => 'MAGENTO_CLOUD_MODE',
-                ],
-            ],
             SystemConfigInterface::VAR_ENV_ENVIRONMENT => [
                 self::SCHEMA_TYPE => ['string'],
                 self::SCHEMA_SYSTEM => [
diff --git a/src/Config/Stage/Deploy/MergedConfig.php b/src/Config/Stage/Deploy/MergedConfig.php
@@ -10,7 +10,6 @@
 use Magento\MagentoCloud\Config\Schema;
 use Magento\MagentoCloud\Config\Stage\DeployInterface;
 use Magento\MagentoCloud\Config\StageConfigInterface;
-use Magento\MagentoCloud\Config\SystemConfigInterface;
 
 /**
  * Returns all merged configuration for deploy phase
@@ -74,7 +73,6 @@ public function get(): array
 
                 $this->mergedConfig = array_replace(
                     $this->schema->getDefaults(StageConfigInterface::STAGE_DEPLOY),
-                    $this->getDeployConfiguration(),
                     $envConfig[DeployInterface::STAGE_GLOBAL] ?? [],
                     $envConfig[DeployInterface::STAGE_DEPLOY] ?? [],
                     $this->environmentConfig->getAll()
@@ -90,26 +88,4 @@ public function get(): array
             );
         }
     }
-
-    /**
-     * Resolves default configuration value for deploy stage.
-     *
-     * SCD_THREADS = 3 for production environment.
-     *
-     * @return array
-     * @deprecated Threads count shouldn't depend on ENV_MODE variable as this variable doesn't set anymore.
-     *             Threads environment variables or .magento.env.yaml must be used instead.
-     */
-    private function getDeployConfiguration(): array
-    {
-        $config = [];
-
-        $envVar = $this->environment->getEnvVarName(SystemConfigInterface::VAR_ENV_MODE);
-
-        if ($this->environment->getEnv($envVar) === Environment::CLOUD_MODE_ENTERPRISE) {
-            $config[DeployInterface::VAR_SCD_THREADS] = 3;
-        }
-
-        return $config;
-    }
 }
diff --git a/src/Config/StageConfigInterface.php b/src/Config/StageConfigInterface.php
@@ -67,6 +67,11 @@ interface StageConfigInterface
      */
     const OPTION_MERGE = '_merge';
 
+    /**
+     * Default value of SCD_THREADS variable.
+     */
+    const VAR_SCD_THREADS_DEFAULT_VALUE = -1;
+
     /**
      * Retrieves environment configuration per stage.
      *
diff --git a/src/Config/SystemConfigInterface.php b/src/Config/SystemConfigInterface.php
@@ -27,7 +27,6 @@ interface SystemConfigInterface
     const VAR_ENV_ROUTES = 'ENV_ROUTES';
     const VAR_ENV_VARIABLES = 'ENV_VARIABLES';
     const VAR_ENV_APPLICATION = 'ENV_APPLICATION';
-    const VAR_ENV_MODE = 'ENV_MODE';
     const VAR_ENV_ENVIRONMENT = 'ENV_ENVIRONMENT';
 
     /**
diff --git a/src/Process/SetProductionMode.php b/src/Process/SetProductionMode.php
@@ -3,11 +3,9 @@
  * Copyright © Magento, Inc. All rights reserved.
  * See COPYING.txt for license details.
  */
-namespace Magento\MagentoCloud\Process\Deploy\PreDeploy;
+namespace Magento\MagentoCloud\Process;
 
 use Magento\MagentoCloud\Filesystem\FileSystemException;
-use Magento\MagentoCloud\Process\ProcessException;
-use Magento\MagentoCloud\Process\ProcessInterface;
 use Magento\MagentoCloud\Config\Deploy\Writer;
 use Psr\Log\LoggerInterface;
 
diff --git a/src/StaticContent/ThreadCountOptimizer.php b/src/StaticContent/ThreadCountOptimizer.php
@@ -5,7 +5,9 @@
  */
 namespace Magento\MagentoCloud\StaticContent;
 
+use Magento\MagentoCloud\Config\StageConfigInterface;
 use Magento\MagentoCloud\Package\MagentoVersion;
+use Magento\MagentoCloud\Util\Cpu;
 use Psr\Log\LoggerInterface;
 
 /**
@@ -19,11 +21,14 @@ class ThreadCountOptimizer
     const STRATEGY_COMPACT = 'compact';
 
     /**
-     * Recommended tread count value for compact strategy
-     *
-     * @var int
+     * Recommended tread count value for compact strategy before 2.2.4 version.
      */
-    const THREAD_COUNT_COMPACT_STRATEGY = 1;
+    const THREAD_COUNT_COMPACT_STRATEGY_BEFORE_2_2_4 = 1;
+
+    /**
+     * Optimal value for job count if environment has enough cpu cores.
+     */
+    const THREAD_COUNT_OPTIMAL = 4;
 
     /**
      * @var LoggerInterface
@@ -35,14 +40,21 @@ class ThreadCountOptimizer
      */
     private $magentoVersion;
 
+    /**
+     * @var Cpu
+     */
+    private $cpu;
+
     /**
      * @param LoggerInterface $logger
      * @param MagentoVersion $magentoVersion
+     * @param Cpu $cpu
      */
-    public function __construct(LoggerInterface $logger, MagentoVersion $magentoVersion)
+    public function __construct(LoggerInterface $logger, MagentoVersion $magentoVersion, Cpu $cpu)
     {
         $this->logger = $logger;
         $this->magentoVersion = $magentoVersion;
+        $this->cpu = $cpu;
     }
 
     /**
@@ -59,13 +71,18 @@ public function optimize(int $threads, string $strategy): int
          * in the case of using a compact strategy of static content deployment
          */
         if ($strategy === self::STRATEGY_COMPACT && $this->magentoVersion->satisfies('<2.2.4')) {
-            if ($threads !== self::THREAD_COUNT_COMPACT_STRATEGY) {
+            if ($threads !== self::THREAD_COUNT_COMPACT_STRATEGY_BEFORE_2_2_4) {
                 $this->logger->notice(
                     'Threads count was forced to 1 as compact strategy can\'t be run with more than one job'
                 );
             }
 
-            return self::THREAD_COUNT_COMPACT_STRATEGY;
+            return self::THREAD_COUNT_COMPACT_STRATEGY_BEFORE_2_2_4;
+        }
+
+        if ($threads === StageConfigInterface::VAR_SCD_THREADS_DEFAULT_VALUE) {
+            $threads = $this->magentoVersion->satisfies('>2.1.10') ?
+                min($this->cpu->getThreadsCount(), self::THREAD_COUNT_OPTIMAL) : 1;
         }
 
         return $threads;
diff --git a/src/Test/Integration/Cron21Test.php b/src/Test/Integration/Cron21Test.php
@@ -5,13 +5,6 @@
  */
 namespace Magento\MagentoCloud\Test\Integration;
 
-use Magento\MagentoCloud\Command\Build;
-use Magento\MagentoCloud\Command\Deploy;
-use Magento\MagentoCloud\Application;
-use Magento\MagentoCloud\DB\ConnectionInterface;
-use Magento\MagentoCloud\Filesystem\Driver\File;
-use Symfony\Component\Console\Tester\CommandTester;
-
 /**
  * {@inheritdoc}
  *
diff --git a/src/Test/Unit/Config/EnvironmentTest.php b/src/Test/Unit/Config/EnvironmentTest.php
@@ -14,6 +14,7 @@
 use Magento\MagentoCloud\Config\SystemConfigInterface;
 use Magento\MagentoCloud\Config\System\Variables;
 use phpmock\phpunit\PHPMock;
+use PHPUnit\Framework\MockObject\MockObject;
 use PHPUnit\Framework\TestCase;
 
 /**
@@ -43,6 +44,8 @@ class EnvironmentTest extends TestCase
      */
     protected function setUp()
     {
+        /** @var MockObject|Reader $environmentReaderMock */
+        /** @var MockObject|Schema $schemaMock */
         $environmentReaderMock = $this->createMock(Reader::class);
         $schemaMock = $this->createMock(Schema::class);
         $schemaMock->expects($this->any())
@@ -53,7 +56,6 @@ protected function setUp()
                 SystemConfigInterface::VAR_ENV_ROUTES => 'MAGENTO_CLOUD_ROUTES',
                 SystemConfigInterface::VAR_ENV_VARIABLES => 'MAGENTO_CLOUD_VARIABLES',
                 SystemConfigInterface::VAR_ENV_APPLICATION => 'MAGENTO_CLOUD_APPLICATION',
-                SystemConfigInterface::VAR_ENV_MODE => 'MAGENTO_CLOUD_MODE',
                 SystemConfigInterface::VAR_ENV_ENVIRONMENT => 'MAGENTO_CLOUD_ENVIRONMENT',
             ]);
 
diff --git a/src/Test/Unit/Config/SchemaTest.php b/src/Test/Unit/Config/SchemaTest.php
@@ -39,7 +39,7 @@ public function testGetDefaultsForBuild()
                 BuildInterface::VAR_SKIP_SCD => false,
                 BuildInterface::VAR_SCD_COMPRESSION_LEVEL => 6,
                 BuildInterface::VAR_SCD_COMPRESSION_TIMEOUT => 600,
-                BuildInterface::VAR_SCD_THREADS => 1,
+                BuildInterface::VAR_SCD_THREADS => -1,
                 BuildInterface::VAR_SCD_EXCLUDE_THEMES => '',
                 BuildInterface::VAR_VERBOSE_COMMANDS => '',
                 BuildInterface::VAR_SCD_MATRIX => [],
@@ -69,7 +69,7 @@ public function testGetDefaultsForDeploy()
                 DeployInterface::VAR_STATIC_CONTENT_SYMLINK => true,
                 DeployInterface::VAR_UPDATE_URLS => true,
                 DeployInterface::VAR_SKIP_SCD => false,
-                DeployInterface::VAR_SCD_THREADS => 1,
+                DeployInterface::VAR_SCD_THREADS => -1,
                 DeployInterface::VAR_GENERATED_CODE_SYMLINK => true,
                 DeployInterface::VAR_SCD_EXCLUDE_THEMES => '',
                 DeployInterface::VAR_REDIS_USE_SLAVE_CONNECTION => false,
@@ -102,7 +102,6 @@ public function testGetDefaultsForSystemVariables()
                 SystemConfigInterface::VAR_ENV_ROUTES => 'MAGENTO_CLOUD_ROUTES',
                 SystemConfigInterface::VAR_ENV_VARIABLES => 'MAGENTO_CLOUD_VARIABLES',
                 SystemConfigInterface::VAR_ENV_APPLICATION => 'MAGENTO_CLOUD_APPLICATION',
-                SystemConfigInterface::VAR_ENV_MODE => 'MAGENTO_CLOUD_MODE',
                 SystemConfigInterface::VAR_ENV_ENVIRONMENT => 'MAGENTO_CLOUD_ENVIRONMENT',
             ],
             $this->schema->getDefaults(SystemConfigInterface::SYSTEM_VARIABLES)
@@ -155,7 +154,6 @@ public function testGetSchemaItemsExists()
             SystemConfigInterface::VAR_ENV_ROUTES,
             SystemConfigInterface::VAR_ENV_VARIABLES,
             SystemConfigInterface::VAR_ENV_APPLICATION,
-            SystemConfigInterface::VAR_ENV_MODE,
             SystemConfigInterface::VAR_ENV_ENVIRONMENT
         ];
 
diff --git a/src/Test/Unit/Config/Stage/Deploy/MergedConfigTest.php b/src/Test/Unit/Config/Stage/Deploy/MergedConfigTest.php
@@ -11,9 +11,7 @@
 use Magento\MagentoCloud\Config\Stage\Deploy;
 use Magento\MagentoCloud\Config\Stage\Deploy\EnvironmentConfig;
 use Magento\MagentoCloud\Config\Stage\Deploy\MergedConfig;
-use Magento\MagentoCloud\Config\Stage\DeployInterface;
 use Magento\MagentoCloud\Config\StageConfigInterface;
-use Magento\MagentoCloud\Config\SystemConfigInterface;
 use Magento\MagentoCloud\Filesystem\FileSystemException;
 use PHPUnit\Framework\MockObject\MockObject;
 use PHPUnit\Framework\TestCase;
@@ -181,68 +179,6 @@ public function getDataProvider(): array
         ];
     }
 
-    public function testGetEnterpriseEnv()
-    {
-        $this->schemaMock->expects($this->once())
-            ->method('getDefaults')
-            ->with(StageConfigInterface::STAGE_DEPLOY)
-            ->willReturn([
-                Deploy::VAR_SCD_THREADS => 1,
-            ]);
-        $this->environmentMock->expects($this->once())
-            ->method('getEnvVarName')
-            ->with(SystemConfigInterface::VAR_ENV_MODE)
-            ->willReturn('MAGENTO_CLOUD_MODE');
-        $this->environmentMock->expects($this->once())
-            ->method('getEnv')
-            ->with('MAGENTO_CLOUD_MODE')
-            ->willReturn(Environment::CLOUD_MODE_ENTERPRISE);
-
-        $this->assertEquals(
-            [
-                Deploy::VAR_SCD_THREADS => 3,
-            ],
-            $this->mergedConfig->get()
-        );
-    }
-
-    public function testGetEnterpriseEnvOverwrittenByEnvYaml()
-    {
-        $this->schemaMock->expects($this->once())
-            ->method('getDefaults')
-            ->with(StageConfigInterface::STAGE_DEPLOY)
-            ->willReturn([
-                Deploy::VAR_SCD_THREADS => 1,
-            ]);
-        $this->environmentMock->expects($this->once())
-            ->method('getEnvVarName')
-            ->with(SystemConfigInterface::VAR_ENV_MODE)
-            ->willReturn('MAGENTO_CLOUD_MODE');
-        $this->environmentMock->expects($this->once())
-            ->method('getEnv')
-            ->with('MAGENTO_CLOUD_MODE')
-            ->willReturn(Environment::CLOUD_MODE_ENTERPRISE);
-        $this->environmentReaderMock->expects($this->once())
-            ->method('read')
-            ->willReturn([
-                DeployInterface::SECTION_STAGE => [
-                    Deploy::STAGE_GLOBAL => [
-                        Deploy::VAR_SCD_THREADS => 4,
-                    ],
-                    Deploy::STAGE_DEPLOY => [
-                        Deploy::VAR_SCD_THREADS => 5,
-                    ],
-                ]
-            ]);
-
-        $this->assertEquals(
-            [
-                Deploy::VAR_SCD_THREADS => 5,
-            ],
-            $this->mergedConfig->get()
-        );
-    }
-
     /**
      * @expectedExceptionMessage File system error
      * @expectedException \RuntimeException
diff --git a/src/Test/Unit/Config/System/VariablesTest.php b/src/Test/Unit/Config/System/VariablesTest.php
diff --git a/src/Test/Unit/Process/SetProductionModeTest.php b/src/Test/Unit/Process/SetProductionModeTest.php
diff --git a/src/Test/Unit/StaticContent/ThreadCountOptimizerTest.php b/src/Test/Unit/StaticContent/ThreadCountOptimizerTest.php
diff --git a/src/Test/Unit/Util/CpuTest.php b/src/Test/Unit/Util/CpuTest.php
diff --git a/src/Util/Cpu.php b/src/Util/Cpu.php
