Skip to content

Commit b248351

Browse files
slopukhovslopukhov
committed
MAGETWO-93091: Add price indexer parallelization by dimensions (full reindex only)
1 parent 37cc4e3 commit b248351

File tree

6 files changed

+112
-33
lines changed

6 files changed

+112
-33
lines changed

‎app/code/Magento/Catalog/Model/Indexer/Product/Price/AbstractAction.php

Lines changed: 33 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ protected function _syncData(array $processIds = [])
160160
{
161161
// for backward compatibility split data from old idx table on dimension tables
162162
foreach ($this->dimensionCollectionFactory->create() as $dimensions) {
163-
$insertSelect = $this->_connection->select()->from(
163+
$insertSelect = $this->getConnection()->select()->from(
164164
['ip_tmp' => $this->_defaultIndexerResource->getIdxTable()]
165165
);
166166

@@ -174,7 +174,7 @@ protected function _syncData(array $processIds = [])
174174
}
175175

176176
$query = $insertSelect->insertFromSelect($this->tableMaintainer->getMainTable($dimensions));
177-
$this->_connection->query($query);
177+
$this->getConnection()->query($query);
178178
}
179179
return $this;
180180
}
@@ -191,7 +191,7 @@ protected function _prepareWebsiteDateTable()
191191
{
192192
$baseCurrency = $this->_config->getValue(\Magento\Directory\Model\Currency::XML_PATH_CURRENCY_BASE);
193193

194-
$select = $this->_connection->select()->from(
194+
$select = $this->getConnection()->select()->from(
195195
['cw' => $this->_defaultIndexerResource->getTable('store_website')],
196196
['website_id']
197197
)->join(
@@ -203,7 +203,7 @@ protected function _prepareWebsiteDateTable()
203203
);
204204

205205
$data = [];
206-
foreach ($this->_connection->fetchAll($select) as $item) {
206+
foreach ($this->getConnection()->fetchAll($select) as $item) {
207207
/** @var $website \Magento\Store\Model\Website */
208208
$website = $this->_storeManager->getWebsite($item['website_id']);
209209

@@ -237,7 +237,7 @@ protected function _prepareWebsiteDateTable()
237237
$this->_emptyTable($table);
238238
if ($data) {
239239
foreach ($data as $row) {
240-
$this->_connection->insertOnDuplicate($table, $row, array_keys($row));
240+
$this->getConnection()->insertOnDuplicate($table, $row, array_keys($row));
241241
}
242242
}
243243

@@ -325,19 +325,19 @@ protected function _getIndexer($productTypeId)
325325
*/
326326
protected function _insertFromTable($sourceTable, $destTable, $where = null)
327327
{
328-
$sourceColumns = array_keys($this->_connection->describeTable($sourceTable));
329-
$targetColumns = array_keys($this->_connection->describeTable($destTable));
330-
$select = $this->_connection->select()->from($sourceTable, $sourceColumns);
328+
$sourceColumns = array_keys($this->getConnection()->describeTable($sourceTable));
329+
$targetColumns = array_keys($this->getConnection()->describeTable($destTable));
330+
$select = $this->getConnection()->select()->from($sourceTable, $sourceColumns);
331331
if ($where) {
332332
$select->where($where);
333333
}
334-
$query = $this->_connection->insertFromSelect(
334+
$query = $this->getConnection()->insertFromSelect(
335335
$select,
336336
$destTable,
337337
$targetColumns,
338338
\Magento\Framework\DB\Adapter\AdapterInterface::INSERT_ON_DUPLICATE
339339
);
340-
$this->_connection->query($query);
340+
$this->getConnection()->query($query);
341341
}
342342

343343
/**
@@ -348,7 +348,7 @@ protected function _insertFromTable($sourceTable, $destTable, $where = null)
348348
*/
349349
protected function _emptyTable($table)
350350
{
351-
$this->_connection->delete($table);
351+
$this->getConnection()->delete($table);
352352
}
353353

354354
/**
@@ -407,12 +407,12 @@ protected function _reindexRows($changedIds = [])
407407
private function deleteIndexData(array $entityIds)
408408
{
409409
foreach ($this->dimensionCollectionFactory->create() as $dimensions) {
410-
$select = $this->_connection->select()->from(
410+
$select = $this->getConnection()->select()->from(
411411
['index_price' => $this->tableMaintainer->getMainTable($dimensions)],
412412
null
413413
)->where('index_price.entity_id IN (?)', $entityIds);
414414
$query = $select->deleteFromSelect('index_price');
415-
$this->_connection->query($query);
415+
$this->getConnection()->query($query);
416416
}
417417
}
418418

@@ -430,7 +430,7 @@ private function deleteIndexData(array $entityIds)
430430
protected function _copyRelationIndexData($parentIds, $excludeIds = null)
431431
{
432432
$linkField = $this->getProductIdFieldName();
433-
$select = $this->_connection->select()->from(
433+
$select = $this->getConnection()->select()->from(
434434
$this->_defaultIndexerResource->getTable('catalog_product_relation'),
435435
['child_id']
436436
)->join(
@@ -445,18 +445,18 @@ protected function _copyRelationIndexData($parentIds, $excludeIds = null)
445445
$select->where('child_id NOT IN(?)', $excludeIds);
446446
}
447447

448-
$children = $this->_connection->fetchCol($select);
448+
$children = $this->getConnection()->fetchCol($select);
449449

450450
if ($children) {
451451
foreach ($this->dimensionCollectionFactory->create() as $dimensions) {
452-
$select = $this->_connection->select()->from(
452+
$select = $this->getConnection()->select()->from(
453453
$this->getIndexTargetTableByDimension($dimensions)
454454
)->where(
455455
'entity_id IN(?)',
456456
$children
457457
);
458458
$query = $select->insertFromSelect($this->_defaultIndexerResource->getIdxTable(), [], false);
459-
$this->_connection->query($query);
459+
$this->getConnection()->query($query);
460460
}
461461
}
462462

@@ -502,8 +502,8 @@ protected function getIndexTargetTable()
502502
protected function getProductIdFieldName()
503503
{
504504
$table = $this->_defaultIndexerResource->getTable('catalog_product_entity');
505-
$indexList = $this->_connection->getIndexList($table);
506-
return $indexList[$this->_connection->getPrimaryKeyName($table)]['COLUMNS_LIST'][0];
505+
$indexList = $this->getConnection()->getIndexList($table);
506+
return $indexList[$this->getConnection()->getPrimaryKeyName($table)]['COLUMNS_LIST'][0];
507507
}
508508

509509
/**
@@ -514,14 +514,14 @@ protected function getProductIdFieldName()
514514
*/
515515
private function getProductsTypes(array $changedIds = [])
516516
{
517-
$select = $this->_connection->select()->from(
517+
$select = $this->getConnection()->select()->from(
518518
$this->_defaultIndexerResource->getTable('catalog_product_entity'),
519519
['entity_id', 'type_id']
520520
);
521521
if ($changedIds) {
522522
$select->where('entity_id IN (?)', $changedIds);
523523
}
524-
$pairs = $this->_connection->fetchPairs($select);
524+
$pairs = $this->getConnection()->fetchPairs($select);
525525

526526
$byType = [];
527527
foreach ($pairs as $productId => $productType) {
@@ -540,7 +540,7 @@ private function getProductsTypes(array $changedIds = [])
540540
*/
541541
private function getParentProductsTypes(array $productsIds)
542542
{
543-
$select = $this->_connection->select()->from(
543+
$select = $this->getConnection()->select()->from(
544544
['l' => $this->_defaultIndexerResource->getTable('catalog_product_relation')],
545545
''
546546
)->join(
@@ -551,7 +551,7 @@ private function getParentProductsTypes(array $productsIds)
551551
'l.child_id IN(?)',
552552
$productsIds
553553
);
554-
$pairs = $this->_connection->fetchPairs($select);
554+
$pairs = $this->getConnection()->fetchPairs($select);
555555

556556
$byType = [];
557557
foreach ($pairs as $productId => $productType) {
@@ -560,4 +560,14 @@ private function getParentProductsTypes(array $productsIds)
560560

561561
return $byType;
562562
}
563+
564+
/**
565+
* Get connection
566+
*
567+
* @return \Magento\Framework\DB\Adapter\AdapterInterface
568+
*/
569+
private function getConnection()
570+
{
571+
return $this->_defaultIndexerResource->getConnection();
572+
}
563573
}

‎app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,11 @@ class Full extends \Magento\Catalog\Model\Indexer\Product\Price\AbstractAction
5656
*/
5757
private $dimensionTableMaintainer;
5858

59+
/**
60+
* @var \Magento\Indexer\Model\ProcessManager
61+
*/
62+
private $processManager;
63+
5964
/**
6065
* @param \Magento\Framework\App\Config\ScopeConfigInterface $config
6166
* @param \Magento\Store\Model\StoreManagerInterface $storeManager
@@ -71,7 +76,7 @@ class Full extends \Magento\Catalog\Model\Indexer\Product\Price\AbstractAction
7176
* @param \Magento\Catalog\Model\ResourceModel\Indexer\ActiveTableSwitcher|null $activeTableSwitcher
7277
* @param \Magento\Catalog\Model\Indexer\Product\Price\DimensionCollectionFactory|null $dimensionCollectionFactory
7378
* @param \Magento\Catalog\Model\Indexer\Product\Price\TableMaintainer|null $dimensionTableMaintainer
74-
*
79+
* @param \Magento\Indexer\Model\ProcessManager $processManager
7580
* @SuppressWarnings(PHPMD.ExcessiveParameterList)
7681
*/
7782
public function __construct(
@@ -88,7 +93,8 @@ public function __construct(
8893
\Magento\Framework\Indexer\BatchProviderInterface $batchProvider = null,
8994
\Magento\Catalog\Model\ResourceModel\Indexer\ActiveTableSwitcher $activeTableSwitcher = null,
9095
\Magento\Catalog\Model\Indexer\Product\Price\DimensionCollectionFactory $dimensionCollectionFactory = null,
91-
\Magento\Catalog\Model\Indexer\Product\Price\TableMaintainer $dimensionTableMaintainer = null
96+
\Magento\Catalog\Model\Indexer\Product\Price\TableMaintainer $dimensionTableMaintainer = null,
97+
\Magento\Indexer\Model\ProcessManager $processManager = null
9298
) {
9399
parent::__construct(
94100
$config,
@@ -118,6 +124,9 @@ public function __construct(
118124
$this->dimensionTableMaintainer = $dimensionTableMaintainer ?: ObjectManager::getInstance()->get(
119125
\Magento\Catalog\Model\Indexer\Product\Price\TableMaintainer::class
120126
);
127+
$this->processManager = $processManager ?: ObjectManager::getInstance()->get(
128+
\Magento\Indexer\Model\ProcessManager::class
129+
);
121130
}
122131

123132
/**
@@ -195,9 +204,13 @@ private function truncateReplicaTables()
195204
*/
196205
private function reindexProductTypeWithDimensions(DimensionalIndexerInterface $priceIndexer, string $typeId)
197206
{
207+
$userFunctions = [];
198208
foreach ($this->dimensionCollectionFactory->create() as $dimensions) {
199-
$this->reindexByBatches($priceIndexer, $dimensions, $typeId);
209+
$userFunctions[] = function () use ($priceIndexer, $dimensions, $typeId) {
210+
return $this->reindexByBatches($priceIndexer, $dimensions, $typeId);
211+
};
200212
}
213+
$this->processManager->execute($userFunctions);
201214
}
202215

203216
/**

‎app/code/Magento/Catalog/Model/Indexer/Product/Price/ModeSwitcher.php

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ public function __construct(
5555
* @param string $currentMode
5656
*
5757
* @return void
58+
* @throws \Zend_Db_Exception
5859
*/
5960
public function createTables(string $currentMode)
6061
{
@@ -154,7 +155,12 @@ private function insertFromOldTablesToNew(string $newTable, string $oldTable, ar
154155
}
155156
}
156157
$this->tableMaintainer->getConnection()->query(
157-
$this->tableMaintainer->getConnection()->insertFromSelect($select, $newTable)
158+
$this->tableMaintainer->getConnection()->insertFromSelect(
159+
$select,
160+
$newTable,
161+
[],
162+
\Magento\Framework\DB\Adapter\AdapterInterface::INSERT_ON_DUPLICATE
163+
)
158164
);
159165
}
160166
}

‎app/code/Magento/CatalogInventory/Model/Indexer/ProductPriceIndexFilter.php

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
use Magento\CatalogInventory\Model\Stock;
1313
use Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\PriceModifierInterface;
1414
use Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\IndexTableStructure;
15+
use Magento\Framework\App\ResourceConnection;
16+
use Magento\Framework\App\ObjectManager;
1517

1618
/**
1719
* Class for filter product price index.
@@ -28,14 +30,32 @@ class ProductPriceIndexFilter implements PriceModifierInterface
2830
*/
2931
private $stockItem;
3032

33+
/**
34+
* @var ResourceConnection
35+
*/
36+
private $resourceConnection;
37+
38+
/**
39+
* @var string
40+
*/
41+
private $connectionName;
42+
3143
/**
3244
* @param StockConfigurationInterface $stockConfiguration
3345
* @param Item $stockItem
46+
* @param ResourceConnection $resourceConnection
47+
* @param string $connectionName
3448
*/
35-
public function __construct(StockConfigurationInterface $stockConfiguration, Item $stockItem)
36-
{
49+
public function __construct(
50+
StockConfigurationInterface $stockConfiguration,
51+
Item $stockItem,
52+
ResourceConnection $resourceConnection = null,
53+
$connectionName = 'indexer'
54+
) {
3755
$this->stockConfiguration = $stockConfiguration;
3856
$this->stockItem = $stockItem;
57+
$this->resourceConnection = $resourceConnection ?: ObjectManager::getInstance()->get(ResourceConnection::class);
58+
$this->connectionName = $connectionName;
3959
}
4060

4161
/**
@@ -44,6 +64,8 @@ public function __construct(StockConfigurationInterface $stockConfiguration, Ite
4464
* @param IndexTableStructure $priceTable
4565
* @param array $entityIds
4666
* @return void
67+
*
68+
* @throws \Magento\Framework\Exception\LocalizedException
4769
* @SuppressWarnings(PHPMD.UnusedFormalParameter)
4870
*/
4971
public function modifyPrice(IndexTableStructure $priceTable, array $entityIds = [])
@@ -52,7 +74,7 @@ public function modifyPrice(IndexTableStructure $priceTable, array $entityIds =
5274
return;
5375
}
5476

55-
$connection = $this->stockItem->getConnection();
77+
$connection = $this->resourceConnection->getConnection($this->connectionName);
5678
$select = $connection->select();
5779
$select->from(
5880
['price_index' => $priceTable->getTableName()],

‎app/code/Magento/CatalogRule/Model/Indexer/ProductPriceIndexModifier.php

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@
1010
use Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\PriceModifierInterface;
1111
use Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\IndexTableStructure;
1212
use Magento\CatalogRule\Model\ResourceModel\Rule\Product\Price;
13+
use Magento\Framework\App\ResourceConnection;
14+
use Magento\Framework\App\ObjectManager;
1315

1416
/**
1517
* Class for adding catalog rule prices to price index table.
@@ -21,20 +23,38 @@ class ProductPriceIndexModifier implements PriceModifierInterface
2123
*/
2224
private $priceResourceModel;
2325

26+
/**
27+
* @var ResourceConnection
28+
*/
29+
private $resourceConnection;
30+
31+
/**
32+
* @var string
33+
*/
34+
private $connectionName;
35+
2436
/**
2537
* @param Price $priceResourceModel
38+
* @param ResourceConnection $resourceConnection
39+
* @param string $connectionName
2640
*/
27-
public function __construct(Price $priceResourceModel)
28-
{
41+
public function __construct(
42+
Price $priceResourceModel,
43+
ResourceConnection $resourceConnection,
44+
$connectionName = 'indexer'
45+
) {
2946
$this->priceResourceModel = $priceResourceModel;
47+
$this->resourceConnection = $resourceConnection ?: ObjectManager::getInstance()->get(ResourceConnection::class);
48+
$this->connectionName = $connectionName;
3049
}
3150

3251
/**
3352
* @inheritdoc
3453
*/
3554
public function modifyPrice(IndexTableStructure $priceTable, array $entityIds = [])
3655
{
37-
$connection = $this->priceResourceModel->getConnection();
56+
$connection = $this->resourceConnection->getConnection($this->connectionName);
57+
3858
$select = $connection->select();
3959

4060
$select->join(

‎app/code/Magento/ConfigurableProduct/etc/di.xml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,14 @@
176176
<argument name="batchSizeAdjusters" xsi:type="array">
177177
<item name="configurable" xsi:type="object">Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\CompositeProductBatchSizeAdjuster</item>
178178
</argument>
179+
<!--
180+
real batch size will be smaller.
181+
It depends on amount configurable product variations.
182+
E.g for 100 variations real batch size will be 50000/100=500
183+
-->
184+
<argument name="batchRowsCount" xsi:type="array">
185+
<item name="configurable" xsi:type="number">50000</item>
186+
</argument>
179187
</arguments>
180188
</type>
181189
<type name="Magento\ConfigurableProduct\Model\ResourceModel\Product\Indexer\Price\Configurable">

0 commit comments

Comments
 (0)