Skip to content

Commit c3cb46a

Browse files
author
Alexander Akimov
authored
Merge pull request #1060 from magento-folks/eav_batching
[Folks] MAGETWO-65146: Batch data processing for EAV indexer implementation
2 parents a9e8595 + 27bcc0a commit c3cb46a

File tree

20 files changed

+1018
-98
lines changed

20 files changed

+1018
-98
lines changed

‎app/code/Magento/Catalog/Model/Indexer/Product/Eav/AbstractAction.php

Lines changed: 62 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,15 +28,24 @@ abstract class AbstractAction
2828
protected $_eavDecimalFactory;
2929

3030
/**
31+
* @var array
32+
*/
33+
private $frontendResources;
34+
35+
/**
36+
* AbstractAction constructor.
3137
* @param \Magento\Catalog\Model\ResourceModel\Product\Indexer\Eav\DecimalFactory $eavDecimalFactory
3238
* @param \Magento\Catalog\Model\ResourceModel\Product\Indexer\Eav\SourceFactory $eavSourceFactory
39+
* @param array $frontendResources
3340
*/
3441
public function __construct(
3542
\Magento\Catalog\Model\ResourceModel\Product\Indexer\Eav\DecimalFactory $eavDecimalFactory,
36-
\Magento\Catalog\Model\ResourceModel\Product\Indexer\Eav\SourceFactory $eavSourceFactory
43+
\Magento\Catalog\Model\ResourceModel\Product\Indexer\Eav\SourceFactory $eavSourceFactory,
44+
$frontendResources = []
3745
) {
3846
$this->_eavDecimalFactory = $eavDecimalFactory;
3947
$this->_eavSourceFactory = $eavSourceFactory;
48+
$this->frontendResources = $frontendResources;
4049
}
4150

4251
/**
@@ -84,16 +93,67 @@ public function getIndexer($type)
8493
* Reindex entities
8594
*
8695
* @param null|array|int $ids
96+
* @throws \Exception
8797
* @return void
8898
*/
8999
public function reindex($ids = null)
90100
{
91-
foreach ($this->getIndexers() as $indexer) {
101+
foreach ($this->getIndexers() as $type => $indexer) {
92102
if ($ids === null) {
93103
$indexer->reindexAll();
94104
} else {
105+
if (!is_array($ids)) {
106+
$ids = [$ids];
107+
}
108+
$ids = $this->processRelations($indexer, $ids);
95109
$indexer->reindexEntities($ids);
110+
$resource = isset($this->frontendResources[$type])
111+
? $this->frontendResources[$type]
112+
: $this->frontendResources['default'];
113+
$destinationTable = $resource->getMainTable();
114+
$this->syncData($indexer, $destinationTable, $ids);
96115
}
97116
}
98117
}
118+
119+
/**
120+
* Synchronize data between index storage and original storage
121+
*
122+
* @param \Magento\Catalog\Model\ResourceModel\Product\Indexer\Eav\AbstractEav $indexer
123+
* @param string $destinationTable
124+
* @param array $ids
125+
* @throws \Exception
126+
* @return void
127+
*/
128+
protected function syncData($indexer, $destinationTable, $ids)
129+
{
130+
$connection = $indexer->getConnection();
131+
$connection->beginTransaction();
132+
try {
133+
// remove old index
134+
$where = $connection->quoteInto('entity_id IN(?)', $ids);
135+
$connection->delete($destinationTable, $where);
136+
// insert new index
137+
$indexer->insertFromTable($indexer->getIdxTable(), $destinationTable);
138+
$connection->commit();
139+
} catch (\Exception $e) {
140+
$connection->rollBack();
141+
throw $e;
142+
}
143+
}
144+
145+
/**
146+
* Retrieve product relations by children and parent
147+
*
148+
* @param \Magento\Catalog\Model\ResourceModel\Product\Indexer\Eav\AbstractEav $indexer
149+
* @param array $ids
150+
*
151+
* @return $ids
152+
*/
153+
protected function processRelations($indexer, $ids)
154+
{
155+
$parentIds = $indexer->getRelationsByChild($ids);
156+
$childIds = $indexer->getRelationsByParent($ids);
157+
return array_unique(array_merge($ids, $childIds, $parentIds));
158+
}
99159
}

‎app/code/Magento/Catalog/Model/Indexer/Product/Eav/Action/Full.php

Lines changed: 93 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,49 @@
1010
*/
1111
class Full extends \Magento\Catalog\Model\Indexer\Product\Eav\AbstractAction
1212
{
13+
/**
14+
* @var \Magento\Framework\EntityManager\MetadataPool
15+
*/
16+
private $metadataPool;
17+
18+
/**
19+
* @var \Magento\Framework\Indexer\BatchProviderInterface
20+
*/
21+
private $batchProvider;
22+
23+
/**
24+
* @var \Magento\Catalog\Model\ResourceModel\Product\Indexer\Eav\BatchSizeCalculator
25+
*/
26+
private $batchSizeCalculator;
27+
28+
/**
29+
* @param \Magento\Catalog\Model\ResourceModel\Product\Indexer\Eav\DecimalFactory $eavDecimalFactory
30+
* @param \Magento\Catalog\Model\ResourceModel\Product\Indexer\Eav\SourceFactory $eavSourceFactory
31+
* @param \Magento\Framework\EntityManager\MetadataPool|null $metadataPool
32+
* @param \Magento\Framework\Indexer\BatchProviderInterface|null $batchProvider
33+
* @param \Magento\Catalog\Model\ResourceModel\Product\Indexer\Eav\BatchSizeCalculator $batchSizeCalculator
34+
* @param array $frontendResources
35+
*/
36+
public function __construct(
37+
\Magento\Catalog\Model\ResourceModel\Product\Indexer\Eav\DecimalFactory $eavDecimalFactory,
38+
\Magento\Catalog\Model\ResourceModel\Product\Indexer\Eav\SourceFactory $eavSourceFactory,
39+
\Magento\Framework\EntityManager\MetadataPool $metadataPool = null,
40+
\Magento\Framework\Indexer\BatchProviderInterface $batchProvider = null,
41+
\Magento\Catalog\Model\ResourceModel\Product\Indexer\Eav\BatchSizeCalculator $batchSizeCalculator = null,
42+
array $frontendResources = []
43+
) {
44+
parent::__construct($eavDecimalFactory, $eavSourceFactory, $frontendResources);
45+
$this->metadataPool = $metadataPool ?: \Magento\Framework\App\ObjectManager::getInstance()->get(
46+
\Magento\Framework\EntityManager\MetadataPool::class
47+
);
48+
$this->batchProvider = $batchProvider ?: \Magento\Framework\App\ObjectManager::getInstance()->get(
49+
\Magento\Framework\Indexer\BatchProviderInterface::class
50+
);
51+
$this->batchSizeCalculator = $batchSizeCalculator ?: \Magento\Framework\App\ObjectManager::getInstance()->get(
52+
\Magento\Catalog\Model\ResourceModel\Product\Indexer\Eav\BatchSizeCalculator::class
53+
);
54+
}
55+
1356
/**
1457
* Execute Full reindex
1558
*
@@ -21,9 +64,58 @@ class Full extends \Magento\Catalog\Model\Indexer\Product\Eav\AbstractAction
2164
public function execute($ids = null)
2265
{
2366
try {
24-
$this->reindex();
67+
foreach ($this->getIndexers() as $indexerName => $indexer) {
68+
$connection = $indexer->getConnection();
69+
$mainTable = $indexer->getMainTable();
70+
$connection->truncateTable($mainTable);
71+
$entityMetadata = $this->metadataPool->getMetadata(\Magento\Catalog\Api\Data\ProductInterface::class);
72+
$batches = $this->batchProvider->getBatches(
73+
$connection,
74+
$entityMetadata->getEntityTable(),
75+
$entityMetadata->getIdentifierField(),
76+
$this->batchSizeCalculator->estimateBatchSize($connection, $indexerName)
77+
);
78+
79+
foreach ($batches as $batch) {
80+
/** @var \Magento\Framework\DB\Select $select */
81+
$select = $connection->select();
82+
$select->distinct(true);
83+
$select->from(['e' => $entityMetadata->getEntityTable()], $entityMetadata->getIdentifierField());
84+
$entityIds = $this->batchProvider->getBatchIds($connection, $select, $batch);
85+
if (!empty($entityIds)) {
86+
$indexer->reindexEntities($this->processRelations($indexer, $entityIds));
87+
$this->syncData($indexer, $mainTable);
88+
}
89+
}
90+
}
2591
} catch (\Exception $e) {
2692
throw new \Magento\Framework\Exception\LocalizedException(__($e->getMessage()), $e);
2793
}
2894
}
95+
96+
/**
97+
* @inheritdoc
98+
*/
99+
protected function syncData($indexer, $destinationTable, $ids = null)
100+
{
101+
$connection = $indexer->getConnection();
102+
$connection->beginTransaction();
103+
try {
104+
$sourceTable = $indexer->getIdxTable();
105+
$sourceColumns = array_keys($connection->describeTable($sourceTable));
106+
$targetColumns = array_keys($connection->describeTable($destinationTable));
107+
$select = $connection->select()->from($sourceTable, $sourceColumns);
108+
$query = $connection->insertFromSelect(
109+
$select,
110+
$destinationTable,
111+
$targetColumns,
112+
\Magento\Framework\DB\Adapter\AdapterInterface::INSERT_ON_DUPLICATE
113+
);
114+
$connection->query($query);
115+
$connection->commit();
116+
} catch (\Exception $e) {
117+
$connection->rollBack();
118+
throw $e;
119+
}
120+
}
29121
}

‎app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Eav/AbstractEav.php

Lines changed: 2 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -94,41 +94,11 @@ public function reindexAll()
9494
*/
9595
public function reindexEntities($processIds)
9696
{
97-
$connection = $this->getConnection();
98-
9997
$this->clearTemporaryIndexTable();
10098

101-
if (!is_array($processIds)) {
102-
$processIds = [$processIds];
103-
}
104-
105-
$parentIds = $this->getRelationsByChild($processIds);
106-
if ($parentIds) {
107-
$processIds = array_unique(array_merge($processIds, $parentIds));
108-
}
109-
$childIds = $this->getRelationsByParent($processIds);
110-
if ($childIds) {
111-
$processIds = array_unique(array_merge($processIds, $childIds));
112-
}
113-
11499
$this->_prepareIndex($processIds);
115100
$this->_prepareRelationIndex($processIds);
116101
$this->_removeNotVisibleEntityFromIndex();
117-
118-
$connection->beginTransaction();
119-
try {
120-
// remove old index
121-
$where = $connection->quoteInto('entity_id IN(?)', $processIds);
122-
$mainTable = $this->frontendResource->getMainTable();
123-
$connection->delete($this->frontendResource->getMainTable(), $where);
124-
125-
// insert new index
126-
$this->insertFromTable($this->getIdxTable(), $mainTable);
127-
$connection->commit();
128-
} catch (\Exception $e) {
129-
$connection->rollBack();
130-
throw $e;
131-
}
132102
return $this;
133103
}
134104

@@ -238,7 +208,8 @@ protected function _prepareRelationIndexSelect($parentIds = null)
238208
]
239209
);
240210
if ($parentIds !== null) {
241-
$select->where('e.entity_id IN(?)', $parentIds);
211+
$ids = implode(',', array_map('intval', $parentIds));
212+
$select->where("e.entity_id IN({$ids})");
242213
}
243214

244215
/**

‎app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Eav/BatchSizeCalculator.php

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
<?php
2+
/**
3+
* Copyright © Magento, Inc. All rights reserved.
4+
* See COPYING.txt for license details.
5+
*/
6+
7+
namespace Magento\Catalog\Model\ResourceModel\Product\Indexer\Eav;
8+
9+
use Magento\Framework\DB\Adapter\AdapterInterface;
10+
use Magento\Framework\Exception\NoSuchEntityException;
11+
12+
/**
13+
* Composite batch size calculator for EAV related indexers.
14+
*
15+
* Can be configured to provide batch sizes for different indexer types.
16+
*/
17+
class BatchSizeCalculator
18+
{
19+
/**
20+
* @var array
21+
*/
22+
private $batchSizes;
23+
24+
/**
25+
* @var \Magento\Framework\Indexer\BatchSizeManagement[]
26+
*/
27+
private $batchSizeManagers;
28+
29+
/**
30+
* @param array $batchSizes preferable sizes (number of rows in batch) of batches per index type
31+
* @param array $batchSizeManagers batch managers per index type
32+
*/
33+
public function __construct(
34+
array $batchSizes,
35+
array $batchSizeManagers
36+
) {
37+
$this->batchSizes = $batchSizes;
38+
$this->batchSizeManagers = $batchSizeManagers;
39+
}
40+
41+
/**
42+
* Estimate batch size and ensure that database will be able to handle it properly.
43+
*
44+
* @param AdapterInterface $connection
45+
* @param string $indexerTypeId unique identifier of the indexer
46+
* @return int estimated batch size
47+
* @throws NoSuchEntityException thrown if indexer identifier is not recognized
48+
*/
49+
public function estimateBatchSize(
50+
AdapterInterface $connection,
51+
$indexerTypeId
52+
) {
53+
if (!isset($this->batchSizes[$indexerTypeId])) {
54+
throw NoSuchEntityException::singleField('indexTypeId', $indexerTypeId);
55+
}
56+
$this->batchSizeManagers[$indexerTypeId]->ensureBatchSize($connection, $this->batchSizes[$indexerTypeId]);
57+
58+
return $this->batchSizes[$indexerTypeId];
59+
}
60+
}

‎app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Eav/DecimalRowSizeEstimator.php

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
<?php
2+
/**
3+
* Copyright © Magento, Inc. All rights reserved.
4+
* See COPYING.txt for license details.
5+
*/
6+
7+
namespace Magento\Catalog\Model\ResourceModel\Product\Indexer\Eav;
8+
9+
use Magento\Store\Api\StoreManagementInterface;
10+
use Magento\Framework\Indexer\IndexTableRowSizeEstimatorInterface;
11+
use Magento\Framework\EntityManager\MetadataPool;
12+
use Magento\Catalog\Api\Data\ProductInterface;
13+
14+
/**
15+
* Estimator of the EAV decimal index table row size.
16+
*
17+
* Estimates the amount of memory required to store the index data of the product
18+
* with the highest number of attributes/values.
19+
*
20+
* Can be used with batch size manager to ensure that the batch will be handled correctly by the database.
21+
* @see \Magento\Framework\Indexer\BatchSizeManagement
22+
*/
23+
class DecimalRowSizeEstimator implements IndexTableRowSizeEstimatorInterface
24+
{
25+
/**
26+
* @var Decimal
27+
*/
28+
private $indexerResource;
29+
30+
/**
31+
* @var StoreManagementInterface
32+
*/
33+
private $storeManagement;
34+
35+
/**
36+
* @var MetadataPool
37+
*/
38+
private $metadataPool;
39+
40+
/**
41+
* @param StoreManagementInterface $storeManagement
42+
* @param Decimal $indexerResource
43+
* @param MetadataPool $metadataPool
44+
*/
45+
public function __construct(
46+
StoreManagementInterface $storeManagement,
47+
Decimal $indexerResource,
48+
MetadataPool $metadataPool
49+
) {
50+
$this->storeManagement = $storeManagement;
51+
$this->indexerResource = $indexerResource;
52+
$this->metadataPool = $metadataPool;
53+
}
54+
55+
/**
56+
* @inheritdoc
57+
*/
58+
public function estimateRowSize()
59+
{
60+
$connection = $this->indexerResource->getConnection();
61+
$entityIdField = $this->metadataPool->getMetadata(ProductInterface::class)->getLinkField();
62+
63+
$valueSelect = $connection->select();
64+
$valueSelect->from(
65+
['value_table' => $this->indexerResource->getTable('catalog_product_entity_decimal')],
66+
['count' => new \Zend_Db_Expr('count(value_table.value_id)')]
67+
);
68+
$valueSelect->group([$entityIdField, 'store_id']);
69+
70+
$maxSelect = $connection->select();
71+
$maxSelect->from(
72+
['max_value' => $valueSelect],
73+
['count' => new \Zend_Db_Expr('MAX(count)')]
74+
);
75+
$maxRowsPerStore = $connection->fetchOne($maxSelect);
76+
77+
return ceil($maxRowsPerStore * $this->storeManagement->getCount() * 500);
78+
}
79+
}

0 commit comments

Comments
 (0)