Skip to content

Commit e4adb6b

Browse files
author
Alexander Akimov
authored
Merge pull request #1005 from magento-folks/category_product_indexer
[Folks] Update ProductCategory indexer
2 parents 2771bb2 + e42dc6e commit e4adb6b

File tree

26 files changed

+1070
-125
lines changed

26 files changed

+1070
-125
lines changed

‎app/code/Magento/Catalog/Model/Indexer/Category/Product/AbstractAction.php

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,10 @@
88

99
namespace Magento\Catalog\Model\Indexer\Category\Product;
1010

11-
use Magento\Framework\App\ResourceConnection;
1211
use Magento\Framework\DB\Query\Generator as QueryGenerator;
12+
use Magento\Framework\App\ResourceConnection;
1313
use Magento\Framework\EntityManager\MetadataPool;
14+
use Magento\Catalog\Model\ResourceModel\Product\Indexer\Category\Product\FrontendResource as CategoryProductFrontend;
1415

1516
/**
1617
* Class AbstractAction
@@ -108,24 +109,33 @@ abstract class AbstractAction
108109
*/
109110
private $queryGenerator;
110111

112+
/**
113+
* @var \Magento\Indexer\Model\ResourceModel\FrontendResource|null
114+
*/
115+
private $categoryProductIndexerFrontend;
116+
111117
/**
112118
* @param ResourceConnection $resource
113119
* @param \Magento\Store\Model\StoreManagerInterface $storeManager
114120
* @param \Magento\Catalog\Model\Config $config
115121
* @param QueryGenerator $queryGenerator
122+
* @param \Magento\Indexer\Model\ResourceModel\FrontendResource|null $categoryProductIndexerFrontend
116123
*/
117124
public function __construct(
118125
\Magento\Framework\App\ResourceConnection $resource,
119126
\Magento\Store\Model\StoreManagerInterface $storeManager,
120127
\Magento\Catalog\Model\Config $config,
121-
QueryGenerator $queryGenerator = null
128+
QueryGenerator $queryGenerator = null,
129+
\Magento\Indexer\Model\ResourceModel\FrontendResource $categoryProductIndexerFrontend = null
122130
) {
123131
$this->resource = $resource;
124132
$this->connection = $resource->getConnection();
125133
$this->storeManager = $storeManager;
126134
$this->config = $config;
127135
$this->queryGenerator = $queryGenerator ?: \Magento\Framework\App\ObjectManager::getInstance()
128136
->get(QueryGenerator::class);
137+
$this->categoryProductIndexerFrontend = $categoryProductIndexerFrontend
138+
?: \Magento\Framework\App\ObjectManager::getInstance()->get(CategoryProductFrontend::class);
129139
}
130140

131141
/**
@@ -165,11 +175,14 @@ protected function getTable($table)
165175
/**
166176
* Return main index table name
167177
*
178+
* This table should be used on frontend(clients)
179+
* The name is switched between 'catalog_category_product_index' and 'catalog_category_product_index_replica'
180+
*
168181
* @return string
169182
*/
170183
protected function getMainTable()
171184
{
172-
return $this->getTable(self::MAIN_INDEX_TABLE);
185+
return $this->categoryProductIndexerFrontend->getMainTable();
173186
}
174187

175188
/**

‎app/code/Magento/Catalog/Model/Indexer/Category/Product/Action/Full.php

Lines changed: 179 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -5,22 +5,97 @@
55
*/
66
namespace Magento\Catalog\Model\Indexer\Category\Product\Action;
77

8+
use Magento\Framework\DB\Query\Generator as QueryGenerator;
9+
use Magento\Framework\App\ResourceConnection;
10+
11+
/**
12+
* Class Full reindex action
13+
*
14+
* @package Magento\Catalog\Model\Indexer\Category\Product\Action
15+
* @SuppressWarnings(PHPMD.CouplingBetweenObjects)
16+
*/
817
class Full extends \Magento\Catalog\Model\Indexer\Category\Product\AbstractAction
918
{
19+
/**
20+
* @var \Magento\Framework\Indexer\BatchSizeManagementInterface
21+
*/
22+
private $batchSizeManagement;
23+
24+
/**
25+
* @var \Magento\Framework\Indexer\BatchProviderInterface
26+
*/
27+
private $batchProvider;
28+
29+
/**
30+
* @var \Magento\Indexer\Model\Indexer\StateFactory
31+
*/
32+
private $indexerStateFactory;
33+
34+
/**
35+
* @var \Magento\Framework\EntityManager\MetadataPool
36+
*/
37+
protected $metadataPool;
38+
39+
/**
40+
* Row count to process in a batch
41+
*
42+
* @var int
43+
*/
44+
private $batchRowsCount;
45+
46+
/**
47+
* @param ResourceConnection $resource
48+
* @param \Magento\Store\Model\StoreManagerInterface $storeManager
49+
* @param \Magento\Catalog\Model\Config $config
50+
* @param QueryGenerator|null $queryGenerator
51+
* @param \Magento\Framework\Indexer\BatchSizeManagementInterface|null $batchSizeManagement
52+
* @param \Magento\Framework\Indexer\BatchProviderInterface|null $batchProvider
53+
* @param \Magento\Framework\EntityManager\MetadataPool|null $metadataPool
54+
* @param \Magento\Indexer\Model\Indexer\StateFactory|null $stateFactory
55+
* @param int|null $batchRowsCount
56+
*/
57+
public function __construct(
58+
\Magento\Framework\App\ResourceConnection $resource,
59+
\Magento\Store\Model\StoreManagerInterface $storeManager,
60+
\Magento\Catalog\Model\Config $config,
61+
QueryGenerator $queryGenerator = null,
62+
\Magento\Framework\Indexer\BatchSizeManagementInterface $batchSizeManagement = null,
63+
\Magento\Framework\Indexer\BatchProviderInterface $batchProvider = null,
64+
\Magento\Framework\EntityManager\MetadataPool $metadataPool = null,
65+
\Magento\Indexer\Model\Indexer\StateFactory $stateFactory = null,
66+
$batchRowsCount = null
67+
) {
68+
parent::__construct(
69+
$resource,
70+
$storeManager,
71+
$config,
72+
$queryGenerator
73+
);
74+
$objectManager = \Magento\Framework\App\ObjectManager::getInstance();
75+
$this->batchSizeManagement = $batchSizeManagement ?: $objectManager->get(
76+
\Magento\Framework\Indexer\BatchSizeManagementInterface::class
77+
);
78+
$this->batchProvider = $batchProvider ?: $objectManager->get(
79+
\Magento\Framework\Indexer\BatchProviderInterface::class
80+
);
81+
$this->metadataPool = $metadataPool ?: $objectManager->get(
82+
\Magento\Framework\EntityManager\MetadataPool::class
83+
);
84+
$this->indexerStateFactory = $stateFactory ?: $objectManager->get(
85+
\Magento\Indexer\Model\Indexer\StateFactory::class
86+
);
87+
$this->batchRowsCount = $batchRowsCount;
88+
}
89+
1090
/**
1191
* Refresh entities index
1292
*
1393
* @return $this
1494
*/
1595
public function execute()
1696
{
17-
$this->clearTmpData();
18-
1997
$this->reindex();
2098

21-
$this->publishData();
22-
$this->removeUnnecessaryData();
23-
2499
return $this;
25100
}
26101

@@ -35,7 +110,7 @@ protected function getSelectUnnecessaryData()
35110
$this->getMainTable(),
36111
[]
37112
)->joinLeft(
38-
['t' => $this->getMainTmpTable()],
113+
['t' => $this->getMainTable()],
39114
$this->getMainTable() .
40115
'.category_id = t.category_id AND ' .
41116
$this->getMainTable() .
@@ -69,27 +144,116 @@ protected function publishData()
69144
{
70145
$select = $this->connection->select()->from($this->getMainTmpTable());
71146

72-
$queries = $this->prepareSelectsByRange($select, 'category_id');
147+
$columns = array_keys($this->connection->describeTable($this->getMainTable()));
148+
149+
$this->connection->query(
150+
$this->connection->insertFromSelect(
151+
$select,
152+
$this->getMainTable(),
153+
$columns,
154+
\Magento\Framework\DB\Adapter\AdapterInterface::INSERT_ON_DUPLICATE
155+
)
156+
);
157+
}
158+
159+
/**
160+
* Clear all index data
161+
*
162+
* @return void
163+
*/
164+
protected function clearTmpData()
165+
{
166+
$this->connection->delete($this->getMainTmpTable());
167+
}
168+
169+
/**
170+
* {@inheritdoc}
171+
*/
172+
protected function reindexRootCategory(\Magento\Store\Model\Store $store)
173+
{
174+
if ($this->isIndexRootCategoryNeeded()) {
175+
$this->reindexCategoriesBySelect($this->getAllProducts($store), 'cp.entity_id IN (?)');
176+
}
177+
}
178+
179+
/**
180+
* Reindex products of anchor categories
181+
*
182+
* @param \Magento\Store\Model\Store $store
183+
* @return void
184+
*/
185+
protected function reindexAnchorCategories(\Magento\Store\Model\Store $store)
186+
{
187+
$this->reindexCategoriesBySelect($this->getAnchorCategoriesSelect($store), 'ccp.product_id IN (?)');
188+
}
189+
190+
/**
191+
* Reindex products of non anchor categories
192+
*
193+
* @param \Magento\Store\Model\Store $store
194+
* @return void
195+
*/
196+
protected function reindexNonAnchorCategories(\Magento\Store\Model\Store $store)
197+
{
198+
$this->reindexCategoriesBySelect($this->getNonAnchorCategoriesSelect($store), 'ccp.product_id IN (?)');
199+
}
73200

74-
foreach ($queries as $query) {
201+
/**
202+
* Reindex categories using given SQL select and condition.
203+
*
204+
* @param \Magento\Framework\DB\Select $basicSelect
205+
* @param string $whereCondition
206+
* @return void
207+
*/
208+
private function reindexCategoriesBySelect(\Magento\Framework\DB\Select $basicSelect, $whereCondition)
209+
{
210+
$entityMetadata = $this->metadataPool->getMetadata(\Magento\Catalog\Api\Data\ProductInterface::class);
211+
$columns = array_keys($this->connection->describeTable($this->getMainTmpTable()));
212+
$this->batchSizeManagement->ensureBatchSize($this->connection, $this->batchRowsCount);
213+
$batches = $this->batchProvider->getBatches(
214+
$this->connection,
215+
$entityMetadata->getEntityTable(),
216+
$entityMetadata->getIdentifierField(),
217+
$this->batchRowsCount
218+
);
219+
foreach ($batches as $batch) {
220+
$this->clearTmpData();
221+
$resultSelect = clone $basicSelect;
222+
$select = $this->connection->select();
223+
$select->distinct(true);
224+
$select->from(['e' => $entityMetadata->getEntityTable()], $entityMetadata->getIdentifierField());
225+
$entityIds = $this->batchProvider->getBatchIds($this->connection, $select, $batch);
226+
$resultSelect->where($whereCondition, $entityIds);
75227
$this->connection->query(
76228
$this->connection->insertFromSelect(
77-
$query,
78-
$this->getMainTable(),
79-
['category_id', 'product_id', 'position', 'is_parent', 'store_id', 'visibility'],
229+
$resultSelect,
230+
$this->getMainTmpTable(),
231+
$columns,
80232
\Magento\Framework\DB\Adapter\AdapterInterface::INSERT_ON_DUPLICATE
81233
)
82234
);
235+
$this->publishData();
236+
$this->removeUnnecessaryData();
83237
}
84238
}
85239

86240
/**
87-
* Clear all index data
241+
* This overridden method returns ALTERNATIVE table name to work with.
88242
*
89-
* @return void
243+
* When the table used on frontend is 'catalog_category_product_index' this indexer should work
244+
* with 'catalog_category_product_index_replica' and vice versa.
245+
*
246+
* @return string table name which is NOT used on frontend
90247
*/
91-
protected function clearTmpData()
248+
protected function getMainTable()
92249
{
93-
$this->connection->delete($this->getMainTmpTable());
250+
$table = $this->getTable(self::MAIN_INDEX_TABLE);
251+
$indexerState = $this->indexerStateFactory->create()->loadByIndexer(
252+
\Magento\Catalog\Model\Indexer\Category\Product::INDEXER_ID
253+
);
254+
$destinationTableSuffix = ($indexerState->getTableSuffix() === '')
255+
? \Magento\Framework\Indexer\StateInterface::ADDITIONAL_TABLE_SUFFIX
256+
: '';
257+
return $table . $destinationTableSuffix;
94258
}
95259
}

‎app/code/Magento/Catalog/Model/Indexer/Category/Product/RowSizeEstimator.php

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
<?php
2+
/**
3+
* Copyright © Magento, Inc. All rights reserved.
4+
* See COPYING.txt for license details.
5+
*/
6+
7+
namespace Magento\Catalog\Model\Indexer\Category\Product;
8+
9+
use Magento\Framework\Indexer\IndexTableRowSizeEstimatorInterface;
10+
11+
/**
12+
* Class RowSizeEstimator
13+
* Intended to estimate amount of memory necessary for saving the biggest category in the DB
14+
* @package Magento\Catalog\Model\Indexer\Category\Product
15+
*/
16+
class RowSizeEstimator implements IndexTableRowSizeEstimatorInterface
17+
{
18+
/**
19+
* Amount of memory for index data row.
20+
*/
21+
const ROW_MEMORY_SIZE = 100;
22+
23+
/**
24+
* @var \Magento\Framework\App\ResourceConnection
25+
*/
26+
private $resourceConnection;
27+
28+
/**
29+
* @param \Magento\Framework\App\ResourceConnection $resourceConnection
30+
*/
31+
public function __construct(
32+
\Magento\Framework\App\ResourceConnection $resourceConnection
33+
) {
34+
$this->resourceConnection = $resourceConnection;
35+
}
36+
37+
/**
38+
* Calculate memory size for largest possible category in database.
39+
*
40+
* Result value is a multiplication of
41+
* a) maximum amount of products in one category
42+
* b) amount of store groups
43+
* c) memory amount per each index row in DB table
44+
*
45+
* {@inheritdoc}
46+
*/
47+
public function estimateRowSize()
48+
{
49+
$connection = $this->resourceConnection->getConnection();
50+
51+
// get store groups count except the default
52+
$storeGroupSelect = $connection->select()
53+
->from(
54+
$this->resourceConnection->getTableName('store_group'),
55+
['count' => new \Zend_Db_Expr('count(*)')]
56+
)->where('group_id > 0');
57+
$storeGroupCount = $connection->fetchOne($storeGroupSelect);
58+
59+
// get max possible categories per product
60+
// subselect with categories count per product
61+
$categoryCounterSubSelect = $connection->select()
62+
->from(
63+
$this->resourceConnection->getTableName('catalog_category_product'),
64+
['counter' => new \Zend_Db_Expr('count(category_id)')]
65+
)->group('product_id');
66+
67+
// select maximum value from subselect
68+
$productCountSelect = $connection->select()
69+
->from(
70+
['counters' => $categoryCounterSubSelect],
71+
[new \Zend_Db_Expr('max(counter)')]
72+
);
73+
$maxProducts = $connection->fetchOne($productCountSelect);
74+
75+
return ceil($storeGroupCount * $maxProducts * self::ROW_MEMORY_SIZE);
76+
}
77+
}

0 commit comments

Comments
 (0)