Skip to content

Commit 747db7c

Browse files
kandykandy
authored
Merge pull request #2333 from magento-performance/MAGETWO-89545
[performance] MAGETWO-89545: Implement segmentation for Category Product Indexer
2 parents 9d489d0 + 5ea71dd commit 747db7c

File tree

153 files changed

+1609
-410
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

153 files changed

+1609
-410
lines changed

‎app/code/Magento/Catalog/Model/Indexer/Category/Product/AbstractAction.php

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
use Magento\Framework\DB\Select;
1717
use Magento\Framework\EntityManager\MetadataPool;
1818
use Magento\Store\Model\Store;
19+
use Magento\Catalog\Model\Indexer\Category\Product\TableResolver;
1920

2021
/**
2122
* Class AbstractAction
@@ -39,11 +40,13 @@ abstract class AbstractAction
3940

4041
/**
4142
* Catalog category index table name
43+
* @deprecated
4244
*/
4345
const MAIN_INDEX_TABLE = 'catalog_category_product_index';
4446

4547
/**
4648
* Suffix for table to show it is temporary
49+
* @deprecated
4750
*/
4851
const TEMPORARY_TABLE_SUFFIX = '_tmp';
4952

@@ -108,6 +111,11 @@ abstract class AbstractAction
108111
*/
109112
protected $metadataPool;
110113

114+
/**
115+
* @var TableResolver
116+
*/
117+
protected $tableResolver;
118+
111119
/**
112120
* @var string
113121
* @since 101.0.0
@@ -125,20 +133,23 @@ abstract class AbstractAction
125133
* @param \Magento\Catalog\Model\Config $config
126134
* @param QueryGenerator $queryGenerator
127135
* @param MetadataPool|null $metadataPool
136+
* @param TableResolver|null $tableResolver
128137
*/
129138
public function __construct(
130139
\Magento\Framework\App\ResourceConnection $resource,
131140
\Magento\Store\Model\StoreManagerInterface $storeManager,
132141
\Magento\Catalog\Model\Config $config,
133142
QueryGenerator $queryGenerator = null,
134-
MetadataPool $metadataPool = null
143+
MetadataPool $metadataPool = null,
144+
TableResolver $tableResolver = null
135145
) {
136146
$this->resource = $resource;
137147
$this->connection = $resource->getConnection();
138148
$this->storeManager = $storeManager;
139149
$this->config = $config;
140150
$this->queryGenerator = $queryGenerator ?: ObjectManager::getInstance()->get(QueryGenerator::class);
141151
$this->metadataPool = $metadataPool ?: ObjectManager::getInstance()->get(MetadataPool::class);
152+
$this->tableResolver = $tableResolver ?: ObjectManager::getInstance()->get(TableResolver::class);
142153
}
143154

144155
/**
@@ -182,6 +193,7 @@ protected function getTable($table)
182193
* The name is switched between 'catalog_category_product_index' and 'catalog_category_product_index_replica'
183194
*
184195
* @return string
196+
* @deprecated
185197
*/
186198
protected function getMainTable()
187199
{
@@ -192,6 +204,7 @@ protected function getMainTable()
192204
* Return temporary index table name
193205
*
194206
* @return string
207+
* @deprecated
195208
*/
196209
protected function getMainTmpTable()
197210
{
@@ -200,6 +213,19 @@ protected function getMainTmpTable()
200213
: $this->getMainTable();
201214
}
202215

216+
/**
217+
* Return index table name
218+
*
219+
* @param int $storeId
220+
* @return string
221+
*/
222+
protected function getIndexTable($storeId)
223+
{
224+
return $this->useTempTable
225+
? $this->tableResolver->getMainReplicaTable($storeId)
226+
: $this->tableResolver->getMainTable($storeId);
227+
}
228+
203229
/**
204230
* Return category path by id
205231
*
@@ -421,7 +447,7 @@ protected function reindexNonAnchorCategories(Store $store)
421447
$this->connection->query(
422448
$this->connection->insertFromSelect(
423449
$select,
424-
$this->getMainTmpTable(),
450+
$this->getIndexTable($store->getId()),
425451
['category_id', 'product_id', 'position', 'is_parent', 'store_id', 'visibility'],
426452
\Magento\Framework\DB\Adapter\AdapterInterface::INSERT_ON_DUPLICATE
427453
)
@@ -680,7 +706,7 @@ protected function reindexAnchorCategories(Store $store)
680706
$this->connection->query(
681707
$this->connection->insertFromSelect(
682708
$select,
683-
$this->getMainTmpTable(),
709+
$this->getIndexTable($store->getId()),
684710
['category_id', 'product_id', 'position', 'is_parent', 'store_id', 'visibility'],
685711
\Magento\Framework\DB\Adapter\AdapterInterface::INSERT_ON_DUPLICATE
686712
)
@@ -812,7 +838,7 @@ protected function reindexRootCategory(Store $store)
812838
$this->connection->query(
813839
$this->connection->insertFromSelect(
814840
$select,
815-
$this->getMainTmpTable(),
841+
$this->getIndexTable($store->getId()),
816842
['category_id', 'product_id', 'position', 'is_parent', 'store_id', 'visibility'],
817843
\Magento\Framework\DB\Adapter\AdapterInterface::INSERT_ON_DUPLICATE
818844
)

‎app/code/Magento/Catalog/Model/Indexer/Category/Product/Action/Full.php

Lines changed: 46 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -50,11 +50,6 @@ class Full extends \Magento\Catalog\Model\Indexer\Category\Product\AbstractActio
5050
*/
5151
private $processManager;
5252

53-
/**
54-
* @var string[]
55-
*/
56-
private $mainTmpTable;
57-
5853
/**
5954
* @param ResourceConnection $resource
6055
* @param \Magento\Store\Model\StoreManagerInterface $storeManager
@@ -102,17 +97,35 @@ public function __construct(
10297
}
10398

10499
/**
105-
* Clear the table we'll be writing de-normalized data into
106-
* to prevent archived data getting in the way of actual data.
107-
*
108100
* @return void
109101
*/
110-
private function clearCurrentTable()
102+
private function createTables()
111103
{
112-
$this->connection->delete(
113-
$this->activeTableSwitcher
114-
->getAdditionalTableName($this->getMainTable())
115-
);
104+
foreach ($this->storeManager->getStores() as $store) {
105+
$this->tableResolver->createTablesForStore($store->getId());
106+
}
107+
}
108+
109+
/**
110+
* @return void
111+
*/
112+
private function clearReplicaTables()
113+
{
114+
foreach ($this->storeManager->getStores() as $store) {
115+
$this->connection->truncateTable($this->tableResolver->getMainReplicaTable($store->getId()));
116+
}
117+
}
118+
119+
/**
120+
* @return void
121+
*/
122+
private function switchTables()
123+
{
124+
$tablesToSwitch = [];
125+
foreach ($this->storeManager->getStores() as $store) {
126+
$tablesToSwitch[] = $this->tableResolver->getMainTable($store->getId());
127+
}
128+
$this->activeTableSwitcher->switchTable($this->connection, $tablesToSwitch);
116129
}
117130

118131
/**
@@ -122,9 +135,10 @@ private function clearCurrentTable()
122135
*/
123136
public function execute()
124137
{
125-
$this->clearCurrentTable();
138+
$this->createTables();
139+
$this->clearReplicaTables();
126140
$this->reindex();
127-
$this->activeTableSwitcher->switchTable($this->connection, [$this->getMainTable()]);
141+
$this->switchTables();
128142
return $this;
129143
}
130144

@@ -161,55 +175,18 @@ private function reindexStore($store)
161175
}
162176

163177
/**
164-
* Return select for remove unnecessary data
165-
*
166-
* @return \Magento\Framework\DB\Select
167-
* @deprecated 102.0.1 Not needed anymore.
168-
*/
169-
protected function getSelectUnnecessaryData()
170-
{
171-
return $this->connection->select()->from(
172-
$this->getMainTable(),
173-
[]
174-
)->joinLeft(
175-
['t' => $this->getMainTable()],
176-
$this->getMainTable() .
177-
'.category_id = t.category_id AND ' .
178-
$this->getMainTable() .
179-
'.store_id = t.store_id AND ' .
180-
$this->getMainTable() .
181-
'.product_id = t.product_id',
182-
[]
183-
)->where(
184-
't.category_id IS NULL'
185-
);
186-
}
187-
188-
/**
189-
* Remove unnecessary data
190-
*
191-
* @return void
192-
*
193-
* @deprecated 102.0.1 Not needed anymore.
194-
*/
195-
protected function removeUnnecessaryData()
196-
{
197-
//Called for backwards compatibility.
198-
$this->getSelectUnnecessaryData();
199-
//This method is useless,
200-
//left it here just in case somebody's using it in child classes.
201-
}
202-
203-
/**
204-
* Publish data from tmp to index
178+
* Publish data from tmp to replica table
205179
*
180+
* @param \Magento\Store\Model\Store $store
206181
* @return void
207182
*/
208-
protected function publishData()
183+
private function publishData($store)
209184
{
210-
$select = $this->connection->select()->from($this->getMainTmpTable());
211-
$columns = array_keys($this->connection->describeTable($this->getMainTable()));
212-
$tableName = $this->activeTableSwitcher->getAdditionalTableName($this->getMainTable());
185+
$select = $this->connection->select()->from($this->tableResolver->getMainTmpTable($store->getId()));
186+
$columns = array_keys(
187+
$this->connection->describeTable($this->tableResolver->getMainReplicaTable($store->getId()))
188+
);
189+
$tableName = $this->tableResolver->getMainReplicaTable($store->getId());
213190

214191
$this->connection->query(
215192
$this->connection->insertFromSelect(
@@ -221,23 +198,13 @@ protected function publishData()
221198
);
222199
}
223200

224-
/**
225-
* Clear all index data
226-
*
227-
* @return void
228-
*/
229-
protected function clearTmpData()
230-
{
231-
$this->connection->delete($this->getMainTmpTable());
232-
}
233-
234201
/**
235202
* {@inheritdoc}
236203
*/
237204
protected function reindexRootCategory(\Magento\Store\Model\Store $store)
238205
{
239206
if ($this->isIndexRootCategoryNeeded()) {
240-
$this->reindexCategoriesBySelect($this->getAllProducts($store), 'cp.entity_id IN (?)');
207+
$this->reindexCategoriesBySelect($this->getAllProducts($store), 'cp.entity_id IN (?)', $store);
241208
}
242209
}
243210

@@ -249,7 +216,7 @@ protected function reindexRootCategory(\Magento\Store\Model\Store $store)
249216
*/
250217
protected function reindexAnchorCategories(\Magento\Store\Model\Store $store)
251218
{
252-
$this->reindexCategoriesBySelect($this->getAnchorCategoriesSelect($store), 'ccp.product_id IN (?)');
219+
$this->reindexCategoriesBySelect($this->getAnchorCategoriesSelect($store), 'ccp.product_id IN (?)', $store);
253220
}
254221

255222
/**
@@ -260,20 +227,21 @@ protected function reindexAnchorCategories(\Magento\Store\Model\Store $store)
260227
*/
261228
protected function reindexNonAnchorCategories(\Magento\Store\Model\Store $store)
262229
{
263-
$this->reindexCategoriesBySelect($this->getNonAnchorCategoriesSelect($store), 'ccp.product_id IN (?)');
230+
$this->reindexCategoriesBySelect($this->getNonAnchorCategoriesSelect($store), 'ccp.product_id IN (?)', $store);
264231
}
265232

266233
/**
267234
* Reindex categories using given SQL select and condition.
268235
*
269236
* @param \Magento\Framework\DB\Select $basicSelect
270237
* @param string $whereCondition
238+
* @param \Magento\Store\Model\Store $store
271239
* @return void
272240
*/
273-
private function reindexCategoriesBySelect(\Magento\Framework\DB\Select $basicSelect, $whereCondition)
241+
private function reindexCategoriesBySelect(\Magento\Framework\DB\Select $basicSelect, $whereCondition, $store)
274242
{
275243
$entityMetadata = $this->metadataPool->getMetadata(\Magento\Catalog\Api\Data\ProductInterface::class);
276-
$columns = array_keys($this->connection->describeTable($this->getMainTmpTable()));
244+
$columns = array_keys($this->connection->describeTable($this->tableResolver->getMainTmpTable($store->getId())));
277245
$this->batchSizeManagement->ensureBatchSize($this->connection, $this->batchRowsCount);
278246
$batches = $this->batchProvider->getBatches(
279247
$this->connection,
@@ -282,7 +250,7 @@ private function reindexCategoriesBySelect(\Magento\Framework\DB\Select $basicSe
282250
$this->batchRowsCount
283251
);
284252
foreach ($batches as $batch) {
285-
$this->clearTmpData();
253+
$this->connection->delete($this->tableResolver->getMainTmpTable($store->getId()));
286254
$resultSelect = clone $basicSelect;
287255
$select = $this->connection->select();
288256
$select->distinct(true);
@@ -292,30 +260,12 @@ private function reindexCategoriesBySelect(\Magento\Framework\DB\Select $basicSe
292260
$this->connection->query(
293261
$this->connection->insertFromSelect(
294262
$resultSelect,
295-
$this->getMainTmpTable(),
263+
$this->tableResolver->getMainTmpTable($store->getId()),
296264
$columns,
297265
\Magento\Framework\DB\Adapter\AdapterInterface::INSERT_ON_DUPLICATE
298266
)
299267
);
300-
$this->publishData();
301-
$this->removeUnnecessaryData();
302-
}
303-
}
304-
305-
/**
306-
* Create and return temporary index table name for current pid
307-
*
308-
* @return string
309-
*/
310-
protected function getMainTmpTable()
311-
{
312-
$pid = getmypid();
313-
if (!isset($this->mainTmpTable[$pid])) {
314-
$originTableName = $this->connection->getTableName(parent::getMainTmpTable());
315-
$temporaryTableName = $originTableName . $pid;
316-
$this->connection->createTemporaryTableLike($temporaryTableName, $originTableName, true);
317-
$this->mainTmpTable[$pid] = $temporaryTableName;
268+
$this->publishData($store);
318269
}
319-
return $this->mainTmpTable[$pid];
320270
}
321271
}

‎app/code/Magento/Catalog/Model/Indexer/Category/Product/Action/Rows.php

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -36,28 +36,32 @@ public function execute(array $entityIds = [], $useTempTable = false)
3636
/**
3737
* Return array of all category root IDs + tree root ID
3838
*
39-
* @return int[]
39+
* @param \Magento\Store\Model\Store $store
40+
* @return int
4041
*/
41-
protected function getRootCategoryIds()
42+
private function getRootCategoryId($store)
4243
{
43-
$rootIds = [\Magento\Catalog\Model\Category::TREE_ROOT_ID];
44-
foreach ($this->storeManager->getStores() as $store) {
45-
if ($this->getPathFromCategoryId($store->getRootCategoryId())) {
46-
$rootIds[] = $store->getRootCategoryId();
47-
}
44+
$rootId = \Magento\Catalog\Model\Category::TREE_ROOT_ID;
45+
if ($this->getPathFromCategoryId($store->getRootCategoryId())) {
46+
$rootId = $store->getRootCategoryId();
4847
}
49-
return $rootIds;
48+
return $rootId;
5049
}
5150

5251
/**
5352
* Remove index entries before reindexation
5453
*
5554
* @return void
5655
*/
57-
protected function removeEntries()
56+
private function removeEntries()
5857
{
59-
$removalCategoryIds = array_diff($this->limitationByCategories, $this->getRootCategoryIds());
60-
$this->connection->delete($this->getMainTable(), ['category_id IN (?)' => $removalCategoryIds]);
58+
foreach ($this->storeManager->getStores() as $store) {
59+
$removalCategoryIds = array_diff($this->limitationByCategories, [$this->getRootCategoryId($store)]);
60+
$this->connection->delete(
61+
$this->getIndexTable($store->getId()),
62+
['category_id IN (?)' => $removalCategoryIds]
63+
);
64+
}
6165
}
6266

6367
/**

0 commit comments

Comments
 (0)