Skip to content

Commit 7bbefcc

Browse files
omiroshnichenkoomiroshnichenko
committed
Merge branch 'indexers' of github.com:magento-folks/magento2ce into stock_indexer_batching
# Conflicts: # app/code/Magento/ConfigurableProduct/etc/di.xml
2 parents 8f62368 + 757fd49 commit 7bbefcc

File tree

231 files changed

+3957
-1549
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

231 files changed

+3957
-1549
lines changed

‎app/code/Magento/Bundle/etc/di.xml

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -137,10 +137,13 @@
137137
</argument>
138138
</arguments>
139139
</type>
140-
<type name="Magento\Catalog\Model\Indexer\Product\Price\Action\Full">
140+
<type name="Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\BatchSizeCalculator">
141141
<arguments>
142-
<argument name="memoryTablesMinRows" xsi:type="array">
143-
<item name="bundle" xsi:type="number">136</item>
142+
<argument name="batchRowsCount" xsi:type="array">
143+
<item name="bundle" xsi:type="number">100000</item>
144+
</argument>
145+
<argument name="estimators" xsi:type="array">
146+
<item name="bundle" xsi:type="object">Magento\Catalog\Model\Indexer\Price\CompositeProductBatchSizeManagement</item>
144147
</argument>
145148
</arguments>
146149
</type>

‎app/code/Magento/CacheInvalidate/Model/PurgeCache.php

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ public function sendPurgeRequest($tagsPattern)
5858
$headers = [self::HEADER_X_MAGENTO_TAGS_PATTERN => $tagsPattern];
5959
$socketAdapter->setOptions(['timeout' => 10]);
6060
foreach ($servers as $server) {
61+
$headers['Host'] = $server->getHost();
6162
try {
6263
$socketAdapter->connect($server->getHost(), $server->getPort());
6364
$socketAdapter->write(

‎app/code/Magento/CacheInvalidate/Test/Unit/Model/PurgeCacheTest.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ public function testSendPurgeRequest($hosts)
7070
->with($uri->getHost(), $uri->getPort());
7171
$this->socketAdapterMock->expects($this->at($i++))
7272
->method('write')
73-
->with('PURGE', $uri, '1.1', ['X-Magento-Tags-Pattern' => 'tags']);
73+
->with('PURGE', $uri, '1.1', ['X-Magento-Tags-Pattern' => 'tags', 'Host' => $uri->getHost()]);
7474
$i++;
7575
}
7676
$this->socketAdapterMock->expects($this->exactly(count($uris)))

‎app/code/Magento/Catalog/Block/Product/View/Attributes.php

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
namespace Magento\Catalog\Block\Product\View;
1313

1414
use Magento\Catalog\Model\Product;
15+
use Magento\Framework\Phrase;
1516
use Magento\Framework\Pricing\PriceCurrencyInterface;
1617

1718
class Attributes extends \Magento\Framework\View\Element\Template
@@ -85,8 +86,8 @@ public function getAdditionalData(array $excludeAttr = [])
8586
} elseif ($attribute->getFrontendInput() == 'price' && is_string($value)) {
8687
$value = $this->priceCurrency->convertAndFormat($value);
8788
}
88-
89-
if (is_string($value) && strlen($value)) {
89+
90+
if ($value instanceof Phrase || (is_string($value) && strlen($value))) {
9091
$data[$attribute->getAttributeCode()] = [
9192
'label' => __($attribute->getStoreLabel()),
9293
'value' => $value,

‎app/code/Magento/Catalog/Model/Indexer/Product/Flat/TableBuilder.php

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,9 @@ public function build($storeId, $changedIds, $valueFieldSuffix)
7474
$attributes = $this->_productIndexerHelper->getAttributes();
7575
$eavAttributes = $this->_productIndexerHelper->getTablesStructure($attributes);
7676
$entityTableColumns = $eavAttributes[$entityTableName];
77+
$linkField = $this->getMetadataPool()
78+
->getMetadata(\Magento\Catalog\Api\Data\ProductInterface::class)
79+
->getLinkField();
7780

7881
$temporaryEavAttributes = $eavAttributes;
7982

@@ -101,11 +104,11 @@ public function build($storeId, $changedIds, $valueFieldSuffix)
101104
$temporaryTableName = $this->_getTemporaryTableName($tableName);
102105

103106
//Add primary key to temporary table for increase speed of joins in future
104-
$this->_addPrimaryKeyToTable($temporaryTableName);
107+
$this->_addPrimaryKeyToTable($temporaryTableName, $linkField);
105108

106109
//Create temporary table for composite attributes
107110
if (isset($valueTables[$temporaryTableName . $valueFieldSuffix])) {
108-
$this->_addPrimaryKeyToTable($temporaryTableName . $valueFieldSuffix);
111+
$this->_addPrimaryKeyToTable($temporaryTableName . $valueFieldSuffix, $linkField);
109112
}
110113

111114
//Fill temporary tables with attributes grouped by it type

‎app/code/Magento/Catalog/Model/Indexer/Product/Price/AbstractAction.php

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ abstract class AbstractAction
7676
private $productResource;
7777

7878
/**
79-
* @var \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\FrontendResource
79+
* @var \Magento\Indexer\Model\ResourceModel\FrontendResource
8080
*/
8181
private $indexerFrontendResource;
8282

@@ -135,7 +135,7 @@ protected function _syncData(array $processIds = [])
135135
{
136136
// delete invalid rows
137137
$select = $this->_connection->select()->from(
138-
['index_price' => $this->indexerFrontendResource->getMainTable()],
138+
['index_price' => $this->getIndexerDefaultTable()],
139139
null
140140
)->joinLeft(
141141
['ip_tmp' => $this->_defaultIndexerResource->getIdxTable()],
@@ -152,7 +152,7 @@ protected function _syncData(array $processIds = [])
152152

153153
$this->_insertFromTable(
154154
$this->_defaultIndexerResource->getIdxTable(),
155-
$this->indexerFrontendResource->getMainTable()
155+
$this->getIndexerDefaultTable()
156156
);
157157
return $this;
158158
}
@@ -471,7 +471,7 @@ protected function _copyRelationIndexData($parentIds, $excludeIds = null)
471471

472472
if ($children) {
473473
$select = $this->_connection->select()->from(
474-
$this->indexerFrontendResource->getMainTable()
474+
$this->getIndexerDefaultTable()
475475
)->where(
476476
'entity_id IN(?)',
477477
$children
@@ -483,6 +483,19 @@ protected function _copyRelationIndexData($parentIds, $excludeIds = null)
483483
return $this;
484484
}
485485

486+
/**
487+
* Returns default table name for writing
488+
*
489+
* Method support writing to frontend table in case with partial reindex
490+
* and case with writing to replica table during full reindex
491+
*
492+
* @return string
493+
*/
494+
protected function getIndexerDefaultTable()
495+
{
496+
return $this->indexerFrontendResource->getMainTable();
497+
}
498+
486499
/**
487500
* @return string
488501
*/

‎app/code/Magento/Catalog/Model/Indexer/Product/Price/Action/Full.php

Lines changed: 17 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -5,24 +5,21 @@
55
*/
66
namespace Magento\Catalog\Model\Indexer\Product\Price\Action;
77

8+
use Magento\Framework\App\ObjectManager;
9+
810
/**
911
* Class Full reindex action
1012
* @SuppressWarnings(PHPMD.CouplingBetweenObjects)
1113
*/
1214
class Full extends \Magento\Catalog\Model\Indexer\Product\Price\AbstractAction
1315
{
14-
/**
15-
* @var array
16-
*/
17-
private $memoryTablesMinRows;
18-
1916
/**
2017
* @var \Magento\Framework\EntityManager\MetadataPool
2118
*/
2219
private $metadataPool;
2320

2421
/**
25-
* @var \Magento\Framework\Indexer\BatchSizeCalculatorInterface
22+
* @var \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\BatchSizeCalculator
2623
*/
2724
private $batchSizeCalculator;
2825

@@ -42,9 +39,8 @@ class Full extends \Magento\Catalog\Model\Indexer\Product\Price\AbstractAction
4239
* @param \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\DefaultPrice $defaultIndexerResource
4340
* @param \Magento\Indexer\Model\ResourceModel\FrontendResource|null $indexerFrontendResource
4441
* @param \Magento\Framework\EntityManager\MetadataPool|null $metadataPool
45-
* @param \Magento\Framework\Indexer\BatchSizeCalculatorInterface|null $batchSizeCalculator
42+
* @param \Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\BatchSizeCalculator|null $batchSizeCalculator
4643
* @param \Magento\Framework\Indexer\BatchProviderInterface|null $batchProvider
47-
* @param array $memoryTablesMinRows
4844
* @SuppressWarnings(PHPMD.ExcessiveParameterList)
4945
*/
5046
public function __construct(
@@ -58,21 +54,18 @@ public function __construct(
5854
\Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\DefaultPrice $defaultIndexerResource,
5955
\Magento\Indexer\Model\ResourceModel\FrontendResource $indexerFrontendResource = null,
6056
\Magento\Framework\EntityManager\MetadataPool $metadataPool = null,
61-
\Magento\Framework\Indexer\BatchSizeCalculatorInterface $batchSizeCalculator = null,
62-
\Magento\Framework\Indexer\BatchProviderInterface $batchProvider = null,
63-
array $memoryTablesMinRows = []
57+
\Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\BatchSizeCalculator $batchSizeCalculator = null,
58+
\Magento\Framework\Indexer\BatchProviderInterface $batchProvider = null
6459
) {
65-
$objectManager = \Magento\Framework\App\ObjectManager::getInstance();
66-
$this->metadataPool = $metadataPool ?: $objectManager->get(
60+
$this->metadataPool = $metadataPool ?: ObjectManager::getInstance()->get(
6761
\Magento\Framework\EntityManager\MetadataPool::class
6862
);
69-
$this->batchSizeCalculator = $batchSizeCalculator ?: $objectManager->get(
70-
\Magento\Framework\Indexer\BatchSizeCalculatorInterface::class
63+
$this->batchSizeCalculator = $batchSizeCalculator ?: ObjectManager::getInstance()->get(
64+
\Magento\Catalog\Model\ResourceModel\Product\Indexer\Price\BatchSizeCalculator::class
7165
);
72-
$this->batchProvider = $batchProvider ?: $objectManager->get(
66+
$this->batchProvider = $batchProvider ?: ObjectManager::getInstance()->get(
7367
\Magento\Framework\Indexer\BatchProviderInterface::class
7468
);
75-
$this->memoryTablesMinRows = $memoryTablesMinRows;
7669
parent::__construct(
7770
$config,
7871
$storeManager,
@@ -97,27 +90,21 @@ public function __construct(
9790
public function execute($ids = null)
9891
{
9992
try {
100-
$this->_defaultIndexerResource->beginTransaction();
10193
$this->_defaultIndexerResource->getTableStrategy()->setUseIdxTable(false);
10294
$this->_prepareWebsiteDateTable();
103-
$this->_emptyTable($this->_defaultIndexerResource->getIdxTable());
104-
$this->emptyPriceIndexTable();
10595

10696
$entityMetadata = $this->metadataPool->getMetadata(\Magento\Catalog\Api\Data\ProductInterface::class);
10797

10898
/** @var \Magento\Catalog\Model\ResourceModel\Product\Indexer\AbstractIndexer $indexer */
10999
foreach ($this->getTypeIndexers() as $indexer) {
110-
111-
$memoryTableMinRows = isset($this->memoryTablesMinRows[$indexer->getTypeId()])
112-
? $this->memoryTablesMinRows[$indexer->getTypeId()]
113-
: $this->memoryTablesMinRows['default'];
114-
100+
$indexer->getTableStrategy()->setUseIdxTable(false);
115101
$connection = $indexer->getConnection();
102+
116103
$batches = $this->batchProvider->getBatches(
117104
$connection,
118105
$entityMetadata->getEntityTable(),
119106
$entityMetadata->getIdentifierField(),
120-
$this->batchSizeCalculator->estimateBatchSize($connection, $memoryTableMinRows)
107+
$this->batchSizeCalculator->estimateBatchSize($connection, $indexer->getTypeId())
121108
);
122109

123110
foreach ($batches as $batch) {
@@ -146,32 +133,17 @@ public function execute($ids = null)
146133
}
147134
}
148135
}
149-
$this->_defaultIndexerResource->commit();
150136
} catch (\Exception $e) {
151-
$this->_defaultIndexerResource->rollBack();
152137
throw new \Magento\Framework\Exception\LocalizedException(__($e->getMessage()), $e);
153138
}
154139
}
155140

156141
/**
157-
* Remove price index data
158-
*
159-
* Remove all price data from index table for current website
160-
* @return void
142+
* @inheritdoc
143+
* @return string
161144
*/
162-
private function emptyPriceIndexTable()
145+
protected function getIndexerDefaultTable()
163146
{
164-
$select = $this->_connection->select()->from(
165-
['index_price' => $this->_defaultIndexerResource->getMainTable()],
166-
null
167-
)->joinLeft(
168-
['ip_tmp' => $this->_defaultIndexerResource->getIdxTable()],
169-
'index_price.entity_id = ip_tmp.entity_id AND index_price.website_id = ip_tmp.website_id',
170-
[]
171-
)->where(
172-
'ip_tmp.entity_id IS NULL'
173-
);
174-
$sql = $select->deleteFromSelect('index_price');
175-
$this->_connection->query($sql);
147+
return $this->_defaultIndexerResource->getMainTable();
176148
}
177149
}

‎app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/BatchSizeCalculator.php

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
<?php
2+
/**
3+
* Copyright © 2013-2017 Magento, Inc. All rights reserved.
4+
* See COPYING.txt for license details.
5+
*/
6+
7+
namespace Magento\Catalog\Model\ResourceModel\Product\Indexer\Price;
8+
9+
class BatchSizeCalculator
10+
{
11+
/**
12+
* @var array
13+
*/
14+
private $batchRowsCount;
15+
16+
/**
17+
* @var \Magento\Framework\Indexer\BatchSizeManagementInterface[]
18+
*/
19+
private $estimators;
20+
21+
/**
22+
* BatchSizeCalculator constructor.
23+
* @param array $batchRowsCount
24+
* @param array $estimators
25+
*/
26+
public function __construct(array $batchRowsCount, array $estimators)
27+
{
28+
$this->batchRowsCount = $batchRowsCount;
29+
$this->estimators = $estimators;
30+
}
31+
32+
/**
33+
* Composite object for batch size calculators
34+
*
35+
* @param \Magento\Framework\DB\Adapter\AdapterInterface $connection
36+
* @param string $indexerTypeId
37+
* @return int
38+
*/
39+
public function estimateBatchSize(\Magento\Framework\DB\Adapter\AdapterInterface $connection, $indexerTypeId)
40+
{
41+
$batchRowsCount = isset($this->batchRowsCount[$indexerTypeId])
42+
? $this->batchRowsCount[$indexerTypeId]
43+
: $this->batchRowsCount['default'];
44+
45+
/** @var \Magento\Framework\Indexer\BatchSizeManagementInterface $calculator */
46+
$calculator = isset($this->estimators[$indexerTypeId])
47+
? $this->estimators[$indexerTypeId]
48+
: $this->estimators['default'];
49+
50+
$calculator->ensureBatchSize($connection, $batchRowsCount);
51+
52+
return $batchRowsCount;
53+
}
54+
}

‎app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/CompositeProductRowSizeEstimator.php

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
<?php
2+
/**
3+
* Copyright © 2013-2017 Magento, Inc. All rights reserved.
4+
* See COPYING.txt for license details.
5+
*/
6+
7+
namespace Magento\Catalog\Model\ResourceModel\Product\Indexer\Price;
8+
9+
use Magento\Framework\Indexer\IndexTableRowSizeEstimatorInterface;
10+
11+
class CompositeProductRowSizeEstimator implements IndexTableRowSizeEstimatorInterface
12+
{
13+
/**
14+
* @var IndexTableRowSizeEstimator
15+
*/
16+
private $indexTableRowSizeEstimator;
17+
18+
/**
19+
* @var DefaultPrice
20+
*/
21+
private $indexerResource;
22+
23+
/**
24+
* @param DefaultPrice $indexerResource
25+
* @param IndexTableRowSizeEstimator $indexTableRowSizeEstimator
26+
*/
27+
public function __construct(
28+
DefaultPrice $indexerResource,
29+
IndexTableRowSizeEstimator $indexTableRowSizeEstimator
30+
) {
31+
$this->indexerResource = $indexerResource;
32+
$this->indexTableRowSizeEstimator = $indexTableRowSizeEstimator;
33+
}
34+
35+
/**
36+
* Calculate memory size for largest composite product in database.
37+
*
38+
* @inheritdoc
39+
*/
40+
public function estimateRowSize()
41+
{
42+
$connection = $this->indexerResource->getConnection();
43+
$relationSelect = $connection->select();
44+
$relationSelect->from(
45+
['relation' => $this->indexerResource->getTable('catalog_product_relation')],
46+
['count' => new \Zend_Db_Expr('count(relation.child_id)')]
47+
);
48+
$relationSelect->group('parent_id');
49+
50+
$maxSelect = $connection->select();
51+
$maxSelect->from(
52+
['max_value' => $relationSelect],
53+
['count' => new \Zend_Db_Expr('MAX(count)')]
54+
);
55+
$maxRelatedProductCount = $connection->fetchOne($maxSelect);
56+
57+
/**
58+
* Calculate memory size for largest composite product in database.
59+
*
60+
* $maxRelatedProductCount - maximum number of related products
61+
*/
62+
return ceil($maxRelatedProductCount * $this->indexTableRowSizeEstimator->estimateRowSize());
63+
}
64+
}

0 commit comments

Comments
 (0)