Skip to content

Commit 5a20131

Browse files
RomaKisRomaKis
committed
MAGETWO-65362: [Backport] - [Performance] Is there a recommended way to set indexes when importing 1000's of products via .csv - for 2.1.6
1 parent f97c312 commit 5a20131

File tree

6 files changed

+320
-135
lines changed

6 files changed

+320
-135
lines changed

‎app/code/Magento/CatalogImportExport/Model/Import/Product.php

Lines changed: 76 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
use Magento\ImportExport\Model\Import\ErrorProcessing\ProcessingErrorAggregatorInterface;
1919
use Magento\ImportExport\Model\Import\Entity\AbstractEntity;
2020
use Magento\Catalog\Model\Product\Visibility;
21+
2122
/**
2223
* Import entity product model
2324
* @SuppressWarnings(PHPMD.TooManyFields)
@@ -1190,7 +1191,7 @@ protected function _saveLinks()
11901191
}
11911192

11921193
$linkKey = "{$productId}-{$linkedId}-{$linkId}";
1193-
if(empty($productLinkKeys[$linkKey])) {
1194+
if (empty($productLinkKeys[$linkKey])) {
11941195
$productLinkKeys[$linkKey] = $nextLinkId;
11951196
}
11961197
if (!isset($linkRows[$linkKey])) {
@@ -2087,65 +2088,15 @@ protected function _saveProductWebsites(array $websiteData)
20872088
protected function _saveStockItem()
20882089
{
20892090
$indexer = $this->indexerRegistry->get('catalog_product_category');
2091+
20902092
/** @var $stockResource \Magento\CatalogInventory\Model\ResourceModel\Stock\Item */
20912093
$stockResource = $this->_stockResItemFac->create();
20922094
$entityTable = $stockResource->getMainTable();
2093-
while ($bunch = $this->_dataSourceModel->getNextBunch()) {
2094-
$stockData = [];
2095-
$productIdsToReindex = [];
2096-
// Format bunch to stock data rows
2097-
foreach ($bunch as $rowNum => $rowData) {
2098-
if (!$this->isRowAllowedToImport($rowData, $rowNum)) {
2099-
continue;
2100-
}
21012095

2102-
$row = [];
2103-
$row['product_id'] = $this->skuProcessor->getNewSku($rowData[self::COL_SKU])['entity_id'];
2104-
$productIdsToReindex[] = $row['product_id'];
2105-
2106-
$row['website_id'] = $this->stockConfiguration->getDefaultScopeId();
2107-
$row['stock_id'] = $this->stockRegistry->getStock($row['website_id'])->getStockId();
2108-
2109-
$stockItemDo = $this->stockRegistry->getStockItem($row['product_id'], $row['website_id']);
2110-
$existStockData = $stockItemDo->getData();
2111-
2112-
$row = array_merge(
2113-
$this->defaultStockData,
2114-
array_intersect_key($existStockData, $this->defaultStockData),
2115-
array_intersect_key($rowData, $this->defaultStockData),
2116-
$row
2117-
);
2118-
2119-
if ($this->stockConfiguration->isQty(
2120-
$this->skuProcessor->getNewSku($rowData[self::COL_SKU])['type_id']
2121-
)) {
2122-
$stockItemDo->setData($row);
2123-
$row['is_in_stock'] = $this->stockStateProvider->verifyStock($stockItemDo);
2124-
if ($this->stockStateProvider->verifyNotification($stockItemDo)) {
2125-
$row['low_stock_date'] = $this->dateTime->gmDate(
2126-
'Y-m-d H:i:s',
2127-
(new \DateTime())->getTimestamp()
2128-
);
2129-
}
2130-
$row['stock_status_changed_auto'] =
2131-
(int) !$this->stockStateProvider->verifyStock($stockItemDo);
2132-
} else {
2133-
$row['qty'] = 0;
2134-
}
2135-
if (!isset($stockData[$rowData[self::COL_SKU]])) {
2136-
$stockData[$rowData[self::COL_SKU]] = $row;
2137-
}
2138-
}
2139-
2140-
// Insert rows
2141-
if (!empty($stockData)) {
2142-
$this->_connection->insertOnDuplicate($entityTable, array_values($stockData));
2143-
}
2144-
2145-
if ($productIdsToReindex) {
2146-
$indexer->reindexList($productIdsToReindex);
2147-
}
2096+
while ($bunch = $this->_dataSourceModel->getNextBunch()) {
2097+
$this->formatBunchToStockDataRows($bunch, $entityTable, $indexer);
21482098
}
2099+
21492100
return $this;
21502101
}
21512102

@@ -2739,4 +2690,74 @@ private function getProductIdentifierField()
27392690
}
27402691
return $this->productEntityIdentifierField;
27412692
}
2693+
2694+
/**
2695+
* Get stock data rows from bunch.
2696+
*
2697+
* @param array $bunch
2698+
* @param string $entityTable
2699+
* @param \Magento\Framework\Indexer\IndexerInterface $indexer
2700+
* @return void
2701+
*/
2702+
private function formatBunchToStockDataRows(
2703+
$bunch,
2704+
$entityTable,
2705+
\Magento\Framework\Indexer\IndexerInterface $indexer
2706+
) {
2707+
$stockData = [];
2708+
$productIdsToReindex = [];
2709+
2710+
foreach ($bunch as $rowNum => $rowData) {
2711+
if (!$this->isRowAllowedToImport($rowData, $rowNum)) {
2712+
continue;
2713+
}
2714+
2715+
$row = [];
2716+
$row['product_id'] = $this->skuProcessor->getNewSku($rowData[self::COL_SKU])['entity_id'];
2717+
$row['website_id'] = $this->stockConfiguration->getDefaultScopeId();
2718+
$row['stock_id'] = $this->stockRegistry->getStock($row['website_id'])->getStockId();
2719+
2720+
$productIdsToReindex[] = $row['product_id'];
2721+
$stockItemDo = $this->stockRegistry->getStockItem($row['product_id'], $row['website_id']);
2722+
$existStockData = $stockItemDo->getData();
2723+
2724+
$row = array_merge(
2725+
$this->defaultStockData,
2726+
array_intersect_key($existStockData, $this->defaultStockData),
2727+
array_intersect_key($rowData, $this->defaultStockData),
2728+
$row
2729+
);
2730+
2731+
if ($this->stockConfiguration->isQty(
2732+
$this->skuProcessor->getNewSku($rowData[self::COL_SKU])['type_id']
2733+
)
2734+
) {
2735+
$stockItemDo->setData($row);
2736+
$row['is_in_stock'] = $this->stockStateProvider->verifyStock($stockItemDo);
2737+
if ($this->stockStateProvider->verifyNotification($stockItemDo)) {
2738+
$row['low_stock_date'] = $this->dateTime->gmDate(
2739+
'Y-m-d H:i:s',
2740+
(new \DateTime())->getTimestamp()
2741+
);
2742+
}
2743+
$row['stock_status_changed_auto'] =
2744+
(int)!$this->stockStateProvider->verifyStock($stockItemDo);
2745+
} else {
2746+
$row['qty'] = 0;
2747+
}
2748+
2749+
if (!isset($stockData[$rowData[self::COL_SKU]])) {
2750+
$stockData[$rowData[self::COL_SKU]] = $row;
2751+
}
2752+
}
2753+
2754+
// Insert rows
2755+
if (!empty($stockData)) {
2756+
$this->_connection->insertOnDuplicate($entityTable, array_values($stockData));
2757+
}
2758+
2759+
if ($productIdsToReindex && !$indexer->isScheduled()) {
2760+
$indexer->reindexList($productIdsToReindex);
2761+
}
2762+
}
27422763
}

‎app/code/Magento/CatalogImportExport/Test/Unit/Model/Import/ProductTest.php

Lines changed: 87 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,7 @@
55
*/
66
namespace Magento\CatalogImportExport\Test\Unit\Model\Import;
77

8-
use Magento\CatalogImportExport\Model\Import\Product;
98
use Magento\Framework\App\Filesystem\DirectoryList;
10-
use Magento\Framework\Stdlib\DateTime;
119
use Magento\ImportExport\Model\Import;
1210

1311
/**
@@ -398,7 +396,7 @@ protected function setUp()
398396
'data' => $this->data
399397
]
400398
);
401-
$reflection = new \ReflectionClass('\Magento\CatalogImportExport\Model\Import\Product');
399+
$reflection = new \ReflectionClass(\Magento\CatalogImportExport\Model\Import\Product::class);
402400
$reflectionProperty = $reflection->getProperty('metadataPool');
403401
$reflectionProperty->setAccessible(true);
404402
$reflectionProperty->setValue($this->importProduct, $metadataPoolMock);
@@ -410,7 +408,7 @@ protected function setUp()
410408
protected function _objectConstructor()
411409
{
412410
$this->optionFactory = $this->getMock(
413-
'\Magento\CatalogImportExport\Model\Import\Product\OptionFactory',
411+
\Magento\CatalogImportExport\Model\Import\Product\OptionFactory::class,
414412
['create'],
415413
[],
416414
'',
@@ -1683,4 +1681,89 @@ protected function createModelMockWithErrorAggregator(array $methods = [], array
16831681

16841682
return $importProduct;
16851683
}
1684+
1685+
/**
1686+
* Test indexer not run reindexList in update by schedule mode.
1687+
*
1688+
* @return void
1689+
*/
1690+
public function testStockItemReindexListNotCall()
1691+
{
1692+
$indexer = $this->getMockBuilder(\Magento\Framework\Indexer\IndexerInterface::class)
1693+
->disableOriginalConstructor()
1694+
->getMock();
1695+
1696+
$stockResource = $this->getMockBuilder(\Magento\CatalogInventory\Model\ResourceModel\Stock\Item::class)
1697+
->disableOriginalConstructor()
1698+
->getMock();
1699+
1700+
$stock = $this->getMockBuilder(\Magento\CatalogInventory\Api\Data\StockInterface::class)
1701+
->disableOriginalConstructor()
1702+
->getMock();
1703+
1704+
$stockItem = $this->getMockBuilder(\Magento\CatalogInventory\Api\Data\StockItemInterface::class)
1705+
->setMethods(['getData'])
1706+
->disableOriginalConstructor()
1707+
->getMockForAbstractClass();
1708+
1709+
$this->indexerRegistry->expects($this->once())
1710+
->method('get')
1711+
->with('catalog_product_category')
1712+
->willReturn($indexer);
1713+
1714+
$this->_stockResItemFac->expects($this->once())
1715+
->method('create')
1716+
->willReturn($stockResource);
1717+
1718+
$stockResource->expects($this->once())
1719+
->method('getMainTable')
1720+
->willReturn('mainTable');
1721+
1722+
$this->_dataSourceModel->expects($this->atLeastOnce())
1723+
->method('getNextBunch')
1724+
->willReturnOnConsecutiveCalls(
1725+
[
1726+
0 => [
1727+
'sku' => 'product_dynamic',
1728+
'product_type' => 'simple',
1729+
'_attribute_set' => 'attributeSet1'
1730+
]
1731+
],
1732+
[]
1733+
);
1734+
1735+
$this->validator->expects($this->once())
1736+
->method('isValid')
1737+
->willReturn(true);
1738+
1739+
$this->skuProcessor->expects($this->atLeastOnce())
1740+
->method('getNewSku')
1741+
->willReturn([
1742+
'sku' => 'product_dynamic_3326',
1743+
'type_id' => 'simple',
1744+
'attr_set_code' => 'attributeSet1',
1745+
'entity_id' => 1
1746+
]);
1747+
1748+
$this->stockRegistry->expects($this->once())
1749+
->method('getStock')
1750+
->willReturn($stock);
1751+
1752+
$this->stockRegistry->expects($this->once())
1753+
->method('getStockItem')
1754+
->willReturn($stockItem);
1755+
1756+
$stockItem->expects($this->once())
1757+
->method('getData')
1758+
->willReturn([]);
1759+
1760+
$indexer->expects($this->once())
1761+
->method('isScheduled')
1762+
->willReturn(true);
1763+
1764+
$indexer->expects($this->never())
1765+
->method('reindexList');
1766+
1767+
$this->invokeMethod($this->importProduct, '_saveStockItem');
1768+
}
16861769
}

‎app/code/Magento/ImportExport/Model/Import.php

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -612,7 +612,9 @@ public function invalidateIndex()
612612
foreach (array_keys($relatedIndexers) as $indexerId) {
613613
try {
614614
$indexer = $this->indexerRegistry->get($indexerId);
615-
$indexer->invalidate();
615+
if (!$indexer->isScheduled()) {
616+
$indexer->invalidate();
617+
}
616618
} catch (\InvalidArgumentException $e) {
617619
}
618620
}

0 commit comments

Comments
 (0)