Skip to content

Commit 05eedc0

Browse files
xmavxmav
committed
Merge pull request #9 from magento-goinc/MAGETWO-47877
MAGETWO-47877: Product Import (HUGE SQL statements/slow) #2957
2 parents 214740f + d54d446 commit 05eedc0

File tree

2 files changed

+78
-86
lines changed

2 files changed

+78
-86
lines changed

‎app/code/Magento/CatalogImportExport/Model/Import/Product.php

Lines changed: 77 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,30 @@ class Product extends \Magento\ImportExport\Model\Import\Entity\AbstractEntity
164164
*/
165165
protected $_attrSetNameToId = [];
166166

167+
/**
168+
* @var string
169+
*/
170+
protected $mediaGalleryTableName;
171+
172+
/**
173+
* @var string
174+
*/
175+
protected $mediaGalleryValueTableName;
176+
/**
177+
* @var string
178+
*/
179+
protected $mediaGalleryEntityToValueTableName;
180+
181+
/**
182+
* @var string
183+
*/
184+
protected $productEntityTableName;
185+
186+
/**
187+
* @var string
188+
*/
189+
protected $productEntityLinkField;
190+
167191
/**
168192
* Attributes with index (not label) value.
169193
*
@@ -1262,80 +1286,61 @@ public function saveProductEntity(array $entityRowsIn, array $entityRowsUp)
12621286
}
12631287

12641288
/**
1265-
* Retrieving images from all columns and rows
1266-
*
1267-
* @param array $bunch
1268-
* @return array
1289+
* Init media gallery resources
1290+
* @return void
12691291
*/
1270-
protected function getBunchImages($bunch)
1292+
protected function initMediaGalleryResources()
12711293
{
1272-
$images = [];
1273-
foreach ($bunch as $row) {
1274-
$row = $this->_customFieldsMapping($row);
1275-
foreach ($this->_imagesArrayKeys as $imageColumn) {
1276-
if (empty($row[$imageColumn])) {
1277-
continue;
1278-
}
1279-
1280-
$rowImages = explode($this->getMultipleValueSeparator(), $row[$imageColumn]);
1281-
foreach ($rowImages as $rowImage) {
1282-
$destinationPath = str_replace('\\', '/', $rowImage);
1283-
$destinationPath = explode('/', $destinationPath);
1284-
$destinationPath = array_pop($destinationPath);
1285-
$destinationPath = preg_replace('/[^a-z0-9\._-]+/i', '', $destinationPath);
1286-
1287-
$dispersion = \Magento\Framework\File\Uploader::getDispretionPath($destinationPath);
1288-
$destinationPath = mb_strtolower($dispersion . '/' . $destinationPath);
1289-
1290-
$images[$rowImage] = $destinationPath;
1291-
}
1292-
}
1294+
if (null == $this->mediaGalleryTableName) {
1295+
$this->productEntityTableName = $this->getResource()->getTable('catalog_product_entity');
1296+
$this->mediaGalleryTableName = $this->getResource()->getTable('catalog_product_entity_media_gallery');
1297+
$this->mediaGalleryValueTableName = $this->getResource()->getTable(
1298+
'catalog_product_entity_media_gallery_value'
1299+
);
1300+
$this->mediaGalleryEntityToValueTableName = $this->getResource()->getTable(
1301+
'catalog_product_entity_media_gallery_value_to_entity'
1302+
);
1303+
$this->productEntityLinkField = $this->metadataPool
1304+
->getMetadata(\Magento\Catalog\Api\Data\ProductInterface::class)
1305+
->getLinkField();
12931306
}
1294-
return $images;
12951307
}
12961308

12971309
/**
1298-
* Prepare all media files
1310+
* Get existing images for current bucnh
12991311
*
1300-
* @param array $images
1312+
* @param array $bunch
13011313
* @return array
13021314
*/
1303-
protected function getExistingImages($images)
1315+
protected function getExistingImages($bunch)
13041316
{
1305-
static $productMediaGalleryTableName = null;
1306-
static $resource = null;
1307-
if (!$resource) {
1308-
$resource = $this->_resourceFactory->create();
1309-
}
1310-
if (!$productMediaGalleryTableName) {
1311-
$productMediaGalleryTableName = $resource->getTable('catalog_product_entity_media_gallery');
1317+
$result = [];
1318+
if ($this->getErrorAggregator()->hasToBeTerminated()) {
1319+
return $result;
13121320
}
1313-
$linkField = $this->metadataPool
1314-
->getMetadata(\Magento\Catalog\Api\Data\ProductInterface::class)
1315-
->getLinkField();
1321+
1322+
$this->initMediaGalleryResources();
1323+
$productSKUs = array_map('strval', array_column($bunch, self::COL_SKU));
13161324
$select = $this->_connection->select()->from(
1317-
['mg' => $resource->getTable('catalog_product_entity_media_gallery')],
1325+
['mg' => $this->mediaGalleryTableName],
13181326
['value' => 'mg.value']
1319-
)->joinLeft(
1320-
['mgvte' => $resource->getTable('catalog_product_entity_media_gallery_value_to_entity')],
1327+
)->joinInner(
1328+
['mgvte' => $this->mediaGalleryEntityToValueTableName],
13211329
'(mg.value_id = mgvte.value_id)',
1322-
[$linkField => 'mgvte.' . $linkField]
1330+
[$this->productEntityLinkField => 'mgvte.' . $this->productEntityLinkField]
1331+
)->joinInner(
1332+
['pe' => $this->productEntityTableName],
1333+
"(mgvte.{$this->productEntityLinkField} = pe.{$this->productEntityLinkField})",
1334+
['sku' => 'pe.sku']
13231335
)->where(
1324-
'mg.value IN(?)',
1325-
$images
1336+
'pe.sku IN (?)',
1337+
$productSKUs
13261338
);
1327-
$allMedia = $this->_connection->fetchAll($select);
1328-
$result = [];
1329-
foreach ($allMedia as $image) {
1330-
if (!isset($result[$image['value']])){
1331-
$result[$image['value']] = [];
1332-
}
1333-
foreach ($this->_oldSku as $sku => $oldSkuData) {
1334-
if ($oldSkuData['entity_id'] == $image['entity_id']) {
1335-
$result[$image['value']][$image['entity_id']] = $sku;
1336-
}
1337-
}
1339+
1340+
foreach ($this->_connection->fetchAll($select) as $image) {
1341+
$result[$image['sku']][$image['value']] = true;
13381342
}
1343+
13391344
return $result;
13401345
}
13411346

@@ -1396,8 +1401,7 @@ protected function _saveProducts()
13961401
$uploadedImages = [];
13971402
$previousType = null;
13981403
$prevAttributeSet = null;
1399-
$bunchImages = $this->getBunchImages($bunch);
1400-
$existingImages = $this->getExistingImages($bunchImages);
1404+
$existingImages = $this->getExistingImages($bunch);
14011405

14021406
foreach ($bunch as $rowNum => $rowData) {
14031407
if (!$this->validateRow($rowData, $rowNum)) {
@@ -1517,8 +1521,7 @@ protected function _saveProducts()
15171521
$rowData[$column] = $uploadedFile;
15181522
}
15191523

1520-
$imageNotAssigned = !isset($existingImages[$uploadedFile])
1521-
|| !in_array($rowSku, $existingImages[$uploadedFile]);
1524+
$imageNotAssigned = !isset($existingImages[$rowSku][$uploadedFile]);
15221525

15231526
if ($uploadedFile && $imageNotAssigned) {
15241527
if ($column == self::COL_MEDIA_IMAGE) {
@@ -1531,7 +1534,7 @@ protected function _saveProducts()
15311534
'disabled' => isset($disabledImages[$columnImage]) ? '1' : '0',
15321535
'value' => $uploadedFile,
15331536
];
1534-
$existingImages[$uploadedFile][] = $rowSku;
1537+
$existingImages[$rowSku][$uploadedFile] = true;
15351538
}
15361539
}
15371540
}
@@ -1812,18 +1815,7 @@ protected function _saveMediaGallery(array $mediaGalleryData)
18121815
if (empty($mediaGalleryData)) {
18131816
return $this;
18141817
}
1815-
static $mediaGalleryTableName = null;
1816-
static $mediaValueTableName = null;
1817-
static $mediaEntityToValueTableName = null;
1818-
$mediaGalleryTableName = $mediaGalleryTableName ?: $this->_resourceFactory->create()->getTable(
1819-
'catalog_product_entity_media_gallery'
1820-
);
1821-
$mediaValueTableName = $mediaValueTableName ?: $this->_resourceFactory->create()->getTable(
1822-
'catalog_product_entity_media_gallery_value'
1823-
);
1824-
$mediaEntityToValueTableName = $mediaEntityToValueTableName ?: $this->_resourceFactory->create()->getTable(
1825-
'catalog_product_entity_media_gallery_value_to_entity'
1826-
);
1818+
$this->initMediaGalleryResources();
18271819
$productIds = [];
18281820
$imageNames = [];
18291821
$multiInsertData = [];
@@ -1832,12 +1824,6 @@ protected function _saveMediaGallery(array $mediaGalleryData)
18321824
$productId = $this->skuProcessor->getNewSku($productSku)['entity_id'];
18331825
$productIds[] = $productId;
18341826
$insertedGalleryImgs = [];
1835-
if (Import::BEHAVIOR_APPEND != $this->getBehavior()) {
1836-
$this->_connection->delete(
1837-
$mediaGalleryTableName,
1838-
$this->_connection->quoteInto('entity_id IN (?)', $productId)
1839-
);
1840-
}
18411827
foreach ($mediaGalleryRows as $insertValue) {
18421828
if (!in_array($insertValue['value'], $insertedGalleryImgs)) {
18431829
$valueArr = [
@@ -1852,17 +1838,18 @@ protected function _saveMediaGallery(array $mediaGalleryData)
18521838
}
18531839
}
18541840
$oldMediaValues = $this->_connection->fetchAssoc(
1855-
$this->_connection->select()->from($mediaGalleryTableName, ['value_id', 'value'])
1841+
$this->_connection->select()->from($this->mediaGalleryTableName, ['value_id', 'value'])
18561842
->where('value IN (?)', $imageNames)
18571843
);
1858-
$this->_connection->insertOnDuplicate($mediaGalleryTableName, $multiInsertData, []);
1844+
$this->_connection->insertOnDuplicate($this->mediaGalleryTableName, $multiInsertData, []);
18591845
$multiInsertData = [];
1860-
$newMediaSelect = $this->_connection->select()->from($mediaGalleryTableName, ['value_id', 'value'])
1846+
$newMediaSelect = $this->_connection->select()->from($this->mediaGalleryTableName, ['value_id', 'value'])
18611847
->where('value IN (?)', $imageNames);
18621848
if (array_keys($oldMediaValues)) {
18631849
$newMediaSelect->where('value_id NOT IN (?)', array_keys($oldMediaValues));
18641850
}
18651851

1852+
$dataForSkinnyTable = [];
18661853
$newMediaValues = $this->_connection->fetchAssoc($newMediaSelect);
18671854
foreach ($mediaGalleryData as $productSku => $mediaGalleryRows) {
18681855
foreach ($mediaGalleryRows as $insertValue) {
@@ -1893,14 +1880,18 @@ protected function _saveMediaGallery(array $mediaGalleryData)
18931880
}
18941881
try {
18951882
$this->_connection->insertOnDuplicate(
1896-
$mediaValueTableName,
1883+
$this->mediaGalleryValueTableName,
18971884
$multiInsertData,
18981885
['value_id', 'store_id', 'entity_id', 'label', 'position', 'disabled']
18991886
);
1900-
$this->_connection->insertOnDuplicate($mediaEntityToValueTableName, $dataForSkinnyTable, ['value_id']);
1887+
$this->_connection->insertOnDuplicate(
1888+
$this->mediaGalleryEntityToValueTableName,
1889+
$dataForSkinnyTable,
1890+
['value_id']
1891+
);
19011892
} catch (\Exception $e) {
19021893
$this->_connection->delete(
1903-
$mediaGalleryTableName,
1894+
$this->mediaGalleryTableName,
19041895
$this->_connection->quoteInto('value_id IN (?)', $newMediaValues)
19051896
);
19061897
}

‎dev/tests/static/testsuite/Magento/Test/Legacy/_files/obsolete_methods.php

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2517,4 +2517,5 @@
25172517
['getFieldsByTabs', 'Magento\Backend\Test\Block\Widget\FormTabs', 'Magento\Ui\Test\Block\Adminhtml\AbstractFormContainers::getFixtureFieldsByContainers'],
25182518
['fillFormTab', 'Magento\Backend\Test\Block\Widget\Tab', 'Magento\Ui\Test\Block\Adminhtml\AbstractContainer::setFieldsData'],
25192519
['getDataFormTab', 'Magento\Backend\Test\Block\Widget\Tab', 'Magento\Ui\Test\Block\Adminhtml\AbstractContainer::getFieldsData'],
2520+
['getBunchImages', 'Magento\CatalogImportExport\Model\Import\Product'],
25202521
];

0 commit comments

Comments
 (0)