Skip to content

Commit bb039a6

Browse files
StasKozarStasKozar
committed
Merge remote-tracking branch 'origin/MAGETWO-90808' into 2.3-develop-pr22
2 parents 5630657 + 75b821d commit bb039a6

File tree

12 files changed

+583
-468
lines changed

12 files changed

+583
-468
lines changed

‎app/code/Magento/CustomerImportExport/Model/Import/Address.php

Lines changed: 92 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@
1010
use Magento\Framework\App\ObjectManager;
1111
use Magento\ImportExport\Model\Import\ErrorProcessing\ProcessingErrorAggregatorInterface;
1212
use Magento\Store\Model\Store;
13+
use Magento\CustomerImportExport\Model\ResourceModel\Import\Address\Storage as AddressStorage;
14+
use Magento\ImportExport\Model\Import\AbstractSource;
1315

1416
/**
1517
* Customer address import
@@ -85,20 +87,6 @@ class Address extends AbstractCustomer
8587
*/
8688
protected $_permanentAttributes = [self::COLUMN_WEBSITE, self::COLUMN_EMAIL, self::COLUMN_ADDRESS_ID];
8789

88-
/**
89-
* Existing addresses
90-
*
91-
* Example Array: [customer ID] => array(
92-
* address ID 1,
93-
* address ID 2,
94-
* ...
95-
* address ID N
96-
* )
97-
*
98-
* @var array
99-
*/
100-
protected $_addresses = [];
101-
10290
/**
10391
* Attributes with index (not label) value
10492
*
@@ -180,13 +168,6 @@ class Address extends AbstractCustomer
180168
*/
181169
protected $_attributeCollection;
182170

183-
/**
184-
* Collection of existent addresses
185-
*
186-
* @var \Magento\Customer\Model\ResourceModel\Address\Collection
187-
*/
188-
protected $_addressCollection;
189-
190171
/**
191172
* Store imported row primary keys
192173
*
@@ -252,6 +233,11 @@ class Address extends AbstractCustomer
252233
*/
253234
private $optionsByWebsite = [];
254235

236+
/**
237+
* @var AddressStorage
238+
*/
239+
private $addressStorage;
240+
255241
/**
256242
* @param \Magento\Framework\Stdlib\StringUtils $string
257243
* @param \Magento\Framework\App\Config\ScopeConfigInterface $scopeConfig
@@ -266,12 +252,12 @@ class Address extends AbstractCustomer
266252
* @param \Magento\Customer\Model\AddressFactory $addressFactory
267253
* @param \Magento\Directory\Model\ResourceModel\Region\CollectionFactory $regionColFactory
268254
* @param \Magento\Customer\Model\CustomerFactory $customerFactory
269-
* @param \Magento\Customer\Model\ResourceModel\Address\CollectionFactory $addressColFactory
270255
* @param \Magento\Customer\Model\ResourceModel\Address\Attribute\CollectionFactory $attributesFactory
271256
* @param \Magento\Framework\Stdlib\DateTime $dateTime
272257
* @param \Magento\Customer\Model\Address\Validator\Postcode $postcodeValidator
273258
* @param array $data
274259
* @param CountryWithWebsitesSource|null $countryWithWebsites
260+
* @param AddressStorage|null $addressStorage
275261
*
276262
* @SuppressWarnings(PHPMD.NPathComplexity)
277263
* @SuppressWarnings(PHPMD.ExcessiveParameterList)
@@ -290,12 +276,12 @@ public function __construct(
290276
\Magento\Customer\Model\AddressFactory $addressFactory,
291277
\Magento\Directory\Model\ResourceModel\Region\CollectionFactory $regionColFactory,
292278
\Magento\Customer\Model\CustomerFactory $customerFactory,
293-
\Magento\Customer\Model\ResourceModel\Address\CollectionFactory $addressColFactory,
294279
\Magento\Customer\Model\ResourceModel\Address\Attribute\CollectionFactory $attributesFactory,
295280
\Magento\Framework\Stdlib\DateTime $dateTime,
296281
\Magento\Customer\Model\Address\Validator\Postcode $postcodeValidator,
297282
array $data = [],
298-
CountryWithWebsitesSource $countryWithWebsites = null
283+
CountryWithWebsitesSource $countryWithWebsites = null,
284+
AddressStorage $addressStorage = null
299285
) {
300286
$this->_customerFactory = $customerFactory;
301287
$this->_addressFactory = $addressFactory;
@@ -326,9 +312,6 @@ public function __construct(
326312
$data
327313
);
328314

329-
$this->_addressCollection = isset(
330-
$data['address_collection']
331-
) ? $data['address_collection'] : $addressColFactory->create();
332315
$this->_entityTable = isset(
333316
$data['entity_table']
334317
) ? $data['entity_table'] : $addressFactory->create()->getResource()->getEntityTable();
@@ -346,9 +329,11 @@ public function __construct(
346329
self::ERROR_DUPLICATE_PK,
347330
__('We found another row with this email, website and address ID combination.')
348331
);
332+
$this->addressStorage = $addressStorage
333+
?: ObjectManager::getInstance()->get(AddressStorage::class);
349334

350335
$this->_initAttributes();
351-
$this->_initAddresses()->_initCountryRegions();
336+
$this->_initCountryRegions();
352337
}
353338

354339
/**
@@ -435,27 +420,6 @@ protected function _getNextEntityId()
435420
return $this->_nextEntityId++;
436421
}
437422

438-
/**
439-
* Initialize existent addresses data
440-
*
441-
* @return $this
442-
*/
443-
protected function _initAddresses()
444-
{
445-
/** @var $address \Magento\Customer\Model\Address */
446-
foreach ($this->_addressCollection as $address) {
447-
$customerId = $address->getParentId();
448-
if (!isset($this->_addresses[$customerId])) {
449-
$this->_addresses[$customerId] = [];
450-
}
451-
$addressId = $address->getId();
452-
if (!in_array($addressId, $this->_addresses[$customerId])) {
453-
$this->_addresses[$customerId][] = $addressId;
454-
}
455-
}
456-
return $this;
457-
}
458-
459423
/**
460424
* Initialize country regions hash for clever recognition
461425
*
@@ -475,6 +439,56 @@ protected function _initCountryRegions()
475439
return $this;
476440
}
477441

442+
/**
443+
* Pre-loading customers for existing customers checks in order
444+
* to perform mass validation/import efficiently.
445+
* Also loading existing addresses for requested customers.
446+
*
447+
* @param array|AbstractSource $rows Each row must contain data from columns email
448+
* and website code.
449+
*
450+
* @return void
451+
*/
452+
public function prepareCustomerData($rows): void
453+
{
454+
$customersPresent = [];
455+
foreach ($rows as $rowData) {
456+
$email = $rowData[static::COLUMN_EMAIL] ?? null;
457+
$websiteId = isset($rowData[static::COLUMN_WEBSITE])
458+
? $this->getWebsiteId($rowData[static::COLUMN_WEBSITE]) : false;
459+
if ($email && $websiteId !== false) {
460+
$customersPresent[] = [
461+
'email' => $email,
462+
'website_id' => $websiteId,
463+
];
464+
}
465+
}
466+
$this->getCustomerStorage()->prepareCustomers($customersPresent);
467+
468+
$ids = [];
469+
foreach ($customersPresent as $customerData) {
470+
$id = $this->getCustomerStorage()->getCustomerId(
471+
$customerData['email'],
472+
$customerData['website_id']
473+
);
474+
if ($id) {
475+
$ids[] = $id;
476+
}
477+
}
478+
479+
$this->addressStorage->prepareAddresses($ids);
480+
}
481+
482+
/**
483+
* @inheritDoc
484+
*/
485+
public function validateData()
486+
{
487+
$this->prepareCustomerData($this->getSource());
488+
489+
return parent::validateData();
490+
}
491+
478492
/**
479493
* Import data rows
480494
*
@@ -484,6 +498,16 @@ protected function _initCountryRegions()
484498
*/
485499
protected function _importData()
486500
{
501+
//Preparing data for mass validation/import.
502+
$rows = [];
503+
while ($bunch = $this->_dataSourceModel->getNextBunch()) {
504+
$rows = array_merge($rows, $bunch);
505+
}
506+
$this->prepareCustomerData($rows);
507+
unset($bunch, $rows);
508+
$this->_dataSourceModel->getIterator()->rewind();
509+
510+
//Importing
487511
while ($bunch = $this->_dataSourceModel->getNextBunch()) {
488512
$newRows = [];
489513
$updateRows = [];
@@ -554,7 +578,7 @@ protected function _mergeEntityAttributes(array $newAttributes, array $attribute
554578
* @SuppressWarnings(PHPMD.CyclomaticComplexity)
555579
* @SuppressWarnings(PHPMD.NPathComplexity)
556580
*/
557-
protected function _prepareDataForUpdate(array $rowData)
581+
protected function _prepareDataForUpdate(array $rowData):array
558582
{
559583
$multiSeparator = $this->getMultipleValueSeparator();
560584
$email = strtolower($rowData[self::COLUMN_EMAIL]);
@@ -568,12 +592,11 @@ protected function _prepareDataForUpdate(array $rowData)
568592
$defaults = [];
569593
$newAddress = true;
570594
// get address id
571-
if (isset(
572-
$this->_addresses[$customerId]
573-
) && in_array(
574-
$rowData[self::COLUMN_ADDRESS_ID],
575-
$this->_addresses[$customerId]
576-
)
595+
if ($rowData[self::COLUMN_ADDRESS_ID]
596+
&& $this->addressStorage->doesExist(
597+
$rowData[self::COLUMN_ADDRESS_ID],
598+
(string)$customerId
599+
)
577600
) {
578601
$newAddress = false;
579602
$addressId = $rowData[self::COLUMN_ADDRESS_ID];
@@ -823,12 +846,11 @@ protected function _validateRowForUpdate(array $rowData, $rowNumber)
823846
$rowNumber,
824847
$multiSeparator
825848
);
826-
} elseif ($attributeParams['is_required'] && (!isset(
827-
$this->_addresses[$customerId]
828-
) || !in_array(
829-
$addressId,
830-
$this->_addresses[$customerId]
831-
))
849+
} elseif ($attributeParams['is_required']
850+
&& !$this->addressStorage->doesExist(
851+
(string)$addressId,
852+
(string)$customerId
853+
)
832854
) {
833855
$this->addRowError(self::ERROR_VALUE_IS_REQUIRED, $rowNumber, $attributeCode);
834856
}
@@ -883,7 +905,10 @@ protected function _validateRowForDelete(array $rowData, $rowNumber)
883905
} else {
884906
if (!strlen($addressId)) {
885907
$this->addRowError(self::ERROR_ADDRESS_ID_IS_EMPTY, $rowNumber);
886-
} elseif (!in_array($addressId, $this->_addresses[$customerId])) {
908+
} elseif (!$this->addressStorage->doesExist(
909+
(string)$addressId,
910+
(string)$customerId
911+
)) {
887912
$this->addRowError(self::ERROR_ADDRESS_NOT_FOUND, $rowNumber);
888913
}
889914
}
@@ -899,7 +924,10 @@ protected function _validateRowForDelete(array $rowData, $rowNumber)
899924
*/
900925
protected function _checkRowDuplicate($customerId, $addressId)
901926
{
902-
if (isset($this->_addresses[$customerId]) && in_array($addressId, $this->_addresses[$customerId])) {
927+
if ($this->addressStorage->doesExist(
928+
(string)$addressId,
929+
(string)$customerId
930+
)) {
903931
if (!isset($this->_importedRowPks[$customerId][$addressId])) {
904932
$this->_importedRowPks[$customerId][$addressId] = true;
905933
return false;

‎app/code/Magento/CustomerImportExport/Model/Import/Customer.php

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
use Magento\Customer\Api\Data\CustomerInterface;
99
use Magento\ImportExport\Model\Import;
1010
use Magento\ImportExport\Model\Import\ErrorProcessing\ProcessingErrorAggregatorInterface;
11+
use Magento\ImportExport\Model\Import\AbstractSource;
1112

1213
/**
1314
* Customer entity import
@@ -345,6 +346,40 @@ protected function _getNextEntityId()
345346
return $this->_nextEntityId++;
346347
}
347348

349+
/**
350+
* Prepare customers data for existing customers checks to perform mass validation/import efficiently.
351+
*
352+
* @param array|AbstractSource $rows
353+
*
354+
* @return void
355+
*/
356+
public function prepareCustomerData($rows): void
357+
{
358+
$customersPresent = [];
359+
foreach ($rows as $rowData) {
360+
$email = $rowData[static::COLUMN_EMAIL] ?? null;
361+
$websiteId = isset($rowData[static::COLUMN_WEBSITE])
362+
? $this->getWebsiteId($rowData[static::COLUMN_WEBSITE]) : false;
363+
if ($email && $websiteId !== false) {
364+
$customersPresent[] = [
365+
'email' => $email,
366+
'website_id' => $websiteId,
367+
];
368+
}
369+
}
370+
$this->getCustomerStorage()->prepareCustomers($customersPresent);
371+
}
372+
373+
/**
374+
* @inheritDoc
375+
*/
376+
public function validateData()
377+
{
378+
$this->prepareCustomerData($this->getSource());
379+
380+
return parent::validateData();
381+
}
382+
348383
/**
349384
* Prepare customer data for update
350385
*
@@ -456,6 +491,7 @@ protected function _prepareDataForUpdate(array $rowData)
456491
protected function _importData()
457492
{
458493
while ($bunch = $this->_dataSourceModel->getNextBunch()) {
494+
$this->prepareCustomerData($bunch);
459495
$entitiesToCreate = [];
460496
$entitiesToUpdate = [];
461497
$entitiesToDelete = [];

‎app/code/Magento/CustomerImportExport/Model/Import/CustomerComposite.php

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -287,6 +287,28 @@ public function getEntityTypeCode()
287287
return 'customer_composite';
288288
}
289289

290+
/**
291+
* @inheritDoc
292+
*/
293+
public function validateData()
294+
{
295+
//Preparing both customer and address imports for mass validation.
296+
$source = $this->getSource();
297+
$this->_customerEntity->prepareCustomerData($source);
298+
$source->rewind();
299+
$rows = [];
300+
foreach ($source as $row) {
301+
$rows[] = [
302+
Address::COLUMN_EMAIL => $row[Customer::COLUMN_EMAIL],
303+
Address::COLUMN_WEBSITE => $row[Customer::COLUMN_WEBSITE],
304+
];
305+
}
306+
$source->rewind();
307+
$this->_addressEntity->prepareCustomerData($rows);
308+
309+
return parent::validateData();
310+
}
311+
290312
/**
291313
* Validate data row
292314
*

0 commit comments

Comments
 (0)