Skip to content

Commit df07532

Browse files
serhii-balkoserhii-balko
committed
MC-31449: Customer migration is too slow
1 parent 293cd41 commit df07532

File tree

5 files changed

+76
-227
lines changed

5 files changed

+76
-227
lines changed

‎app/code/Magento/CustomerImportExport/Model/Import/Address.php

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -480,7 +480,7 @@ public function prepareCustomerData($rows): void
480480

481481
$ids = [];
482482
foreach ($customersPresent as $customerData) {
483-
$id = $this->getCustomerStorage()->getCustomerId(
483+
$id = $this->getCustomerStorage()->getLoadedCustomerId(
484484
$customerData['email'],
485485
$customerData['website_id']
486486
);
@@ -606,9 +606,10 @@ protected function _prepareDataForUpdate(array $rowData): array
606606
$newAddress = true;
607607
// get address id
608608
if ($rowData[self::COLUMN_ADDRESS_ID]
609+
&& $customerId
609610
&& $this->addressStorage->doesExist(
610-
$rowData[self::COLUMN_ADDRESS_ID],
611-
(string)$customerId
611+
(int) $rowData[self::COLUMN_ADDRESS_ID],
612+
$customerId
612613
)
613614
) {
614615
$newAddress = false;
@@ -856,7 +857,7 @@ protected function _validateRowForUpdate(array $rowData, $rowNumber)
856857
if ($this->_checkUniqueKey($rowData, $rowNumber)) {
857858
$email = strtolower($rowData[self::COLUMN_EMAIL]);
858859
$website = $rowData[self::COLUMN_WEBSITE];
859-
$addressId = $rowData[self::COLUMN_ADDRESS_ID];
860+
$addressId = (int) $rowData[self::COLUMN_ADDRESS_ID];
860861
$customerId = $this->_getCustomerId($email, $website);
861862

862863
if ($customerId === false) {
@@ -925,16 +926,16 @@ protected function _validateRowForDelete(array $rowData, $rowNumber)
925926
if ($this->_checkUniqueKey($rowData, $rowNumber)) {
926927
$email = strtolower($rowData[self::COLUMN_EMAIL]);
927928
$website = $rowData[self::COLUMN_WEBSITE];
928-
$addressId = $rowData[self::COLUMN_ADDRESS_ID];
929+
$addressId = (int) $rowData[self::COLUMN_ADDRESS_ID];
929930

930931
$customerId = $this->_getCustomerId($email, $website);
931932
if ($customerId === false) {
932933
$this->addRowError(self::ERROR_CUSTOMER_NOT_FOUND, $rowNumber);
933-
} elseif (!strlen($addressId)) {
934+
} elseif (!$addressId) {
934935
$this->addRowError(self::ERROR_ADDRESS_ID_IS_EMPTY, $rowNumber);
935936
} elseif (!$this->addressStorage->doesExist(
936-
(string)$addressId,
937-
(string)$customerId
937+
$addressId,
938+
$customerId
938939
)) {
939940
$this->addRowError(self::ERROR_ADDRESS_NOT_FOUND, $rowNumber);
940941
}
@@ -948,11 +949,11 @@ protected function _validateRowForDelete(array $rowData, $rowNumber)
948949
* @param int $addressId
949950
* @return bool
950951
*/
951-
protected function _checkRowDuplicate($customerId, $addressId)
952+
protected function _checkRowDuplicate(int $customerId, int $addressId)
952953
{
953954
$isAddressExists = $this->addressStorage->doesExist(
954-
(string)$addressId,
955-
(string)$customerId
955+
$addressId,
956+
$customerId
956957
);
957958

958959
$isPkRowSet = isset($this->_importedRowPks[$customerId][$addressId]);

‎app/code/Magento/CustomerImportExport/Model/ResourceModel/Import/Address/Storage.php

Lines changed: 25 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ class Storage
2323
/**
2424
* IDs of addresses grouped by customer IDs.
2525
*
26-
* @var string[][]
26+
* @var int[][]
2727
*/
2828
private $addresses = [];
2929

@@ -62,16 +62,16 @@ public function __construct(
6262
/**
6363
* Record existing address.
6464
*
65-
* @param string $customerId
66-
* @param string $addressId
67-
*
65+
* @param int $customerId
66+
* @param int $addressId
6867
* @return void
6968
*/
70-
private function addRecord(string $customerId, string $addressId): void
69+
private function addRecord(int $customerId, int $addressId): void
7170
{
7271
if (!$customerId || !$addressId) {
7372
return;
7473
}
74+
7575
if (!array_key_exists($customerId, $this->addresses)) {
7676
$this->addresses[$customerId] = [];
7777
}
@@ -84,7 +84,7 @@ private function addRecord(string $customerId, string $addressId): void
8484
/**
8585
* Load addresses IDs for given customers.
8686
*
87-
* @param string[] $customerIds
87+
* @param int[] $customerIds
8888
*
8989
* @return void
9090
*/
@@ -95,27 +95,31 @@ private function loadAddresses(array $customerIds): void
9595
$collection->removeAttributeToSelect();
9696
$select = $collection->getSelect();
9797
$tableId = array_keys($select->getPart(Select::FROM))[0];
98-
$select->where($tableId .'.parent_id in (?)', $customerIds);
98+
$select->reset(Select::COLUMNS)->columns([$tableId . '.entity_id', $tableId . '.parent_id']);
9999

100-
$this->collectionIterator->iterate(
101-
$collection,
102-
$this->config->getValue(AbstractEntity::XML_PATH_PAGE_SIZE),
103-
[
104-
function (DataObject $record) {
105-
$this->addRecord($record->getParentId(), $record->getId());
106-
}
107-
]
108-
);
100+
$pageSize = $this->config->getValue(AbstractEntity::XML_PATH_PAGE_SIZE);
101+
$getChuck = function (int $offset) use ($customerIds, $pageSize) {
102+
return array_slice($customerIds, $offset, $pageSize);
103+
};
104+
$offset = 0;
105+
for ($idsChunk = $getChuck($offset); !empty($idsChunk); $offset += $pageSize, $idsChunk = $getChuck($offset)) {
106+
$chunkSelect = clone $select;
107+
$chunkSelect->where($tableId .'.parent_id IN (?)', $idsChunk);
108+
$addresses = $collection->getConnection()->fetchAll($chunkSelect);
109+
foreach ($addresses as $address) {
110+
$this->addRecord((int) $address['parent_id'], (int) $address['entity_id']);
111+
}
112+
}
109113
}
110114

111115
/**
112116
* Check if given address exists for given customer.
113117
*
114-
* @param string $addressId
115-
* @param string $forCustomerId
118+
* @param int $addressId
119+
* @param int $forCustomerId
116120
* @return bool
117121
*/
118-
public function doesExist(string $addressId, string $forCustomerId): bool
122+
public function doesExist(int $addressId, int $forCustomerId): bool
119123
{
120124
return array_key_exists($forCustomerId, $this->addresses)
121125
&& in_array(
@@ -128,7 +132,7 @@ public function doesExist(string $addressId, string $forCustomerId): bool
128132
/**
129133
* Pre-load addresses for given customers.
130134
*
131-
* @param string[] $forCustomersIds
135+
* @param int[] $forCustomersIds
132136
* @return void
133137
*/
134138
public function prepareAddresses(array $forCustomersIds): void
@@ -138,13 +142,7 @@ public function prepareAddresses(array $forCustomersIds): void
138142
}
139143

140144
$forCustomersIds = array_unique($forCustomersIds);
141-
$customerIdsToUse = [];
142-
foreach ($forCustomersIds as $customerId) {
143-
if (!array_key_exists((string)$customerId, $this->addresses)) {
144-
$customerIdsToUse[] = $customerId;
145-
}
146-
}
147-
145+
$customerIdsToUse = array_diff($forCustomersIds, array_keys($this->addresses));
148146
$this->loadAddresses($customerIdsToUse);
149147
}
150148
}

‎app/code/Magento/CustomerImportExport/Model/ResourceModel/Import/Customer/Storage.php

Lines changed: 34 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,6 @@
99
use Magento\Customer\Model\ResourceModel\Customer\CollectionFactory as CustomerCollectionFactory;
1010
use Magento\Framework\DataObject;
1111
use Magento\Framework\DB\Select;
12-
use Magento\ImportExport\Model\ResourceModel\CollectionByPagesIterator;
13-
use Magento\ImportExport\Model\ResourceModel\CollectionByPagesIteratorFactory;
1412

1513
/**
1614
* Storage to check existing customers.
@@ -38,13 +36,6 @@ class Storage
3836
*/
3937
protected $_pageSize;
4038

41-
/**
42-
* Collection by pages iterator.
43-
*
44-
* @var CollectionByPagesIterator
45-
*/
46-
protected $_byPagesIterator;
47-
4839
/**
4940
* @var CustomerCollectionFactory
5041
*/
@@ -71,65 +62,47 @@ class Storage
7162

7263
/**
7364
* @param CustomerCollectionFactory $collectionFactory
74-
* @param CollectionByPagesIteratorFactory $colIteratorFactory
7565
* @param array $data
7666
*/
7767
public function __construct(
7868
CustomerCollectionFactory $collectionFactory,
79-
CollectionByPagesIteratorFactory $colIteratorFactory,
8069
array $data = []
8170
) {
8271
$this->_customerCollection = isset(
8372
$data['customer_collection']
8473
) ? $data['customer_collection'] : $collectionFactory->create();
85-
$this->_pageSize = isset($data['page_size']) ? $data['page_size'] : 0;
86-
$this->_byPagesIterator = isset(
87-
$data['collection_by_pages_iterator']
88-
) ? $data['collection_by_pages_iterator'] : $colIteratorFactory->create();
74+
$this->_pageSize = isset($data['page_size']) ? (int) $data['page_size'] : 0;
8975
$this->customerCollectionFactory = $collectionFactory;
9076
}
9177

9278
/**
93-
* Create new collection to load customer data with proper filters.
79+
* Load customer's data that can be found by given identifiers.
9480
*
95-
* @param array[] $customerIdentifiers With keys "email" and "website_id".
96-
*
97-
* @return CustomerCollection
81+
* @param array $customerIdentifiers With keys "email" and "website_id".
82+
* @return void
9883
*/
99-
private function prepareCollection(array $customerIdentifiers): CustomerCollection
84+
private function loadCustomersData(array $customerIdentifiers): void
10085
{
10186
/** @var CustomerCollection $collection */
10287
$collection = $this->customerCollectionFactory->create();
10388
$collection->removeAttributeToSelect();
10489
$select = $collection->getSelect();
10590
$customerTableId = array_keys($select->getPart(Select::FROM))[0];
106-
$select->where(
107-
$customerTableId . '.email in (?)',
108-
array_map(
109-
function (array $customer) {
110-
return $customer['email'];
111-
},
112-
$customerIdentifiers
113-
)
114-
);
11591

116-
return $collection;
117-
}
118-
119-
/**
120-
* Load customers' data that can be found by given identifiers.
121-
*
122-
* @param array $customerIdentifiers With keys "email" and "website_id".
123-
*
124-
* @return void
125-
*/
126-
private function loadCustomersData(array $customerIdentifiers)
127-
{
128-
$this->_byPagesIterator->iterate(
129-
$this->prepareCollection($customerIdentifiers),
130-
$this->_pageSize,
131-
[[$this, 'addCustomer']]
132-
);
92+
$pageSize = $this->_pageSize ?: count($customerIdentifiers);
93+
$getChuck = function (int $offset) use ($customerIdentifiers, $pageSize) {
94+
return array_slice($customerIdentifiers, $offset, $pageSize);
95+
};
96+
$offset = 0;
97+
for ($chunk = $getChuck($offset); !empty($chunk); $offset += $pageSize, $chunk = $getChuck($offset)) {
98+
$emails = array_column($chunk, 'email');
99+
$chunkSelect = clone $select;
100+
$chunkSelect->where($customerTableId . '.email IN (?)', $emails);
101+
$customers = $collection->getConnection()->fetchAll($chunkSelect);
102+
foreach ($customers as $customer) {
103+
$this->addCustomerByArray($customer);
104+
}
105+
}
133106
}
134107

135108
/**
@@ -147,8 +120,9 @@ public function addCustomerByArray(array $customer): Storage
147120
if (!isset($this->customerStoreIds[$email])) {
148121
$this->customerStoreIds[$email] = [];
149122
}
150-
$this->_customerIds[$email][$customer['website_id']] = $customer['entity_id'];
151-
$this->customerStoreIds[$email][$customer['website_id']] = $customer['store_id'] ?? null;
123+
$websiteId = (int) $customer['website_id'];
124+
$this->_customerIds[$email][$websiteId] = (int) $customer['entity_id'];
125+
$this->customerStoreIds[$email][$websiteId] = $customer['store_id'] ?? null;
152126

153127
return $this;
154128
}
@@ -190,6 +164,18 @@ public function getCustomerId(string $email, int $websiteId)
190164
return false;
191165
}
192166

167+
/**
168+
* Get previously loaded customer id.
169+
*
170+
* @param string $email
171+
* @param int $websiteId
172+
* @return int|null
173+
*/
174+
public function getLoadedCustomerId(string $email, int $websiteId): ?int
175+
{
176+
return $this->_customerIds[mb_strtolower($email)][$websiteId] ?? null;
177+
}
178+
193179
/**
194180
* Find customer store ID for unique pair of email and website ID.
195181
*

0 commit comments

Comments
 (0)