MAGETWO-83426: [Performance] Customer Import check data does not complete

Oleksandr Gorkun · Oleksandr Gorkun · commit eab031fb9246 · 2018-02-15T12:52:44.000+02:00
diff --git a/app/code/Magento/CustomerImportExport/Model/ResourceModel/Import/Address/Storage.php b/app/code/Magento/CustomerImportExport/Model/ResourceModel/Import/Address/Storage.php
@@ -5,9 +5,11 @@
  */
 namespace Magento\CustomerImportExport\Model\ResourceModel\Import\Address;
 
-use Magento\Customer\Api\AddressRepositoryInterface;
-use Magento\Framework\Api\FilterBuilder;
-use Magento\Framework\Api\SearchCriteriaBuilder;
+use Magento\Customer\Model\ResourceModel\Address\CollectionFactory as AddressCollectionFactory;
+use Magento\Framework\DataObject;
+use Magento\Framework\DB\Select;
+use Magento\ImportExport\Model\ResourceModel\CollectionByPagesIterator as CollectionIterator;
+use Magento\Customer\Model\ResourceModel\Address\Collection as AddressCollection;
 
 /**
  * Storage to check existing addresses.
@@ -22,33 +24,27 @@ class Storage
     private $addresses = [];
 
     /**
-     * @var AddressRepositoryInterface
+     * @var AddressCollectionFactory
      */
-    private $addressRepository;
+    private $addressCollectionFactory;
 
     /**
-     * @var SearchCriteriaBuilder
-     */
-    private $searchCriteriaBuilder;
-
-    /**
-     * @var FilterBuilder
+     * For iterating over large number of addresses.
+     *
+     * @var CollectionIterator
      */
-    private $filterBuilder;
+    protected $collectionIterator;
 
     /**
-     * @param AddressRepositoryInterface $addressRepository
-     * @param SearchCriteriaBuilder $searchCriteriaBuilder
-     * @param FilterBuilder $filterBuilder
+     * @param AddressCollectionFactory $addressCollectionFactory
+     * @param CollectionIterator $byPagesIterator
      */
     public function __construct(
-        AddressRepositoryInterface $addressRepository,
-        SearchCriteriaBuilder $searchCriteriaBuilder,
-        FilterBuilder $filterBuilder
+        AddressCollectionFactory $addressCollectionFactory,
+        CollectionIterator $byPagesIterator
     ) {
-        $this->addressRepository = $addressRepository;
-        $this->searchCriteriaBuilder = $searchCriteriaBuilder;
-        $this->filterBuilder = $filterBuilder;
+        $this->addressCollectionFactory = $addressCollectionFactory;
+        $this->collectionIterator = $byPagesIterator;
     }
 
     /**
@@ -75,6 +71,33 @@ private function addRecord($customerId, $addressId)
         }
     }
 
+    /**
+     * Load addresses IDs for given customers.
+     *
+     * @param string[] $customerIds
+     *
+     * @return void
+     */
+    private function loadAddresses(array $customerIds)
+    {
+        /** @var AddressCollection $collection */
+        $collection = $this->addressCollectionFactory->create();
+        $collection->removeAttributeToSelect();
+        $select = $collection->getSelect();
+        $tableId = array_keys($select->getPart(Select::FROM))[0];
+        $select->where($tableId .'.parent_id in (?)', $customerIds);
+
+        $this->collectionIterator->iterate(
+            $collection,
+            5000,
+            [
+                function (DataObject $record) {
+                    $this->addRecord($record->getParentId(), $record->getId());
+                }
+            ]
+        );
+    }
+
     /**
      * Check if given address exists for given customer.
      *
@@ -104,24 +127,14 @@ public function prepareAddresses(array $forCustomersIds)
             return;
         }
 
-        $filters = [];
+        $forCustomersIds = array_unique($forCustomersIds);
+        $customerIdsToUse = [];
         foreach ($forCustomersIds as $customerId) {
             if (!array_key_exists((string)$customerId, $this->addresses)) {
-                $filters[] = $this->filterBuilder
-                    ->setField('parent_id')
-                    ->setValue($customerId)
-                    ->setConditionType('eq')
-                    ->create();
+                $customerIdsToUse[] = $customerId;
             }
         }
-        $this->searchCriteriaBuilder->addFilters($filters);
 
-        //Adding addresses that we found.
-        $found = $this->addressRepository->getList(
-            $this->searchCriteriaBuilder->create()
-        );
-        foreach ($found->getItems() as $address) {
-            $this->addRecord($address->getCustomerId(), $address->getId());
-        }
+        $this->loadAddresses($customerIdsToUse);
     }
 }
diff --git a/app/code/Magento/CustomerImportExport/Model/ResourceModel/Import/Customer/Storage.php b/app/code/Magento/CustomerImportExport/Model/ResourceModel/Import/Customer/Storage.php
@@ -5,12 +5,10 @@
  */
 namespace Magento\CustomerImportExport\Model\ResourceModel\Import\Customer;
 
-use Magento\Customer\Api\CustomerRepositoryInterface;
-use Magento\Framework\Api\SearchCriteriaBuilder;
-use Magento\Framework\App\ObjectManager;
 use Magento\Framework\DataObject;
-use Magento\Framework\Exception\NoSuchEntityException;
-use Magento\Framework\Api\FilterBuilder;
+use Magento\Framework\DB\Select;
+use Magento\Customer\Model\ResourceModel\Customer\CollectionFactory as CustomerCollectionFactory;
+use Magento\Customer\Model\ResourceModel\Customer\Collection as CustomerCollection;
 
 class Storage
 {
@@ -25,7 +23,7 @@ class Storage
     /**
      * Customer collection
      *
-     * @var \Magento\Customer\Model\ResourceModel\Customer\Collection
+     * @var CustomerCollection
      * @deprecated
      */
     protected $_customerCollection;
@@ -48,48 +46,30 @@ class Storage
      * Number of items to fetch from db in one query
      *
      * @var int
-     * @deprecated
      */
     protected $_pageSize;
 
     /**
      * Collection by pages iterator
      *
      * @var \Magento\ImportExport\Model\ResourceModel\CollectionByPagesIterator
-     * @deprecated
      */
     protected $_byPagesIterator;
 
     /**
-     * @var CustomerRepositoryInterface
-     */
-    private $customerRepository;
-
-    /**
-     * @var SearchCriteriaBuilder
-     */
-    private $searchCriteriaBuilder;
-
-    /**
-     * @var FilterBuilder
+     * @var CustomerCollectionFactory
      */
-    private $filterBuilder;
+    private $customerCollectionFactory;
 
     /**
-     * @param \Magento\Customer\Model\ResourceModel\Customer\CollectionFactory $collectionFactory
+     * @param CustomerCollectionFactory $collectionFactory
      * @param \Magento\ImportExport\Model\ResourceModel\CollectionByPagesIteratorFactory $colIteratorFactory
      * @param array $data
-     * @param CustomerRepositoryInterface|null $customerRepository
-     * @param SearchCriteriaBuilder|null $searchCriteriaBuilder
-     * @param FilterBuilder|null $filterBuilder
      */
     public function __construct(
-        \Magento\Customer\Model\ResourceModel\Customer\CollectionFactory $collectionFactory,
+        CustomerCollectionFactory $collectionFactory,
         \Magento\ImportExport\Model\ResourceModel\CollectionByPagesIteratorFactory $colIteratorFactory,
-        array $data = [],
-        CustomerRepositoryInterface $customerRepository = null,
-        SearchCriteriaBuilder $searchCriteriaBuilder = null,
-        FilterBuilder $filterBuilder = null
+        array $data = []
     ) {
         $this->_customerCollection = isset(
             $data['customer_collection']
@@ -98,13 +78,7 @@ public function __construct(
         $this->_byPagesIterator = isset(
             $data['collection_by_pages_iterator']
         ) ? $data['collection_by_pages_iterator'] : $colIteratorFactory->create();
-        $this->customerRepository = $customerRepository
-            ?: ObjectManager::getInstance()
-                ->get(CustomerRepositoryInterface::class);
-        $this->searchCriteriaBuilder = $searchCriteriaBuilder
-            ?: ObjectManager::getInstance()->get(SearchCriteriaBuilder::class);
-        $this->filterBuilder = $filterBuilder
-            ?: ObjectManager::getInstance()->get(FilterBuilder::class);
+        $this->customerCollectionFactory = $collectionFactory;
     }
 
     /**
@@ -131,6 +105,49 @@ public function load()
         }
     }
 
+    /**
+     * Create new collection to load customer data with proper filters.
+     *
+     * @param array[] $customerIdentifiers With keys "email" and "website_id".
+     *
+     * @return CustomerCollection
+     */
+    private function prepareCollection(array $customerIdentifiers)
+    {
+        /** @var CustomerCollection $collection */
+        $collection = $this->customerCollectionFactory->create();
+        $collection->removeAttributeToSelect();
+        $select = $collection->getSelect();
+        $customerTableId = array_keys($select->getPart(Select::FROM))[0];
+        $select->where(
+            $customerTableId .'.email in (?)',
+            array_map(
+                function (array $customer) {
+                    return $customer['email'];
+                },
+                $customerIdentifiers
+            )
+        );
+
+        return $collection;
+    }
+
+    /**
+     * Load customers' data that can be found by given identifiers.
+     *
+     * @param array $customerIdentifiers With keys "email" and "website_id".
+     *
+     * @return void
+     */
+    private function loadCustomersData(array $customerIdentifiers)
+    {
+        $this->_byPagesIterator->iterate(
+            $this->prepareCollection($customerIdentifiers),
+            $this->_pageSize,
+            [[$this, 'addCustomer']]
+        );
+    }
+
     /**
      * Add customer to array
      *
@@ -163,21 +180,9 @@ public function getCustomerId($email, $websiteId)
         if (!array_key_exists($email, $this->_customerIds)
             || !array_key_exists($websiteId, $this->_customerIds[$email])
         ) {
-            try {
-                $customer = $this->customerRepository->get($email, $websiteId);
-                $customerData = new DataObject([
-                    'id' => $customer->getId(),
-                    'email' => $customer->getEmail(),
-                    'website_id' => $customer->getWebsiteId()
-                ]);
-            } catch (NoSuchEntityException $exception) {
-                $customerData = new DataObject([
-                    'id' => null,
-                    'email' => $email,
-                    'website_id' => $websiteId
-                ]);
-            }
-            $this->addCustomer($customerData);
+            $this->loadCustomersData([
+                ['email' => $email, 'website_id' => $websiteId]
+            ]);
         }
 
         if (isset($this->_customerIds[$email][$websiteId])) {
@@ -195,47 +200,31 @@ public function getCustomerId($email, $websiteId)
      */
     public function prepareCustomers(array $customersToFind)
     {
-        $customersData = [];
-        $filters = [];
+        $identifiers = [];
         foreach ($customersToFind as $customerToFind) {
             $email = mb_strtolower($customerToFind['email']);
             $websiteId = $customerToFind['website_id'];
             if (!array_key_exists($email, $this->_customerIds)
                 || !array_key_exists($websiteId, $this->_customerIds[$email])
             ) {
                 //Only looking for customers we don't already have ID for.
-                $customersData[] = [
+                $uniqueKey = $email .'_' .$websiteId;
+                $identifiers[$uniqueKey] = [
                     'email' => $email,
                     'website_id' => $websiteId
                 ];
-                $filters[] = $this->filterBuilder
-                    ->setField('email')
-                    ->setValue($email)
-                    ->setConditionType('eq')
-                    ->create();
             }
         }
-        if (!$customersData) {
+        if (!$identifiers) {
             return;
         }
+        //Loading customers data.
+        $this->loadCustomersData($identifiers);
 
-        $this->searchCriteriaBuilder->addFilters($filters);
-
-        //Adding customers that we found.
-        $found = $this->customerRepository->getList(
-            $this->searchCriteriaBuilder->create()
-        );
-        foreach ($found->getItems() as $customer) {
-            $this->addCustomer(new DataObject([
-                'id' => $customer->getId(),
-                'email' => $customer->getEmail(),
-                'website_id' => $customer->getWebsiteId()
-            ]));
-        }
         //Adding customers that don't exist.
-        foreach ($customersData as $customerData) {
-            $email = $customerData['email'];
-            $websiteId = $customerData['website_id'];
+        foreach ($identifiers as $customerIdentity) {
+            $email = $customerIdentity['email'];
+            $websiteId = $customerIdentity['website_id'];
             if (!array_key_exists($email, $this->_customerIds)
                 || !array_key_exists($websiteId, $this->_customerIds[$email])
             ) {
