MAGETWO-83426: [Performance] Customer Import check data does not complete

Author: Oleksandr Gorkun

app/code/Magento/CustomerImportExport/Model/Import/AbstractCustomer.php

@@ -174,9 +174,12 @@ protected function _getCustomerId($email, $websiteCode)
     }
 
     /**
-     * Pre-loading customers for existing customers checks.
+     * Pre-loading customers for existing customers checks in order
+     * to perform mass validation/import efficiently.
+     *
+     * @param \Traversable $rows Each row must contain data from columns email
+     * and website code.
      *
-     * @param \Traversable $rows
      * @return void
      */
     public function prepareCustomerData(\Traversable $rows)

app/code/Magento/CustomerImportExport/Model/Import/Address.php

@@ -12,7 +12,6 @@
 use Magento\Eav\Model\Entity\Attribute\AbstractAttribute;
 use Magento\Store\Model\Store;
 use Magento\ImportExport\Model\Import;
-use Zend\Stdlib\ArrayObject;
 
 /**
  * @SuppressWarnings(PHPMD.TooManyFields)
@@ -457,7 +456,7 @@ protected function _getNextEntityId()
      *
      * @return $this
      * @deprecated
-     * @see prepareC
+     * @see prepareCustomerData
      */
     protected function _initAddresses()
     {
@@ -540,28 +539,16 @@ protected function _initCountryRegions()
      */
     protected function _importData()
     {
+        //Preparing data for mass validation/import.
         $rows = [];
-        $customersData = [];
         while ($bunch = $this->_dataSourceModel->getNextBunch()) {
             $rows = array_merge($rows, $bunch);
-            $customersData = array_merge(
-                $customersData,
-                array_map(
-                    function ($row) {
-                        return [
-                            'email' => $row[self::COLUMN_EMAIL],
-                            'website_id' => $this->getWebsiteId(
-                                self::COLUMN_WEBSITE
-                            )
-                        ];
-                    },
-                    $bunch
-                )
-            );
         }
-        $this->prepareCustomerData(new ArrayObject($rows));
+        $this->prepareCustomerData(new \ArrayObject($rows));
         unset($bunch, $rows);
         $this->_dataSourceModel->getIterator()->rewind();
+
+        //Importing
         while ($bunch = $this->_dataSourceModel->getNextBunch()) {
             $newRows = [];
             $updateRows = [];

app/code/Magento/CustomerImportExport/Model/Import/Customer.php

@@ -7,7 +7,6 @@
 
 use Magento\Customer\Api\Data\CustomerInterface;
 use Magento\ImportExport\Model\Import\ErrorProcessing\ProcessingErrorAggregatorInterface;
-use Zend\Stdlib\ArrayObject;
 
 /**
  * @SuppressWarnings(PHPMD.CouplingBetweenObjects)
@@ -429,7 +428,7 @@ protected function _prepareDataForUpdate(array $rowData)
     protected function _importData()
     {
         while ($bunch = $this->_dataSourceModel->getNextBunch()) {
-            $this->prepareCustomerData(new ArrayObject($bunch));
+            $this->prepareCustomerData(new \ArrayObject($bunch));
             $entitiesToCreate = [];
             $entitiesToUpdate = [];
             $entitiesToDelete = [];

app/code/Magento/CustomerImportExport/Model/Import/CustomerComposite.php

@@ -5,9 +5,7 @@
  */
 namespace Magento\CustomerImportExport\Model\Import;
 
-use Magento\Framework\DataObject;
 use Magento\ImportExport\Model\Import\ErrorProcessing\ProcessingErrorAggregatorInterface;
-use Zend\Stdlib\ArrayObject;
 
 /**
  * Import entity customer combined model
@@ -294,6 +292,7 @@ public function getEntityTypeCode()
      */
     public function validateData()
     {
+        //Preparing both customer and address imports for mass validation.
         $source = $this->getSource();
         $this->_customerEntity->prepareCustomerData($source);
         $source->rewind();
@@ -305,7 +304,7 @@ public function validateData()
             ];
         }
         $source->rewind();
-        $this->_addressEntity->prepareCustomerData(new ArrayObject($rows));
+        $this->_addressEntity->prepareCustomerData(new \ArrayObject($rows));
 
         return parent::validateData();
     }

app/code/Magento/CustomerImportExport/Model/ResourceModel/Import/Customer/Storage.php

@@ -143,20 +143,22 @@ public function addCustomer(DataObject $customer)
         if (!isset($this->_customerIds[$email])) {
             $this->_customerIds[$email] = [];
         }
-        $this->_customerIds[$email][$customer->getWebsiteId()] = $customer->getId();
+        $this->_customerIds[$email][$customer->getWebsiteId()]
+            = $customer->getId();
 
         return $this;
     }
 
     /**
-     * Get customer id
+     * Find customer ID for unique pair of email and website ID.
      *
      * @param string $email
      * @param int $websiteId
      * @return bool|int
      */
     public function getCustomerId($email, $websiteId)
     {
+        $email = mb_strtolower($email);
         //Trying to load the customer.
         if (!array_key_exists($email, $this->_customerIds)
             || !array_key_exists($websiteId, $this->_customerIds[$email])
@@ -196,16 +198,19 @@ public function prepareCustomers(array $customersToFind)
         $customersData = [];
         $filters = [];
         foreach ($customersToFind as $customerToFind) {
-            $email = $customerToFind['email'];
+            $email = mb_strtolower($customerToFind['email']);
             $websiteId = $customerToFind['website_id'];
             if (!array_key_exists($email, $this->_customerIds)
                 || !array_key_exists($websiteId, $this->_customerIds[$email])
             ) {
                 //Only looking for customers we don't already have ID for.
-                $customersData[] = $customerToFind;
+                $customersData[] = [
+                    'email' => $email,
+                    'website_id' => $websiteId
+                ];
                 $filters[] = $this->filterBuilder
                     ->setField('email')
-                    ->setValue($customerToFind['email'])
+                    ->setValue($email)
                     ->setConditionType('eq')
                     ->create();
             }

dev/tests/integration/testsuite/Magento/Customer/_files/import_export/customers_for_address_import.php

@@ -4,7 +4,9 @@
  * See COPYING.txt for license details.
  */
 //Create customer
-$customer = \Magento\TestFramework\Helper\Bootstrap::getObjectManager()->create('Magento\Customer\Model\Customer');
+/** @var Magento\Customer\Model\Customer $customer */
+$customer = \Magento\TestFramework\Helper\Bootstrap::getObjectManager()
+    ->create('Magento\Customer\Model\Customer');
 $customer->setWebsiteId(
     0
 )->setEntityId(
@@ -69,5 +71,4 @@
 $addressSecond->isObjectNew(true);
 $customer->addAddress($addressSecond);
 $customer->setDefaultShipping($addressSecond->getId());
-$customer->isObjectNew(true);
 $customer->save();

dev/tests/integration/testsuite/Magento/CustomerImportExport/Model/Import/AddressTest.php

@@ -136,7 +136,11 @@ public function testConstruct()
             $this->_entityAdapter,
             'Addresses must be an array.'
         );
-        $this->assertAttributeNotEmpty('_addresses', $this->_entityAdapter, 'Addresses must not be empty');
+        $this->assertAttributeEmpty(
+            '_addresses',
+            $this->_entityAdapter,
+            'Addresses must not be loaded in constructor'
+        );
 
         // check country regions and regions
         $this->assertAttributeInternalType(
@@ -151,62 +155,6 @@ public function testConstruct()
         $this->assertAttributeNotEmpty('_regions', $this->_entityAdapter, 'Regions must not be empty');
     }
 
-    /**
-     * Test _initAddresses
-     *
-     * @magentoDataFixture Magento/Customer/_files/import_export/customer_with_addresses.php
-     */
-    public function testInitAddresses()
-    {
-        /** @var $objectManager \Magento\TestFramework\ObjectManager */
-        $objectManager = Bootstrap::getObjectManager();
-
-        // get addressed from fixture
-        $customers = $objectManager->get('Magento\Framework\Registry')->registry($this->_fixtureKey);
-        $correctAddresses = [];
-        /** @var $customer \Magento\Customer\Model\Customer */
-        foreach ($customers as $customer) {
-            $correctAddresses[$customer->getId()] = [];
-            /** @var $address \Magento\Customer\Model\Address */
-            foreach ($customer->getAddressesCollection() as $address) {
-                $correctAddresses[$customer->getId()][] = $address->getId();
-            }
-        }
-
-        // invoke _initAddresses
-        $initAddresses = new \ReflectionMethod($this->_testClassName, '_initAddresses');
-        $initAddresses->setAccessible(true);
-        $initAddresses->invoke($this->_entityAdapter);
-
-        // check addresses
-        $this->assertAttributeInternalType(
-            'array',
-            '_addresses',
-            $this->_entityAdapter,
-            'Addresses must be an array.'
-        );
-        $this->assertAttributeNotEmpty('_addresses', $this->_entityAdapter, 'Addresses must not be empty');
-
-        $addressesReflection = new \ReflectionProperty($this->_testClassName, '_addresses');
-        $addressesReflection->setAccessible(true);
-        $testAddresses = $addressesReflection->getValue($this->_entityAdapter);
-
-        $correctCustomerIds = array_keys($correctAddresses);
-        $testCustomerIds = array_keys($testAddresses);
-        sort($correctCustomerIds);
-        sort($testCustomerIds);
-        $this->assertEquals($correctCustomerIds, $testCustomerIds, 'Incorrect customer IDs in addresses array.');
-
-        foreach ($correctCustomerIds as $customerId) {
-            $this->assertInternalType('array', $correctAddresses[$customerId], 'Addresses must be an array.');
-            $correctAddressIds = $correctAddresses[$customerId];
-            $testAddressIds = $testAddresses[$customerId];
-            sort($correctAddressIds);
-            sort($testAddressIds);
-            $this->assertEquals($correctAddressIds, $testAddressIds, 'Incorrect addresses IDs.');
-        }
-    }
-
     /**
      * Test _saveAddressEntity
      *