MAGETWO-87188: Memory error on customer import #97

 - Merge Pull Request magento-engcom/import-export-improvements#97 from dmanners/magento2:memory-error-on-customer-import
 - Merged commits:
   1. f148e35
   2. 62f4722
   3. 3ee8961
   4. 5cc36b7
   5. d45ade0
   6. e653591
   7. c07b97b
   8. 330634b

app/code/Magento/CustomerImportExport/Model/Import/CustomerComposite.php

@@ -308,14 +308,13 @@ public function validateRow(array $rowData, $rowNumber)
                 // Add new customer data into customer storage for address entity instance
                 $websiteId = $this->_customerEntity->getWebsiteId($this->_currentWebsiteCode);
                 if (!$this->_addressEntity->getCustomerStorage()->getCustomerId($this->_currentEmail, $websiteId)) {
-                    $customerData = new \Magento\Framework\DataObject(
+                    $this->_addressEntity->getCustomerStorage()->addCustomerByArray(
                         [
-                            'id' => $this->_nextCustomerId,
+                            'entity_id' => $this->_nextCustomerId,
                             'email' => $this->_currentEmail,
                             'website_id' => $websiteId,
                         ]
                     );
-                    $this->_addressEntity->getCustomerStorage()->addCustomer($customerData);
                     $this->_nextCustomerId++;
                 }
 

app/code/Magento/CustomerImportExport/Model/ResourceModel/Import/Customer/Storage.php

@@ -76,34 +76,47 @@ public function __construct(
     public function load()
     {
         if ($this->_isCollectionLoaded == false) {
-            $collection = clone $this->_customerCollection;
-            $collection->removeAttributeToSelect();
-            $tableName = $collection->getResource()->getEntityTable();
-            $collection->getSelect()->from($tableName, ['entity_id', 'website_id', 'email']);
-
-            $this->_byPagesIterator->iterate(
-                $this->_customerCollection,
-                $this->_pageSize,
-                [[$this, 'addCustomer']]
-            );
+            $connection = $this->_customerCollection->getConnection();
+            $select = $connection->select();
+            $select->from($this->_customerCollection->getMainTable(), ['entity_id', 'website_id', 'email']);
+            $results = $connection->fetchAll($select);
+            foreach ($results as $customer) {
+                $this->addCustomerByArray($customer);
+            }
 
             $this->_isCollectionLoaded = true;
         }
     }
 
+    /**
+     * @param array $customer
+     * @return $this
+     */
+    public function addCustomerByArray(array $customer)
+    {
+        $email = strtolower(trim($customer['email']));
+        if (!isset($this->_customerIds[$email])) {
+            $this->_customerIds[$email] = [];
+        }
+        $this->_customerIds[$email][$customer['website_id']] = $customer['entity_id'];
+
+        return $this;
+    }
+
     /**
      * Add customer to array
      *
+     * @deprecated @see addCustomerByArray
      * @param \Magento\Framework\DataObject|\Magento\Customer\Model\Customer $customer
      * @return $this
      */
     public function addCustomer(\Magento\Framework\DataObject $customer)
     {
-        $email = strtolower(trim($customer->getEmail()));
-        if (!isset($this->_customerIds[$email])) {
-            $this->_customerIds[$email] = [];
+        $customerData = $customer->toArray();
+        if (!isset($customerData['entity_id']) && isset($customer['id'])) {
+            $customerData['entity_id'] = $customerData['id'];
         }
-        $this->_customerIds[$email][$customer->getWebsiteId()] = $customer->getId();
+        $this->addCustomerByArray($customerData);
 
         return $this;
     }

app/code/Magento/CustomerImportExport/Test/Unit/Model/Import/AddressTest.php

@@ -8,6 +8,12 @@
 
 use Magento\CustomerImportExport\Model\Import\Address;
 use Magento\ImportExport\Model\Import\AbstractEntity;
+use Magento\Framework\DB\Select;
+use Magento\Framework\DB\Adapter\AdapterInterface;
+use Magento\Customer\Model\ResourceModel\Customer\Collection;
+use Magento\Customer\Model\ResourceModel\Customer\CollectionFactory;
+use Magento\ImportExport\Model\ResourceModel\CollectionByPagesIteratorFactory;
+use Magento\CustomerImportExport\Model\ResourceModel\Import\Customer\Storage;
 
 /**
  * Class AddressTest
@@ -56,8 +62,8 @@ class AddressTest extends \PHPUnit\Framework\TestCase
      * @var array
      */
     protected $_customers = [
-        ['id' => 1, 'email' => 'test1@email.com', 'website_id' => 1],
-        ['id' => 2, 'email' => 'test2@email.com', 'website_id' => 2],
+        ['entity_id' => 1, 'email' => 'test1@email.com', 'website_id' => 1],
+        ['entity_id' => 2, 'email' => 'test2@email.com', 'website_id' => 2],
     ];
 
     /**
@@ -233,28 +239,54 @@ protected function _createAttrCollectionMock()
      */
     protected function _createCustomerStorageMock()
     {
-        $customerStorage = $this->createPartialMock(
-            \Magento\CustomerImportExport\Model\ResourceModel\Import\Customer\Storage::class,
-            ['load']
-        );
-        $resourceMock = $this->createPartialMock(
-            \Magento\Customer\Model\ResourceModel\Customer::class,
-            ['getIdFieldName']
-        );
-        $resourceMock->expects($this->any())->method('getIdFieldName')->will($this->returnValue('id'));
+        /** @var Select|\PHPUnit_Framework_MockObject_MockObject $selectMock */
+        $selectMock = $this->getMockBuilder(Select::class)
+            ->disableOriginalConstructor()
+            ->setMethods(['from'])
+            ->getMock();
+        $selectMock->expects($this->any())->method('from')->will($this->returnSelf());
+
+        /** @var $connectionMock AdapterInterface|\PHPUnit_Framework_MockObject_MockObject */
+        $connectionMock = $this->getMockBuilder(\Magento\Framework\DB\Adapter\Pdo\Mysql::class)
+            ->disableOriginalConstructor()
+            ->setMethods(['select', 'fetchAll'])
+            ->getMock();
+        $connectionMock->expects($this->any())
+            ->method('select')
+            ->will($this->returnValue($selectMock));
+
+        /** @var Collection|\PHPUnit_Framework_MockObject_MockObject $customerCollection */
+        $customerCollection = $this->getMockBuilder(Collection::class)
+            ->disableOriginalConstructor()
+            ->setMethods(['getConnection'])
+            ->getMock();
+        $customerCollection->expects($this->any())
+            ->method('getConnection')
+            ->will($this->returnValue($connectionMock));
+
+        /** @var CollectionFactory|\PHPUnit_Framework_MockObject_MockObject $collectionFactory */
+        $collectionFactory = $this->getMockBuilder(CollectionFactory::class)
+            ->disableOriginalConstructor()
+            ->setMethods(['create'])
+            ->getMock();
+        $collectionFactory->expects($this->any())
+            ->method('create')
+            ->willReturn($customerCollection);
+
+        /** @var CollectionByPagesIteratorFactory|\PHPUnit_Framework_MockObject_MockObject $byPagesIteratorFactory */
+        $byPagesIteratorFactory = $this->getMockBuilder(CollectionByPagesIteratorFactory::class)
+            ->disableOriginalConstructor()
+            ->setMethods(['create'])
+            ->getMock();
+
+        /** @var Storage|\PHPUnit_Framework_MockObject_MockObject $customerStorage */
+        $customerStorage = $this->getMockBuilder(Storage::class)
+            ->setMethods(['load'])
+            ->setConstructorArgs([$collectionFactory, $byPagesIteratorFactory])
+            ->getMock();
+
         foreach ($this->_customers as $customerData) {
-            $data = [
-                'resource' => $resourceMock,
-                'data' => $customerData,
-                $this->createMock(\Magento\Customer\Model\Config\Share::class),
-                $this->createMock(\Magento\Customer\Model\AddressFactory::class),
-                $this->createMock(\Magento\Customer\Model\ResourceModel\Address\CollectionFactory::class),
-                $this->createMock(\Magento\Customer\Model\GroupFactory::class),
-                $this->createMock(\Magento\Customer\Model\AttributeFactory::class),
-            ];
-            /** @var $customer \Magento\Customer\Model\Customer */
-            $customer = $this->_objectManagerMock->getObject(\Magento\Customer\Model\Customer::class, $data);
-            $customerStorage->addCustomer($customer);
+            $customerStorage->addCustomerByArray($customerData);
         }
         return $customerStorage;
     }

app/code/Magento/CustomerImportExport/Test/Unit/Model/ResourceModel/Import/Customer/StorageTest.php

@@ -6,6 +6,10 @@
 namespace Magento\CustomerImportExport\Test\Unit\Model\ResourceModel\Import\Customer;
 
 use Magento\CustomerImportExport\Model\ResourceModel\Import\Customer\Storage;
+use Magento\Framework\DB\Adapter\AdapterInterface;
+use Magento\Customer\Model\ResourceModel\Customer\Collection;
+use Magento\Customer\Model\ResourceModel\Customer\CollectionFactory;
+use Magento\ImportExport\Model\ResourceModel\CollectionByPagesIteratorFactory;
 
 class StorageTest extends \PHPUnit\Framework\TestCase
 {
@@ -26,74 +30,63 @@ class StorageTest extends \PHPUnit\Framework\TestCase
 
     protected function setUp()
     {
-        $this->_model = new \Magento\CustomerImportExport\Model\ResourceModel\Import\Customer\Storage(
-            $this->getMockBuilder(\Magento\Customer\Model\ResourceModel\Customer\CollectionFactory::class)
-                ->disableOriginalConstructor()
-                ->getMock(),
-            $this->getMockBuilder(\Magento\ImportExport\Model\ResourceModel\CollectionByPagesIteratorFactory::class)
-                ->disableOriginalConstructor()
-                ->getMock(),
-            $this->_getModelDependencies()
-        );
-        $this->_model->load();
-    }
-
-    protected function tearDown()
-    {
-        unset($this->_model);
-    }
-
-    /**
-     * Retrieve all necessary objects mocks which used inside customer storage
-     *
-     * @return array
-     */
-    protected function _getModelDependencies()
-    {
-        $select = $this->getMockBuilder(\Magento\Framework\DB\Select::class)
+        /** @var \Magento\Framework\DB\Select|\PHPUnit_Framework_MockObject_MockObject $selectMock */
+        $selectMock = $this->getMockBuilder(\Magento\Framework\DB\Select::class)
             ->disableOriginalConstructor()
             ->setMethods(['from'])
             ->getMock();
-        $select->expects($this->any())->method('from')->will($this->returnCallback([$this, 'validateFrom']));
-        $customerCollection = $this->getMockBuilder(\Magento\Customer\Model\ResourceModel\Customer\Collection::class)
+        $selectMock->expects($this->any())->method('from')->will($this->returnSelf());
+
+        /** @var $connectionMock AdapterInterface|\PHPUnit_Framework_MockObject_MockObject */
+        $connectionMock = $this->getMockBuilder(\Magento\Framework\DB\Adapter\Pdo\Mysql::class)
+            ->disableOriginalConstructor()
+            ->setMethods(['select', 'fetchAll'])
+            ->getMock();
+        $connectionMock->expects($this->any())
+            ->method('select')
+            ->will($this->returnValue($selectMock));
+        $connectionMock->expects($this->any())
+            ->method('fetchAll')
+            ->will($this->returnValue([]));
+
+        /** @var Collection|\PHPUnit_Framework_MockObject_MockObject $customerCollection */
+        $customerCollection = $this->getMockBuilder(Collection::class)
             ->disableOriginalConstructor()
-            ->setMethods(['load', 'removeAttributeToSelect', 'getResource', 'getSelect'])
+            ->setMethods(['getConnection','getMainTable'])
             ->getMock();
+        $customerCollection->expects($this->any())
+            ->method('getConnection')
+            ->will($this->returnValue($connectionMock));
 
-        $resourceStub = new \Magento\Framework\DataObject();
-        $resourceStub->setEntityTable($this->_entityTable);
-        $customerCollection->expects($this->once())->method('getResource')->will($this->returnValue($resourceStub));
+        $customerCollection->expects($this->any())
+            ->method('getMainTable')
+            ->willReturn('customer_entity');
 
-        $customerCollection->expects($this->once())->method('getSelect')->will($this->returnValue($select));
+        /** @var CollectionFactory|\PHPUnit_Framework_MockObject_MockObject $collectionFactory */
+        $collectionFactory = $this->getMockBuilder(CollectionFactory::class)
+            ->disableOriginalConstructor()
+            ->setMethods(['create'])
+            ->getMock();
+        $collectionFactory->expects($this->any())
+            ->method('create')
+            ->willReturn($customerCollection);
 
-        $byPagesIterator = $this->createPartialMock(\stdClass::class, ['iterate']);
-        $byPagesIterator->expects($this->once())
-            ->method('iterate')
-            ->will($this->returnCallback([$this, 'iterate']));
+        /** @var CollectionByPagesIteratorFactory|\PHPUnit_Framework_MockObject_MockObject $byPagesIteratorFactory */
+        $byPagesIteratorFactory = $this->getMockBuilder(CollectionByPagesIteratorFactory::class)
+            ->disableOriginalConstructor()
+            ->setMethods(['create'])
+            ->getMock();
 
-        return [
-            'customer_collection' => $customerCollection,
-            'collection_by_pages_iterator' => $byPagesIterator,
-            'page_size' => 10
-        ];
+        $this->_model = new Storage(
+            $collectionFactory,
+            $byPagesIteratorFactory
+        );
+        $this->_model->load();
     }
 
-    /**
-     * Iterate stub
-     *
-     * @SuppressWarnings(PHPMD.UnusedFormalParameter)
-     *
-     * @param \Magento\Framework\Data\Collection $collection
-     * @param int $pageSize
-     * @param array $callbacks
-     */
-    public function iterate(\Magento\Framework\Data\Collection $collection, $pageSize, array $callbacks)
+    protected function tearDown()
     {
-        foreach ($collection as $customer) {
-            foreach ($callbacks as $callback) {
-                call_user_func($callback, $customer);
-            }
-        }
+        unset($this->_model);
     }
 
     /**
@@ -112,13 +105,26 @@ public function testLoad()
     }
 
     public function testAddCustomer()
+    {
+        $customer = new \Magento\Framework\DataObject(['id' => 1, 'website_id' => 1, 'email' => 'test@test.com']);
+        $this->_model->addCustomer($customer);
+
+        $propertyName = '_customerIds';
+        $this->assertAttributeCount(1, $propertyName, $this->_model);
+        $this->assertAttributeContains([$customer->getWebsiteId() => $customer->getId()], $propertyName, $this->_model);
+        $this->assertEquals(
+            $customer->getId(),
+            $this->_model->getCustomerId($customer->getEmail(), $customer->getWebsiteId())
+        );
+    }
+
+    public function testAddCustomerByArray()
     {
         $propertyName = '_customerIds';
         $customer = $this->_addCustomerToStorage();
 
         $this->assertAttributeCount(1, $propertyName, $this->_model);
-
-        $expectedCustomerData = [$customer->getWebsiteId() => $customer->getId()];
+        $expectedCustomerData = [$customer['website_id'] => $customer['entity_id']];
         $this->assertAttributeContains($expectedCustomerData, $propertyName, $this->_model);
     }
 
@@ -127,19 +133,19 @@ public function testGetCustomerId()
         $customer = $this->_addCustomerToStorage();
 
         $this->assertEquals(
-            $customer->getId(),
-            $this->_model->getCustomerId($customer->getEmail(), $customer->getWebsiteId())
+            $customer['entity_id'],
+            $this->_model->getCustomerId($customer['email'], $customer['website_id'])
         );
-        $this->assertFalse($this->_model->getCustomerId('new@test.com', $customer->getWebsiteId()));
+        $this->assertFalse($this->_model->getCustomerId('new@test.com', $customer['website_id']));
     }
 
     /**
-     * @return \Magento\Framework\DataObject
+     * @return array
      */
     protected function _addCustomerToStorage()
     {
-        $customer = new \Magento\Framework\DataObject(['id' => 1, 'website_id' => 1, 'email' => 'test@test.com']);
-        $this->_model->addCustomer($customer);
+        $customer = ['entity_id' => 1, 'website_id' => 1, 'email' => 'test@test.com'];
+        $this->_model->addCustomerByArray($customer);
 
         return $customer;
     }