Merge magento-partners/magento2ce#38.

Author: Volodymyr Kublytskyi

lib/internal/Magento/Framework/Mview/Test/Unit/ViewTest.php

@@ -263,7 +263,6 @@ public function testUpdate()
         $currentVersionId = 3;
         $lastVersionId = 1;
         $listId = [2, 3];
-        $defaultBatchSize = 1000;
 
         $this->stateMock->expects($this->any())
             ->method('getViewId')
@@ -297,7 +296,7 @@ public function testUpdate()
             'getList'
         )->with(
             $lastVersionId,
-            $lastVersionId + $defaultBatchSize
+            $currentVersionId
         )->will(
             $this->returnValue($listId)
         );
@@ -327,7 +326,6 @@ public function testUpdateWithException()
         $currentVersionId = 3;
         $lastVersionId = 1;
         $listId = [2, 3];
-        $defaultBatchSize = 1000;
 
         $this->stateMock->expects($this->any())
             ->method('getViewId')
@@ -360,7 +358,7 @@ public function testUpdateWithException()
             'getList'
         )->with(
             $lastVersionId,
-            $lastVersionId + $defaultBatchSize
+            $currentVersionId
         )->will(
             $this->returnValue($listId)
         );

lib/internal/Magento/Framework/Mview/View.php

@@ -21,6 +21,11 @@ class View extends \Magento\Framework\DataObject implements ViewInterface
      */
     const DEFAULT_BATCH_SIZE = 1000;
 
+    /**
+     * Max versions to load from database at a time
+     */
+    private static $maxVersionQueryBatch = 100000;
+
     /**
      * @var string
      */
@@ -272,14 +277,19 @@ public function update()
             try {
                 $this->getState()->setStatus(View\StateInterface::STATUS_WORKING)->save();
 
+                $versionBatchSize = self::$maxVersionQueryBatch;
                 $batchSize = isset($this->changelogBatchSize[$this->getChangelog()->getViewId()])
                     ? $this->changelogBatchSize[$this->getChangelog()->getViewId()]
                     : self::DEFAULT_BATCH_SIZE;
 
-                for ($versionFrom = $lastVersionId; $versionFrom < $currentVersionId; $versionFrom += $batchSize) {
-                    $ids = $this->getChangelog()->getList($versionFrom, $versionFrom + $batchSize);
+                for ($versionFrom = $lastVersionId; $versionFrom < $currentVersionId; $versionFrom += $versionBatchSize) {
+                    // Don't go past the current version for atomicy.
+                    $versionTo = min($currentVersionId, $versionFrom + $versionBatchSize);
+                    $ids = $this->getChangelog()->getList($versionFrom, $versionTo);
 
-                    if (!empty($ids)) {
+                    // We run the actual indexer in batches.  Chunked AFTER loading to avoid duplicates in separate chunks.
+                    $chunks = array_chunk($ids, $batchSize);
+                    foreach ($chunks as $ids) {
                         $action->execute($ids);
                     }
                 }