ACP2E-3138: Cron job aggregate_sales_report_bestsellers_data cause deadlock if handles more that 100K rows

Author: abukatar

app/code/Magento/Sales/Model/ResourceModel/Report/Bestsellers.php

@@ -169,7 +169,67 @@ public function aggregate($from = null, $to = null)
     private function clearByDateRange($from = null, $to = null): void
     {
         $subSelect = $this->getRangeSubSelect($from, $to);
-        $this->_clearTableByDateRange($this->getMainTable(), $from, $to, $subSelect);
+        $this->clearTableRanges($this->getMainTable(), $from, $to, $subSelect);
+    }
+
+    /**
+     * Clear table by date range
+     *
+     * @param string $table
+     * @param ?string $from
+     * @param ?string $to
+     * @param null|\Magento\Framework\DB\Select|string $subSelect
+     * @return void
+     */
+    private function clearTableRanges($table, $from = null, $to = null, $subSelect = null): void
+    {
+        if ($from === null && $to === null) {
+            $this->_truncateTable($table);
+            return;
+        }
+
+        if ($subSelect !== null) {
+            $dataRange = $this->getRange($subSelect);
+            foreach ($dataRange as $date) {
+                $deleteCondition = $this->getConnection()->prepareSqlCondition('period', ['like' => $date]);
+                $this->getConnection()->delete($table, $deleteCondition);
+            }
+            return;
+        } else {
+            $condition = [];
+            if ($from !== null) {
+                $condition[] = $this->getConnection()->quoteInto('period >= ?', $from);
+            }
+
+            if ($to !== null) {
+                $condition[] = $this->getConnection()->quoteInto('period <= ?', $to);
+            }
+            $deleteCondition = implode(' AND ', $condition);
+        }
+        $this->getConnection()->delete($table, $deleteCondition);
+    }
+
+    /**
+     * Get dates range to clear the table
+     *
+     * @param \Magento\Framework\DB\Select $select
+     * @return array|false
+     */
+    private function getRange(\Magento\Framework\DB\Select $select)
+    {
+        $connection = $this->getConnection();
+        try {
+            $range = [];
+
+            $query = $connection->query($select);
+
+            while (true == ($date = $query->fetchColumn())) {
+                $range[] = $date;
+            }
+        } catch (\Exception $e) {
+            $range = false;
+        }
+        return $range;
     }
 
     /**

app/code/Magento/Sales/Test/Unit/Model/ResourceModel/Report/BestsellersTest.php

@@ -139,7 +139,7 @@ public function testAggregatePerStoreCalculationWithInterval(): void
             ->willReturn($periodExpr);
         $connection->expects($this->any())->method('select')->willReturn($select);
         $query = $this->createMock(\Zend_Db_Statement_Interface::class);
-        $connection->expects($this->exactly(3))->method('query')->willReturn($query);
+        $connection->expects($this->exactly(4))->method('query')->willReturn($query);
         $resource = $this->createMock(ResourceConnection::class);
         $resource->expects($this->any())
             ->method('getConnection')
@@ -180,6 +180,68 @@ public function testAggregatePerStoreCalculationWithInterval(): void
         $this->report->aggregate($from, $to);
     }
 
+    public function testClearByDateRange()
+    {
+        $from = new \DateTime('yesterday');
+        $to = new \DateTime();
+        $periodExpr = 'DATE(DATE_ADD(`source_table`.`created_at`, INTERVAL -25200 SECOND))';
+
+        $connection = $this->createMock(AdapterInterface::class);
+        $resource = $this->createMock(ResourceConnection::class);
+        $resource->expects($this->any())
+            ->method('getConnection')
+            ->with($this->connectionName)
+            ->willReturn($connection);
+        $this->context->expects($this->any())->method('getResources')->willReturn($resource);
+
+        $store = $this->createMock(StoreInterface::class);
+        $store->expects($this->once())->method('getId')->willReturn(1);
+        $this->storeManager->expects($this->once())->method('getStores')->with(true)->willReturn([$store]);
+
+        $select = $this->getMockBuilder(Select::class)
+            ->disableOriginalConstructor()
+            ->getMock();
+        $select->expects($this->atLeastOnce())->method('from')->willReturnSelf();
+        $select->expects($this->atLeastOnce())->method('joinInner')->willReturnSelf();
+        $select->expects($this->atLeastOnce())->method('joinLeft')->willReturnSelf();
+        $select->expects($this->atLeastOnce())->method('where')->willReturnSelf();
+        $select->expects($this->atLeastOnce())->method('distinct')->willReturnSelf();
+        $connection->expects($this->any())->method('select')->willReturn($select);
+
+        $date = $this->createMock(\DateTime::class);
+        $date->expects($this->atLeastOnce())->method('format')->with('e');
+        $this->time->expects($this->atLeastOnce())->method('scopeDate')->willReturn($date);
+
+        $flag = $this->createMock(Flag::class);
+        $flag->expects($this->atLeastOnce())->method('setReportFlagCode')->willReturnSelf();
+        $flag->expects($this->atLeastOnce())->method('unsetData')->willReturnSelf();
+        $flag->expects($this->atLeastOnce())->method('loadSelf');
+        $this->flagFactory->expects($this->atLeastOnce())->method('create')->willReturn($flag);
+
+        $query = $this->createMock(\Zend_Db_Statement_Interface::class);
+        $query->method('fetchColumn')->willReturnOnConsecutiveCalls('date1', 'date2', false);
+        $connection->expects($this->atLeastOnce())->method('query')->willReturn($query);
+        $connection->expects($this->atLeastOnce())->method('getDatePartSql')->willReturn($periodExpr);
+
+        $connection->expects($this->exactly(2))->method('delete');
+
+        $this->report = new Bestsellers(
+            $this->context,
+            $this->logger,
+            $this->time,
+            $this->flagFactory,
+            $this->validator,
+            $this->date,
+            $this->product,
+            $this->helper,
+            $this->connectionName,
+            [],
+            $this->storeManager
+        );
+
+        $this->report->aggregate($from, $to);
+    }
+
     /**
      * @return void
      * @throws \Exception