Skip to content

Commit c5d8fc6

Browse files
skovalenkskovalenk
committed
Mview patch update
-- move framework logic to saas-export
1 parent 0e66e6c commit c5d8fc6

File tree

12 files changed

+381
-163
lines changed

12 files changed

+381
-163
lines changed

‎app/code/Magento/Eav/Model/Mview/BatchIterator.php

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
<?php
2+
/**
3+
* Copyright © Magento, Inc. All rights reserved.
4+
* See COPYING.txt for license details.
5+
*/
6+
7+
namespace Magento\Eav\Mview;
8+
9+
use Magento\Framework\App\ResourceConnection;
10+
use Magento\Framework\DB\Sql\Expression;
11+
use Magento\Framework\Mview\View\ChangeLogBatchIteratorInterface;
12+
use Magento\Framework\Mview\View\ChangelogInterface;
13+
use Magento\Framework\Mview\View\ChangelogTableNotExistsException;
14+
use Magento\Framework\Phrase;
15+
16+
/**
17+
* Class BatchIterator
18+
*/
19+
class BatchIterator implements ChangeLogBatchIteratorInterface
20+
{
21+
/**
22+
* @var ResourceConnection
23+
*/
24+
private $resourceConnection;
25+
26+
/**
27+
* @param ResourceConnection $resourceConnection
28+
*/
29+
public function __construct(
30+
ResourceConnection $resourceConnection
31+
) {
32+
$this->resourceConnection = $resourceConnection;
33+
}
34+
35+
/**
36+
* @inheritdoc
37+
*/
38+
public function walk(ChangelogInterface $changelog, int $fromVersionId, int $toVersion, int $batchSize)
39+
{
40+
$connection = $this->resourceConnection->getConnection();
41+
if (!$connection->isTableExists($changelog->getName())) {
42+
throw new ChangelogTableNotExistsException(
43+
new Phrase("Table %1 does not exist", [$changelog->getName()])
44+
);
45+
}
46+
$select = $connection->select()->distinct(true)
47+
->where(
48+
'version_id > ?',
49+
(int)$fromVersionId
50+
)
51+
->where(
52+
'version_id <= ?',
53+
$toVersion
54+
)
55+
->group([$changelog->getColumnName(), 'store_id'])
56+
->limit($batchSize);
57+
58+
$columns = [
59+
$changelog->getColumnName(),
60+
'attribute_ids' => new Expression('GROUP_CONCAT(attribute_id)'),
61+
'store_id'
62+
];
63+
64+
$select->from($changelog->getName(), $columns);
65+
return $connection->fetchAll($select);
66+
}
67+
}

‎lib/internal/Magento/Framework/Mview/Config/Converter.php

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ public function convert($source)
2929
$data['action_class'] = $this->getAttributeValue($viewNode, 'class');
3030
$data['group'] = $this->getAttributeValue($viewNode, 'group');
3131
$data['store_scope'] = $this->getAttributeValue($viewNode, 'store_scope');
32-
$data['attribute_scope'] = $this->getAttributeValue($viewNode, 'attribute_scope');
32+
$data['iterator'] = $this->getAttributeValue($viewNode, 'iterator');
3333
$data['subscriptions'] = [];
3434

3535
/** @var $childNode \DOMNode */
@@ -78,6 +78,7 @@ protected function convertChild(\DOMNode $childNode, $data)
7878
$name = $this->getAttributeValue($subscription, 'name');
7979
$column = $this->getAttributeValue($subscription, 'entity_column');
8080
$subscriptionModel = $this->getAttributeValue($subscription, 'subscription_model');
81+
8182
if (!empty($subscriptionModel)
8283
&& !in_array(
8384
SubscriptionInterface::class,
@@ -91,11 +92,36 @@ class_implements(ltrim($subscriptionModel, '\\'))
9192
$data['subscriptions'][$name] = [
9293
'name' => $name,
9394
'column' => $column,
94-
'subscription_model' => $subscriptionModel
95+
'subscription_model' => $subscriptionModel,
96+
'additional_columns' => $this->getAdditionalColumns($subscription),
97+
'processor' => $this->getAttributeValue($subscription, 'processor')
9598
];
9699
}
97100
break;
98101
}
99102
return $data;
100103
}
104+
105+
/**
106+
* Retrieve additional columns of subscription table
107+
*
108+
* @param \DOMNode $subscription
109+
* @return array
110+
*/
111+
private function getAdditionalColumns(\DOMNode $subscription): array
112+
{
113+
$additionalColumns = [];
114+
foreach ($subscription->childNodes as $childNode) {
115+
if ($childNode->nodeType != XML_ELEMENT_NODE || $childNode->nodeName != 'additionalColumns') {
116+
continue;
117+
}
118+
119+
$additionalColumns[] = [
120+
'name' => $this->getAttributeValue($childNode, 'name'),
121+
'cl_name' => $this->getAttributeValue($childNode, 'cl_name'),
122+
];
123+
}
124+
125+
return $additionalColumns;
126+
}
101127
}

‎lib/internal/Magento/Framework/Mview/Test/Unit/View/ChangelogTest.php

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
use Magento\Framework\DB\Adapter\Pdo\Mysql;
1212
use Magento\Framework\DB\Ddl\Table;
1313
use Magento\Framework\DB\Select;
14+
use Magento\Framework\Mview\Config;
1415
use Magento\Framework\Mview\View\Changelog;
1516
use Magento\Framework\Mview\View\ChangelogInterface;
1617
use PHPUnit\Framework\MockObject\MockObject;
@@ -46,15 +47,30 @@ protected function setUp(): void
4647
$this->resourceMock = $this->createMock(ResourceConnection::class);
4748
$this->mockGetConnection($this->connectionMock);
4849

49-
$this->model = new Changelog($this->resourceMock);
50+
$this->model = new Changelog($this->resourceMock, $this->getMviewConfigMock());
51+
}
52+
53+
/**
54+
* @return Config|MockObject
55+
*/
56+
private function getMviewConfigMock()
57+
{
58+
$mviewConfigMock = $this->createMock(Config::class);
59+
$mviewConfigMock->expects($this->any())
60+
->method('getView')
61+
->willReturn([
62+
'attribute_scope' => false,
63+
'store_scope' => false
64+
]);
65+
return $mviewConfigMock;
5066
}
5167

5268
public function testInstanceOf()
5369
{
5470
$resourceMock =
5571
$this->createMock(ResourceConnection::class);
5672
$resourceMock->expects($this->once())->method('getConnection')->willReturn(true);
57-
$model = new Changelog($resourceMock);
73+
$model = new Changelog($resourceMock, $this->getMviewConfigMock());
5874
$this->assertInstanceOf(ChangelogInterface::class, $model);
5975
}
6076

@@ -65,7 +81,7 @@ public function testCheckConnectionException()
6581
$resourceMock =
6682
$this->createMock(ResourceConnection::class);
6783
$resourceMock->expects($this->once())->method('getConnection')->willReturn(null);
68-
$model = new Changelog($resourceMock);
84+
$model = new Changelog($resourceMock, $this->getMviewConfigMock());
6985
$model->setViewId('ViewIdTest');
7086
$this->assertNull($model);
7187
}

‎lib/internal/Magento/Framework/Mview/Test/Unit/View/SubscriptionTest.php

Lines changed: 26 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
use Magento\Framework\DB\Adapter\Pdo\Mysql;
1212
use Magento\Framework\DB\Ddl\Trigger;
1313
use Magento\Framework\DB\Ddl\TriggerFactory;
14+
use Magento\Framework\Mview\Config;
1415
use Magento\Framework\Mview\View\ChangelogInterface;
1516
use Magento\Framework\Mview\View\CollectionInterface;
1617
use Magento\Framework\Mview\View\StateInterface;
@@ -55,6 +56,10 @@ protected function setUp(): void
5556
->method('quoteIdentifier')
5657
->willReturnArgument(0);
5758

59+
$this->connectionMock->expects($this->any())
60+
->method('describeTable')
61+
->willReturn([]);
62+
5863
$this->resourceMock->expects($this->atLeastOnce())
5964
->method('getConnection')
6065
->willReturn($this->connectionMock);
@@ -78,18 +83,29 @@ protected function setUp(): void
7883
true,
7984
[]
8085
);
81-
86+
$this->viewMock->expects($this->any())
87+
->method('getId')
88+
->willReturn(1);
8289
$this->resourceMock->expects($this->any())
8390
->method('getTableName')
8491
->willReturnArgument(0);
85-
92+
$mviewConfigMock = $this->createMock(Config::class);
93+
$mviewConfigMock->expects($this->any())
94+
->method('getView')
95+
->willReturn([
96+
'attribute_scope' => false,
97+
'store_scope' => false
98+
]);
99+
$this->mviewConfig = $mviewConfigMock;
86100
$this->model = new Subscription(
87101
$this->resourceMock,
88102
$this->triggerFactoryMock,
89103
$this->viewCollectionMock,
90104
$this->viewMock,
91105
$this->tableName,
92-
'columnName'
106+
'columnName',
107+
[],
108+
$mviewConfigMock
93109
);
94110
}
95111

@@ -122,7 +138,7 @@ public function testCreate()
122138
$triggerMock->expects($this->exactly(3))
123139
->method('setName')
124140
->with($triggerName)->willReturnSelf();
125-
$triggerMock->expects($this->exactly(3))
141+
$triggerMock->expects($this->any())
126142
->method('getName')
127143
->willReturn('triggerName');
128144
$triggerMock->expects($this->exactly(3))
@@ -167,7 +183,7 @@ public function testCreate()
167183
true,
168184
[]
169185
);
170-
$changelogMock->expects($this->exactly(3))
186+
$changelogMock->expects($this->any())
171187
->method('getName')
172188
->willReturn('test_view_cl');
173189
$changelogMock->expects($this->exactly(3))
@@ -191,7 +207,7 @@ public function testCreate()
191207
true,
192208
[]
193209
);
194-
$otherChangelogMock->expects($this->exactly(3))
210+
$otherChangelogMock->expects($this->any())
195211
->method('getName')
196212
->willReturn('other_test_view_cl');
197213
$otherChangelogMock->expects($this->exactly(3))
@@ -217,7 +233,7 @@ public function testCreate()
217233
->method('getChangelog')
218234
->willReturn($otherChangelogMock);
219235

220-
$this->viewMock->expects($this->exactly(3))
236+
$this->viewMock->expects($this->any())
221237
->method('getId')
222238
->willReturn('this_id');
223239
$this->viewMock->expects($this->never())
@@ -235,7 +251,6 @@ public function testCreate()
235251
$this->connectionMock->expects($this->exactly(3))
236252
->method('createTrigger')
237253
->with($triggerMock);
238-
239254
$this->model->create();
240255
}
241256

@@ -244,7 +259,7 @@ public function testRemove()
244259
$triggerMock = $this->createMock(Trigger::class);
245260
$triggerMock->expects($this->exactly(3))
246261
->method('setName')->willReturnSelf();
247-
$triggerMock->expects($this->exactly(3))
262+
$triggerMock->expects($this->any())
248263
->method('getName')
249264
->willReturn('triggerName');
250265
$triggerMock->expects($this->exactly(3))
@@ -271,7 +286,7 @@ public function testRemove()
271286
true,
272287
[]
273288
);
274-
$otherChangelogMock->expects($this->exactly(3))
289+
$otherChangelogMock->expects($this->any())
275290
->method('getName')
276291
->willReturn('other_test_view_cl');
277292
$otherChangelogMock->expects($this->exactly(3))
@@ -297,7 +312,7 @@ public function testRemove()
297312
->method('getChangelog')
298313
->willReturn($otherChangelogMock);
299314

300-
$this->viewMock->expects($this->exactly(3))
315+
$this->viewMock->expects($this->any())
301316
->method('getId')
302317
->willReturn('this_id');
303318
$this->viewMock->expects($this->never())

‎lib/internal/Magento/Framework/Mview/View.php

Lines changed: 24 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
namespace Magento\Framework\Mview;
1010

1111
use InvalidArgumentException;
12+
use Magento\Framework\App\ObjectManager;
1213
use Magento\Framework\DataObject;
1314
use Magento\Framework\Mview\View\ChangeLogBatchIteratorInterface;
1415
use Magento\Framework\Mview\View\ChangelogTableNotExistsException;
@@ -28,11 +29,6 @@ class View extends DataObject implements ViewInterface
2829
*/
2930
const DEFAULT_BATCH_SIZE = 1000;
3031

31-
/**
32-
* Max versions to load from database at a time
33-
*/
34-
private static $maxVersionQueryBatch = 100000;
35-
3632
/**
3733
* @var string
3834
*/
@@ -306,52 +302,38 @@ private function executeAction(ActionInterface $action, int $lastVersionId, int
306302

307303
$vsFrom = $lastVersionId;
308304
while ($vsFrom < $currentVersionId) {
309-
if (isset($this->strategies[$this->changelog->getViewId()])) {
310-
$changelogData = [
311-
'name' => $this->changelog->getName(),
312-
'column_name' => $this->changelog->getColumnName(),
313-
'view_id' => $this->changelog->getViewId()
314-
];
315-
$ids = $this->strategies[$this->changelog->getViewId()]->walk($changelogData, $vsFrom, $batchSize);
316-
$vsFrom += $batchSize;
317-
$action->execute($ids);
318-
} else {
319-
$ids = $this->getBatchOfIds($vsFrom, $currentVersionId);
320-
// We run the actual indexer in batches.
321-
// Chunked AFTER loading to avoid duplicates in separate chunks.
322-
$chunks = array_chunk($ids, $batchSize);
323-
foreach ($chunks as $ids) {
324-
$action->execute($ids);
325-
}
326-
}
327-
305+
$iterator = $this->createIterator();
306+
$ids = $iterator->walk($this->getChangelog(), $vsFrom, $currentVersionId, $batchSize);
307+
$vsFrom += $batchSize;
308+
$action->execute($ids);
328309
}
329310
}
330311

331312
/**
332-
* Get batch of entity ids
313+
* Create and validate iterator class for changelog
333314
*
334-
* @param int $lastVersionId
335-
* @param int $currentVersionId
336-
* @return array
315+
* @return ChangeLogBatchIteratorInterface|mixed
316+
* @throws Exception
337317
*/
338-
private function getBatchOfIds(int &$lastVersionId, int $currentVersionId): array
318+
private function createIterator()
339319
{
340-
$ids = [];
341-
$versionBatchSize = self::$maxVersionQueryBatch;
342-
$idsBatchSize = self::$maxVersionQueryBatch;
343-
for ($vsFrom = $lastVersionId; $vsFrom < $currentVersionId; $vsFrom += $versionBatchSize) {
344-
// Don't go past the current version for atomicity.
345-
$versionTo = min($currentVersionId, $vsFrom + $versionBatchSize);
346-
/** To avoid duplicate ids need to flip and merge the array */
347-
$ids += array_flip($this->getChangelog()->getList($vsFrom, $versionTo));
348-
$lastVersionId = $versionTo;
349-
if (count($ids) >= $idsBatchSize) {
350-
break;
351-
}
320+
$config = $this->config->getView($this->changelog->getViewId());
321+
$iteratorClass = $config['iterator'];
322+
323+
if (!class_exists($iteratorClass)) {
324+
throw new \Exception('Iterator class does not exist for view: ' . $this->changelog->getViewId());
325+
}
326+
327+
$iterator = ObjectManager::getInstance()->get($iteratorClass);
328+
329+
if (!$iterator instanceof ChangeLogBatchIteratorInterface) {
330+
throw new \Exception(
331+
'Iterator does not implement the right interface for view: ' .
332+
$this->changelog->getViewId()
333+
);
352334
}
353335

354-
return array_keys($ids);
336+
return $iterator;
355337
}
356338

357339
/**

0 commit comments

Comments
 (0)