Skip to content

Commit 8e6ca73

Browse files
xmavxmav
committed
MAGETWO-58701: memory_limit issue after running customer_grid indexer
2 parents 2409e97 + 1ce84b2 commit 8e6ca73

File tree

7 files changed

+150
-13
lines changed

7 files changed

+150
-13
lines changed

‎app/code/Magento/Customer/Model/Indexer/Source.php

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
<?php
2+
/**
3+
* Copyright © 2016 Magento. All rights reserved.
4+
* See COPYING.txt for license details.
5+
*/
6+
namespace Magento\Customer\Model\Indexer;
7+
8+
use Magento\Customer\Model\ResourceModel\Customer\Indexer\Collection;
9+
use Magento\Framework\App\ResourceConnection\SourceProviderInterface;
10+
use Traversable;
11+
12+
/**
13+
* Customers data batch generator for customer_grid indexer
14+
*/
15+
class Source implements \IteratorAggregate, \Countable, SourceProviderInterface
16+
{
17+
/**
18+
* @var Collection
19+
*/
20+
private $customerCollection;
21+
22+
/**
23+
* @var int
24+
*/
25+
private $batchSize;
26+
27+
/**
28+
* @param \Magento\Customer\Model\ResourceModel\Customer\Indexer\CollectionFactory $collection
29+
* @param int $batchSize
30+
*/
31+
public function __construct(
32+
\Magento\Customer\Model\ResourceModel\Customer\Indexer\CollectionFactory $collectionFactory,
33+
$batchSize = 10000
34+
) {
35+
$this->customerCollection = $collectionFactory->create();
36+
$this->batchSize = $batchSize;
37+
}
38+
39+
/**
40+
* {@inheritdoc}
41+
*/
42+
public function getMainTable()
43+
{
44+
return $this->customerCollection->getMainTable();
45+
}
46+
47+
/**
48+
* {@inheritdoc}
49+
*/
50+
public function getIdFieldName()
51+
{
52+
return $this->customerCollection->getIdFieldName();
53+
}
54+
55+
/**
56+
* {@inheritdoc}
57+
*/
58+
public function addFieldToSelect($fieldName, $alias = null)
59+
{
60+
$this->customerCollection->addFieldToSelect($fieldName, $alias);
61+
return $this;
62+
}
63+
64+
/**
65+
* {@inheritdoc}
66+
*/
67+
public function getSelect()
68+
{
69+
return $this->customerCollection->getSelect();
70+
}
71+
72+
/**
73+
* {@inheritdoc}
74+
*/
75+
public function addFieldToFilter($attribute, $condition = null)
76+
{
77+
$this->customerCollection->addFieldToFilter($attribute, $condition);
78+
return $this;
79+
}
80+
81+
/**
82+
* @return int
83+
*/
84+
public function count()
85+
{
86+
return $this->customerCollection->getSize();
87+
}
88+
89+
/**
90+
* Retrieve an iterator
91+
*
92+
* @return Traversable
93+
*/
94+
public function getIterator()
95+
{
96+
$this->customerCollection->setPageSize($this->batchSize);
97+
$lastPage = $this->customerCollection->getLastPageNumber();
98+
$pageNumber = 0;
99+
do {
100+
$this->customerCollection->clear();
101+
$this->customerCollection->setCurPage($pageNumber);
102+
foreach ($this->customerCollection->getItems() as $key => $value) {
103+
yield $key => $value;
104+
}
105+
$pageNumber++;
106+
} while ($pageNumber <= $lastPage);
107+
}
108+
}

‎app/code/Magento/Customer/Model/ResourceModel/Customer/Indexer/Collection.php

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
<?php
2+
/**
3+
* Copyright © 2016 Magento. All rights reserved.
4+
* See COPYING.txt for license details.
5+
*/
6+
namespace Magento\Customer\Model\ResourceModel\Customer\Indexer;
7+
8+
/**
9+
* Customers collection for customer_grid indexer
10+
*/
11+
class Collection extends \Magento\Customer\Model\ResourceModel\Customer\Collection
12+
{
13+
/**
14+
* @inheritdoc
15+
*/
16+
protected function beforeAddLoadedItem(\Magento\Framework\DataObject $item)
17+
{
18+
return $item;
19+
}
20+
}

‎app/code/Magento/Customer/etc/indexer.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
<title translate="true">Customer Grid</title>
1212
<description translate="true">Rebuild Customer grid index</description>
1313

14-
<fieldset name="customer" source="Magento\Customer\Model\ResourceModel\Customer\Collection"
14+
<fieldset name="customer" source="Magento\Customer\Model\Indexer\Source"
1515
provider="Magento\Customer\Model\Indexer\AttributeProvider">
1616
<field name="name" xsi:type="searchable" dataType="text" handler="CustomerNameHandler"/>
1717
<field name="email" xsi:type="searchable" dataType="varchar"/>

‎lib/internal/Magento/Framework/Indexer/Action/Base.php

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,11 +37,13 @@ class Base implements ActionInterface
3737

3838
/**
3939
* @var AdapterInterface
40+
* @deprecated
4041
*/
4142
protected $connection;
4243

4344
/**
4445
* @var SourceProviderInterface[]
46+
* @deprecated
4547
*/
4648
protected $sources;
4749

@@ -52,6 +54,7 @@ class Base implements ActionInterface
5254

5355
/**
5456
* @var HandlerInterface[]
57+
* @deprecated
5558
*/
5659
protected $handlers;
5760

@@ -62,6 +65,7 @@ class Base implements ActionInterface
6265

6366
/**
6467
* @var array
68+
* @deprecated
6569
*/
6670
protected $columnTypesMap = [
6771
'varchar' => ['type' => Table::TYPE_TEXT, 'size' => 255],
@@ -71,11 +75,13 @@ class Base implements ActionInterface
7175

7276
/**
7377
* @var array
78+
* @deprecated
7479
*/
7580
protected $filterColumns;
7681

7782
/**
7883
* @var array
84+
* @deprecated
7985
*/
8086
protected $searchColumns;
8187

@@ -96,6 +102,7 @@ class Base implements ActionInterface
96102

97103
/**
98104
* @var String
105+
* @deprecated
99106
*/
100107
protected $string;
101108

@@ -106,11 +113,13 @@ class Base implements ActionInterface
106113

107114
/**
108115
* @var array
116+
* @deprecated
109117
*/
110118
protected $filterable = [];
111119

112120
/**
113121
* @var array
122+
* @deprecated
114123
*/
115124
protected $searchable = [];
116125

@@ -272,6 +281,7 @@ protected function getPrimaryFieldset()
272281
protected function createResultCollection()
273282
{
274283
$select = $this->getPrimaryResource()->getSelect();
284+
$select->reset(\Magento\Framework\DB\Select::COLUMNS);
275285
$select->columns($this->getPrimaryResource()->getIdFieldName());
276286
foreach ($this->data['fieldsets'] as $fieldset) {
277287
if (isset($fieldset['references'])) {
@@ -342,6 +352,8 @@ protected function prepareFields()
342352
*
343353
* @param array $field
344354
* @return void
355+
*
356+
* @deprecated
345357
*/
346358
protected function saveFieldByType($field)
347359
{

‎lib/internal/Magento/Framework/Indexer/Action/Entity.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,6 @@ protected function prepareDataSource(array $ids = [])
2424
{
2525
return !count($ids)
2626
? $this->createResultCollection()
27-
: $this->createResultCollection()->addFieldToFilter($this->getPrimaryResource()->getRowIdFieldName(), $ids);
27+
: $this->createResultCollection()->addFieldToFilter($this->getPrimaryResource()->getIdFieldName(), $ids);
2828
}
2929
}

‎lib/internal/Magento/Framework/Indexer/SaveHandler/Batch.php

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -10,27 +10,23 @@ class Batch
1010
/**
1111
* @param \Traversable $documents
1212
* @param int $size
13-
* @return array
13+
* @return \Generator
1414
*/
1515
public function getItems(\Traversable $documents, $size)
1616
{
17-
if (count($documents) == 0) {
18-
return [];
19-
}
20-
2117
$i = 0;
22-
$batch = $items = [];
18+
$batch = [];
19+
2320
foreach ($documents as $documentName => $documentValue) {
2421
$batch[$documentName] = $documentValue;
25-
if (++$i >= $size) {
26-
$items[] = $batch;
22+
if (++$i == $size) {
23+
yield $batch;
2724
$i = 0;
2825
$batch = [];
2926
}
3027
}
3128
if (count($batch) > 0) {
32-
$items[] = $batch;
29+
yield $batch;
3330
}
34-
return $items;
3531
}
3632
}

‎lib/internal/Magento/Framework/Indexer/Test/Unit/BatchTest.php

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@ protected function setUp()
2727
public function testGetItems(array $itemsData, $size, array $expected)
2828
{
2929
$items = new \ArrayObject($itemsData);
30-
$this->assertSame($expected, $this->object->getItems($items, $size));
30+
$data = $this->object->getItems($items, $size);
31+
$this->assertSame($expected, iterator_to_array($data));
3132
}
3233

3334
/**

0 commit comments

Comments
 (0)