Skip to content

Commit dd80b51

Browse files
kandykandy
authored
Merge pull request #1140 from magento-performance/pr-develop
[performance] MAGETWO-68808: catalog_product_attribute is slow
2 parents 59fb155 + 2be508a commit dd80b51

File tree

7 files changed

+143
-18
lines changed

7 files changed

+143
-18
lines changed

‎app/code/Magento/Catalog/Model/Indexer/Product/Eav/AbstractAction.php

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -148,12 +148,13 @@ protected function syncData($indexer, $destinationTable, $ids)
148148
* @param \Magento\Catalog\Model\ResourceModel\Product\Indexer\Eav\AbstractEav $indexer
149149
* @param array $ids
150150
*
151-
* @return $ids
151+
* @param bool $onlyParents
152+
* @return array $ids
152153
*/
153-
protected function processRelations($indexer, $ids)
154+
protected function processRelations($indexer, $ids, $onlyParents = false)
154155
{
155156
$parentIds = $indexer->getRelationsByChild($ids);
156-
$childIds = $indexer->getRelationsByParent($ids);
157+
$childIds = $onlyParents ? [] : $indexer->getRelationsByParent($ids);
157158
return array_unique(array_merge($ids, $childIds, $parentIds));
158159
}
159160
}

‎app/code/Magento/Catalog/Model/Indexer/Product/Eav/Action/Full.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ public function execute($ids = null)
8383
$select->from(['e' => $entityMetadata->getEntityTable()], $entityMetadata->getIdentifierField());
8484
$entityIds = $this->batchProvider->getBatchIds($connection, $select, $batch);
8585
if (!empty($entityIds)) {
86-
$indexer->reindexEntities($this->processRelations($indexer, $entityIds));
86+
$indexer->reindexEntities($this->processRelations($indexer, $entityIds, true));
8787
$this->syncData($indexer, $mainTable);
8888
}
8989
}

‎app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/AbstractIndexer.php

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -196,7 +196,7 @@ public function getRelationsByChild($childIds)
196196
$childIds
197197
);
198198

199-
return $connection->fetchCol($select);
199+
return array_map('intval', (array) $connection->fetchCol($select));
200200
}
201201

202202
/**
@@ -228,7 +228,7 @@ public function getRelationsByParent($parentIds)
228228
$result = $connection->fetchCol($select);
229229
}
230230

231-
return $result;
231+
return array_map('intval', $result);
232232
}
233233

234234
/**

‎app/code/Magento/Catalog/Test/Unit/Model/Indexer/Product/Eav/Action/FullTest.php

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@
55
*/
66
namespace Magento\Catalog\Test\Unit\Model\Indexer\Product\Eav\Action;
77

8+
/**
9+
* @SuppressWarnings(PHPMD.CouplingBetweenObjects)
10+
*/
811
class FullTest extends \PHPUnit_Framework_TestCase
912
{
1013
public function testExecuteWithAdapterErrorThrowsException()
@@ -55,4 +58,103 @@ public function testExecuteWithAdapterErrorThrowsException()
5558

5659
$model->execute();
5760
}
61+
62+
public function testExecute()
63+
{
64+
$eavDecimalFactory = $this->getMock(
65+
\Magento\Catalog\Model\ResourceModel\Product\Indexer\Eav\DecimalFactory::class,
66+
['create'],
67+
[],
68+
'',
69+
false
70+
);
71+
$eavSourceFactory = $this->getMock(
72+
\Magento\Catalog\Model\ResourceModel\Product\Indexer\Eav\SourceFactory::class,
73+
['create'],
74+
[],
75+
'',
76+
false
77+
);
78+
79+
$ids = [1, 2, 3];
80+
$connectionMock = $this->getMockBuilder(\Magento\Framework\DB\Adapter\AdapterInterface::class)
81+
->getMockForAbstractClass();
82+
83+
$connectionMock->expects($this->atLeastOnce())->method('describeTable')->willReturn(['id' => []]);
84+
$eavSource = $this->getMockBuilder(\Magento\Catalog\Model\ResourceModel\Product\Indexer\Eav\Source::class)
85+
->disableOriginalConstructor()
86+
->getMock();
87+
88+
$eavDecimal = $this->getMockBuilder(\Magento\Catalog\Model\ResourceModel\Product\Indexer\Eav\Decimal::class)
89+
->disableOriginalConstructor()
90+
->getMock();
91+
92+
$eavSource->expects($this->once())->method('getRelationsByChild')->with($ids)->willReturn([]);
93+
$eavSource->expects($this->never())->method('getRelationsByParent')->with($ids)->willReturn([]);
94+
95+
$eavDecimal->expects($this->once())->method('getRelationsByChild')->with($ids)->willReturn([]);
96+
$eavDecimal->expects($this->never())->method('getRelationsByParent')->with($ids)->willReturn([]);
97+
98+
$eavSource->expects($this->atLeastOnce())->method('getConnection')->willReturn($connectionMock);
99+
$eavDecimal->expects($this->atLeastOnce())->method('getConnection')->willReturn($connectionMock);
100+
101+
$eavDecimal->expects($this->once())
102+
->method('reindexEntities')
103+
->with($ids);
104+
105+
$eavSource->expects($this->once())
106+
->method('reindexEntities')
107+
->with($ids);
108+
109+
$eavDecimalFactory->expects($this->once())
110+
->method('create')
111+
->will($this->returnValue($eavSource));
112+
113+
$eavSourceFactory->expects($this->once())
114+
->method('create')
115+
->will($this->returnValue($eavDecimal));
116+
117+
$metadataMock = $this->getMock(\Magento\Framework\EntityManager\MetadataPool::class, [], [], '', false);
118+
$entityMetadataMock = $this->getMockBuilder(\Magento\Framework\EntityManager\EntityMetadataInterface::class)
119+
->getMockForAbstractClass();
120+
121+
$metadataMock->expects($this->atLeastOnce())
122+
->method('getMetadata')
123+
->with(\Magento\Catalog\Api\Data\ProductInterface::class)
124+
->willReturn($entityMetadataMock);
125+
126+
$batchProviderMock = $this->getMock(\Magento\Framework\Indexer\BatchProviderInterface::class);
127+
$batchProviderMock->expects($this->atLeastOnce())
128+
->method('getBatches')
129+
->willReturn([['from' => 10, 'to' => 100]]);
130+
$batchProviderMock->expects($this->atLeastOnce())
131+
->method('getBatchIds')
132+
->willReturn($ids);
133+
134+
$batchManagementMock = $this->getMock(
135+
\Magento\Catalog\Model\ResourceModel\Product\Indexer\Eav\BatchSizeCalculator::class,
136+
[],
137+
[],
138+
'',
139+
false
140+
);
141+
$selectMock = $this->getMockBuilder(\Magento\Framework\DB\Select::class)
142+
->disableOriginalConstructor()
143+
->getMock();
144+
145+
$connectionMock->method('select')->willReturn($selectMock);
146+
$selectMock->expects($this->atLeastOnce())->method('distinct')->willReturnSelf();
147+
$selectMock->expects($this->atLeastOnce())->method('from')->willReturnSelf();
148+
149+
$model = new \Magento\Catalog\Model\Indexer\Product\Eav\Action\Full(
150+
$eavDecimalFactory,
151+
$eavSourceFactory,
152+
$metadataMock,
153+
$batchProviderMock,
154+
$batchManagementMock,
155+
[]
156+
);
157+
158+
$model->execute();
159+
}
58160
}

‎app/code/Magento/PageCache/etc/varnish4.vcl

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -96,11 +96,15 @@ sub vcl_recv {
9696
set req.url = regsuball(req.url,"\?gclid=[^&]+&","?"); # strips when QS = "?gclid=AAA&foo=bar"
9797
set req.url = regsuball(req.url,"&gclid=[^&]+",""); # strips when QS = "?foo=bar&gclid=AAA" or QS = "?foo=bar&gclid=AAA&bar=baz"
9898

99-
# static files are always cacheable. remove SSL flag and cookie
100-
if (req.url ~ "^/(pub/)?(media|static)/.*\.(ico|html|css|js|jpg|jpeg|png|gif|tiff|bmp|mp3|ogg|svg|swf|woff|woff2|eot|ttf|otf)$") {
101-
unset req.http.Https;
102-
unset req.http./* {{ ssl_offloaded_header }} */;
103-
unset req.http.Cookie;
99+
# Static files caching
100+
if (req.url ~ "^/(pub/)?(media|static)/.*\.(7z|avi|bmp|bz2|css|csv|doc|docx|eot|flac|flv|gif|gz|html|ico|jpeg|jpg|js|less|mka|mkv|mov|mp3|mp4|mpeg|mpg|odt|otf|ogg|ogm|opus|pdf|png|ppt|pptx|rar|rtf|svg|svgz|swf|tar|tbz|tgz|tiff|ttf|txt|txz|wav|webm|webp|woff|woff2|xls|xlsx|xml|xz|zip)$") {
101+
# Static files should not be cached by default
102+
return (pass);
103+
104+
# But if you use a few locales and don't use CDN you can enable caching static files by commenting previous line (#return (pass);) and uncommenting next 3 lines
105+
#unset req.http.Https;
106+
#unset req.http./* {{ ssl_offloaded_header }} */;
107+
#unset req.http.Cookie;
104108
}
105109

106110
return (hash);
@@ -156,7 +160,7 @@ sub vcl_backend_response {
156160
# images, css and js are cacheable by default so we have to remove cookie also
157161
if (beresp.ttl > 0s && (bereq.method == "GET" || bereq.method == "HEAD")) {
158162
unset beresp.http.set-cookie;
159-
if (bereq.url !~ "\.(ico|css|js|jpg|jpeg|png|gif|tiff|bmp|gz|tgz|bz2|tbz|mp3|ogg|svg|swf|woff|woff2|eot|ttf|otf)(\?|$)") {
163+
if (bereq.url !~ "\.(7z|avi|bmp|bz2|css|csv|doc|docx|eot|flac|flv|gif|gz|html|ico|jpeg|jpg|js|less|mka|mkv|mov|mp3|mp4|mpeg|mpg|odt|otf|ogg|ogm|opus|pdf|png|ppt|pptx|rar|rtf|svg|svgz|swf|tar|tbz|tgz|tiff|ttf|txt|txz|wav|webm|webp|woff|woff2|xls|xlsx|xml|xz|zip)(\?|$)") {
160164
set beresp.http.Pragma = "no-cache";
161165
set beresp.http.Expires = "-1";
162166
set beresp.http.Cache-Control = "no-store, no-cache, must-revalidate, max-age=0";

‎app/code/Magento/PageCache/etc/varnish5.vcl

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -97,11 +97,15 @@ sub vcl_recv {
9797
set req.url = regsuball(req.url,"\?gclid=[^&]+&","?"); # strips when QS = "?gclid=AAA&foo=bar"
9898
set req.url = regsuball(req.url,"&gclid=[^&]+",""); # strips when QS = "?foo=bar&gclid=AAA" or QS = "?foo=bar&gclid=AAA&bar=baz"
9999

100-
# static files are always cacheable. remove SSL flag and cookie
101-
if (req.url ~ "^/(pub/)?(media|static)/.*\.(ico|html|css|js|jpg|jpeg|png|gif|tiff|bmp|mp3|ogg|svg|swf|woff|woff2|eot|ttf|otf)$") {
102-
unset req.http.Https;
103-
unset req.http./* {{ ssl_offloaded_header }} */;
104-
unset req.http.Cookie;
100+
# Static files caching
101+
if (req.url ~ "^/(pub/)?(media|static)/.*\.(7z|avi|bmp|bz2|css|csv|doc|docx|eot|flac|flv|gif|gz|html|ico|jpeg|jpg|js|less|mka|mkv|mov|mp3|mp4|mpeg|mpg|odt|otf|ogg|ogm|opus|pdf|png|ppt|pptx|rar|rtf|svg|svgz|swf|tar|tbz|tgz|tiff|ttf|txt|txz|wav|webm|webp|woff|woff2|xls|xlsx|xml|xz|zip)$") {
102+
# Static files should not be cached by default
103+
return (pass);
104+
105+
# But if you use a few locales and don't use CDN you can enable caching static files by commenting previous line (#return (pass);) and uncommenting next 3 lines
106+
#unset req.http.Https;
107+
#unset req.http./* {{ ssl_offloaded_header }} */;
108+
#unset req.http.Cookie;
105109
}
106110

107111
return (hash);
@@ -157,7 +161,7 @@ sub vcl_backend_response {
157161
# images, css and js are cacheable by default so we have to remove cookie also
158162
if (beresp.ttl > 0s && (bereq.method == "GET" || bereq.method == "HEAD")) {
159163
unset beresp.http.set-cookie;
160-
if (bereq.url !~ "\.(ico|css|js|jpg|jpeg|png|gif|tiff|bmp|gz|tgz|bz2|tbz|mp3|ogg|svg|swf|woff|woff2|eot|ttf|otf)(\?|$)") {
164+
if (bereq.url !~ "\.(7z|avi|bmp|bz2|css|csv|doc|docx|eot|flac|flv|gif|gz|html|ico|jpeg|jpg|js|less|mka|mkv|mov|mp3|mp4|mpeg|mpg|odt|otf|ogg|ogm|opus|pdf|png|ppt|pptx|rar|rtf|svg|svgz|swf|tar|tbz|tgz|tiff|ttf|txt|txz|wav|webm|webp|woff|woff2|xls|xlsx|xml|xz|zip)(\?|$)") {
161165
set beresp.http.Pragma = "no-cache";
162166
set beresp.http.Expires = "-1";
163167
set beresp.http.Cache-Control = "no-store, no-cache, must-revalidate, max-age=0";

‎lib/internal/Magento/Framework/Indexer/Test/Unit/BatchProviderTest.php

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,4 +57,18 @@ public function getBatchesDataProvider()
5757
[200, 0, []],
5858
];
5959
}
60+
61+
public function testGetBatchIds()
62+
{
63+
$selectMock = $this->getMock(Select::class, [], [], '', false);
64+
$adapterMock = $this->getMock(AdapterInterface::class);
65+
66+
$selectMock->expects($this->once())->method('where')->with('(entity_id BETWEEN 10 AND 100)')->willReturnSelf();
67+
$adapterMock->expects($this->atLeastOnce())->method('quote')->willReturnArgument(0);
68+
$adapterMock->expects($this->once())->method('fetchCol')->with($selectMock, [])->willReturn([1, 2, 3]);
69+
$this->assertEquals(
70+
[1, 2, 3],
71+
$this->model->getBatchIds($adapterMock, $selectMock, ['from' => 10, 'to' => 100])
72+
);
73+
}
6074
}

0 commit comments

Comments
 (0)