Price Index optimization

kandy · adifucan · commit 263c17fd0515 · 2022-03-02T18:48:46.000-06:00
diff --git a/app/code/Magento/Bundle/Model/ResourceModel/Indexer/Price/DisabledProductOptionPriceModifier.php b/app/code/Magento/Bundle/Model/ResourceModel/Indexer/Price/DisabledProductOptionPriceModifier.php
@@ -72,11 +72,8 @@ public function __construct(
      */
     public function modifyPrice(IndexTableStructure $priceTable, array $entityIds = []) : void
     {
-        foreach ($entityIds as $entityId) {
+        foreach ($this->getBundleIds($entityIds) as $entityId) {
             $entityId = (int) $entityId;
-            if (!$this->isBundle($entityId)) {
-                continue;
-            }
             foreach ($this->getWebsiteIdsOfProduct($entityId) as $websiteId) {
                 $productIdsDisabledRequired = $this->selectionProductsDisabledRequired
                     ->getChildProductIds($entityId, (int)$websiteId);
@@ -138,4 +135,24 @@ private function isBundle(int $entityId): bool
         $this->isBundle[$entityId] = $typeId === Type::TYPE_BUNDLE;
         return $this->isBundle[$entityId];
     }
+
+    /**
+     * @param array $entityIds
+     * @return \Traversable
+     */
+    protected function getBundleIds(array $entityIds): \Traversable
+    {
+        $connection = $this->resourceConnection->getConnection('indexer');
+        $select = $connection->select();
+        $select->from(
+            ['cpe' => $this->resourceConnection->getTableName('catalog_product_entity')],
+            ['entity_id']
+        )->where('cpe.entity_id in ( ? )', $entityIds ?? [0], \Zend_Db::INT_TYPE)
+        ->where('type_id = ?', Type::TYPE_BUNDLE);
+
+        $statement = $select->query();
+        while ($id = $statement->fetchColumn()) {
+            yield $id;
+        }
+    }
 }
diff --git a/app/code/Magento/Catalog/Model/Indexer/Product/Price/AbstractAction.php b/app/code/Magento/Catalog/Model/Indexer/Product/Price/AbstractAction.php
@@ -343,16 +343,15 @@ protected function _getIndexer($productTypeId)
      */
     protected function _insertFromTable($sourceTable, $destTable, $where = null)
     {
-        $sourceColumns = array_keys($this->getConnection()->describeTable($sourceTable));
-        $targetColumns = array_keys($this->getConnection()->describeTable($destTable));
-        $select = $this->getConnection()->select()->from($sourceTable, $sourceColumns);
+        $columns = array_keys($this->getConnection()->describeTable($destTable));
+        $select = $this->getConnection()->select()->from($sourceTable, $columns);
         if ($where) {
             $select->where($where);
         }
         $query = $this->getConnection()->insertFromSelect(
             $select,
             $destTable,
-            $targetColumns,
+            $columns,
             AdapterInterface::INSERT_ON_DUPLICATE
         );
         $this->getConnection()->query($query);
diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/Query/BaseFinalPrice.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/Query/BaseFinalPrice.php
@@ -110,62 +110,14 @@ public function getQuery(array $dimensions, string $productType, array $entityId
 
         $select = $connection->select()->from(
             ['e' => $this->getTable('catalog_product_entity')],
-            ['entity_id']
-        )->joinInner(
-            ['cg' => $this->getTable('customer_group')],
-            array_key_exists(CustomerGroupDimensionProvider::DIMENSION_NAME, $dimensions)
-                ? sprintf(
-                    '%s = %s',
-                    $this->dimensionToFieldMapper[CustomerGroupDimensionProvider::DIMENSION_NAME],
-                    $dimensions[CustomerGroupDimensionProvider::DIMENSION_NAME]->getValue()
-                ) : '',
-            ['customer_group_id']
-        )->joinInner(
-            ['pw' => $this->getTable('catalog_product_website')],
-            'pw.product_id = e.entity_id',
-            ['pw.website_id']
-        )->joinInner(
-            ['cwd' => $this->getTable('catalog_product_index_website')],
-            'pw.website_id = cwd.website_id',
-            []
-        )->joinLeft(
-            // customer group website limitations
-            ['cgw' => $this->getTable('customer_group_excluded_website')],
-            'cg.customer_group_id = cgw.customer_group_id AND pw.website_id = cgw.website_id',
-            []
-        )->joinLeft(
-            // we need this only for BCC in case someone expects table `tp` to be present in query
-            ['tp' => $this->getTable('catalog_product_index_tier_price')],
-            'tp.entity_id = e.entity_id AND' .
-            ' tp.customer_group_id = cg.customer_group_id AND tp.website_id = pw.website_id',
-            []
-        )->joinLeft(
-            // calculate tier price specified as Website = `All Websites` and Customer Group = `Specific Customer Group`
-            ['tier_price_1' => $this->getTable('catalog_product_entity_tier_price')],
-            'tier_price_1.' . $linkField . ' = e.' . $linkField . ' AND tier_price_1.all_groups = 0' .
-            ' AND tier_price_1.customer_group_id = cg.customer_group_id AND tier_price_1.qty = 1' .
-            ' AND tier_price_1.website_id = 0',
             []
         )->joinLeft(
-            // calculate tier price specified as Website = `Specific Website`
-            //and Customer Group = `Specific Customer Group`
-            ['tier_price_2' => $this->getTable('catalog_product_entity_tier_price')],
-            'tier_price_2.' . $linkField . ' = e.' . $linkField . ' AND tier_price_2.all_groups = 0 ' .
-            'AND tier_price_2.customer_group_id = cg.customer_group_id AND tier_price_2.qty = 1' .
-            ' AND tier_price_2.website_id = pw.website_id',
-            []
-        )->joinLeft(
-            // calculate tier price specified as Website = `All Websites` and Customer Group = `ALL GROUPS`
-            ['tier_price_3' => $this->getTable('catalog_product_entity_tier_price')],
-            'tier_price_3.' . $linkField . ' = e.' . $linkField . ' AND tier_price_3.all_groups = 1 ' .
-            'AND tier_price_3.customer_group_id = 0 AND tier_price_3.qty = 1 AND tier_price_3.website_id = 0',
+            ['pw' => $this->getTable('catalog_product_website')],
+            'pw.product_id = e.entity_id',
             []
         )->joinLeft(
-            // calculate tier price specified as Website = `Specific Website` and Customer Group = `ALL GROUPS`
-            ['tier_price_4' => $this->getTable('catalog_product_entity_tier_price')],
-            'tier_price_4.' . $linkField . ' = e.' . $linkField . ' AND tier_price_4.all_groups = 1' .
-            ' AND tier_price_4.customer_group_id = 0 AND tier_price_4.qty = 1' .
-            ' AND tier_price_4.website_id = pw.website_id',
+            ['cwd' => $this->getTable('catalog_product_index_website')],
+            'pw.website_id = cwd.website_id',
             []
         );
 
@@ -183,7 +135,7 @@ public function getQuery(array $dimensions, string $productType, array $entityId
         } else {
             $taxClassId = new \Zend_Db_Expr(0);
         }
-        $select->columns(['tax_class_id' => $taxClassId]);
+
 
         $this->joinAttributeProcessor->process($select, 'status', Status::STATUS_ENABLED);
 
@@ -213,8 +165,56 @@ public function getQuery(array $dimensions, string $productType, array $entityId
             ]
         );
 
+        $select->join(
+            ['cg' => $this->getTable('customer_group')],
+            array_key_exists(CustomerGroupDimensionProvider::DIMENSION_NAME, $dimensions)
+                ? sprintf(
+                '%s = %s',
+                $this->dimensionToFieldMapper[CustomerGroupDimensionProvider::DIMENSION_NAME],
+                $dimensions[CustomerGroupDimensionProvider::DIMENSION_NAME]->getValue()
+            ) : '',
+            []
+        )->joinLeft(
+        // customer group website limitations
+            ['cgw' => $this->getTable('customer_group_excluded_website')],
+            'cg.customer_group_id = cgw.customer_group_id AND pw.website_id = cgw.website_id',
+            []
+        )->joinLeft(
+        // calculate tier price specified as Website = `All Websites` and Customer Group = `Specific Customer Group`
+            ['tier_price_1' => $this->getTable('catalog_product_entity_tier_price')],
+            'tier_price_1.' . $linkField . ' = e.' . $linkField . ' AND tier_price_1.all_groups = 0' .
+            ' AND tier_price_1.customer_group_id = cg.customer_group_id AND tier_price_1.qty = 1' .
+            ' AND tier_price_1.website_id = 0',
+            []
+        )->joinLeft(
+        // calculate tier price specified as Website = `All Websites` and Customer Group = `ALL GROUPS`
+            ['tier_price_3' => $this->getTable('catalog_product_entity_tier_price')],
+            'tier_price_3.' . $linkField . ' = e.' . $linkField . ' AND tier_price_3.all_groups = 1 ' .
+            'AND tier_price_3.customer_group_id = 0 AND tier_price_3.qty = 1 AND tier_price_3.website_id = 0',
+            []
+        )->joinLeft(
+        // calculate tier price specified as Website = `Specific Website` and Customer Group = `ALL GROUPS`
+            ['tier_price_4' => $this->getTable('catalog_product_entity_tier_price')],
+            'tier_price_4.' . $linkField . ' = e.' . $linkField . ' AND tier_price_4.all_groups = 1' .
+            ' AND tier_price_4.customer_group_id = 0 AND tier_price_4.qty = 1' .
+            ' AND tier_price_4.website_id = pw.website_id',
+            []
+        )->joinLeft(
+        // calculate tier price specified as Website = `Specific Website`
+        //and Customer Group = `Specific Customer Group`
+            ['tier_price_2' => $this->getTable('catalog_product_entity_tier_price')],
+            'tier_price_2.' . $linkField . ' = e.' . $linkField . ' AND tier_price_2.all_groups = 0 ' .
+            'AND tier_price_2.customer_group_id = cg.customer_group_id AND tier_price_2.qty = 1' .
+            ' AND tier_price_2.website_id = pw.website_id',
+            []
+        );
+        // the order of fields is important!
         $select->columns(
             [
+                'entity_id',
+                'cg.customer_group_id',
+                'pw.website_id',
+                'tax_class_id' => $taxClassId,
                 //orig_price in catalog_product_index_price_final_tmp
                 'price' => $connection->getIfNullSql($price, 0),
                 //price in catalog_product_index_price_final_tmp
@@ -227,13 +227,15 @@ public function getQuery(array $dimensions, string $productType, array $entityId
 
         $select->where("e.type_id = ?", $productType);
 
+        // exclude websites that are limited for customer group
+        $select->where('cgw.website_id IS NULL');
+
         if ($entityIds !== null) {
             $select->where(sprintf('e.entity_id BETWEEN %s AND %s', min($entityIds), max($entityIds)));
-            $select->where('e.entity_id IN(?)', $entityIds);
+            $select->where('e.entity_id IN(?)', $entityIds, \Zend_Db::INT_TYPE);
         }
 
-        // exclude websites that are limited for customer group
-        $select->where('cgw.website_id IS NULL');
+
 
         /**
          * throw event for backward compatibility
@@ -248,6 +250,8 @@ public function getQuery(array $dimensions, string $productType, array $entityId
             ]
         );
 
+       //  static $p; if (!$p) { echo $select; $p =1; }
+       // var_dump(__METHOD__, $connection->query($connection->select()->from((clone $select)->having('customer_group_id = 0'), "COUNT(*)"))->fetchColumn());
         return $select;
     }
 
diff --git a/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/SimpleProductPrice.php b/app/code/Magento/Catalog/Model/ResourceModel/Product/Indexer/Price/SimpleProductPrice.php
@@ -81,7 +81,21 @@ public function executeByDimensions(array $dimensions, \Traversable $entityIds)
             'tierPriceField' => 'tier_price',
         ]);
         $select = $this->baseFinalPrice->getQuery($dimensions, $this->productType, iterator_to_array($entityIds));
-        $this->tableMaintainer->insertFromSelect($select, $temporaryPriceTable->getTableName(), []);
+        $this->tableMaintainer->insertFromSelect(
+            $select,
+            $temporaryPriceTable->getTableName(),
+            [
+                "entity_id",
+                "customer_group_id",
+                "website_id",
+                "tax_class_id",
+                "price",
+                "final_price",
+                "min_price",
+                "max_price",
+                "tier_price",
+            ]
+        );
 
         $this->basePriceModifier->modifyPrice($temporaryPriceTable, iterator_to_array($entityIds));
     }
diff --git a/app/code/Magento/Catalog/etc/db_schema.xml b/app/code/Magento/Catalog/etc/db_schema.xml
@@ -1623,7 +1623,7 @@
     <table name="catalog_product_index_price_tmp" resource="default" engine="innodb"
            comment="Catalog Product Price Indexer Temp Table">
         <column xsi:type="int" name="entity_id" unsigned="true" nullable="false" identity="false"
-                comment="Entity ID"/>
+                comment="Product ID"/>
         <column xsi:type="int" name="customer_group_id" unsigned="true" nullable="false" identity="false"
                 default="0" comment="Customer Group ID"/>
         <column xsi:type="smallint" name="website_id" unsigned="true" nullable="false" identity="false"
@@ -1640,20 +1640,11 @@
                 comment="Max Price"/>
         <column xsi:type="decimal" name="tier_price" scale="6" precision="20" unsigned="false" nullable="true"
                 comment="Tier Price"/>
+        <column xsi:type="int" name="id" unsigned="true" nullable="false" identity="true"
+                comment="ID"/>
         <constraint xsi:type="primary" referenceId="PRIMARY">
-            <column name="entity_id"/>
-            <column name="customer_group_id"/>
-            <column name="website_id"/>
+            <column name="id"/>
         </constraint>
-        <index referenceId="CATALOG_PRODUCT_INDEX_PRICE_TMP_CUSTOMER_GROUP_ID" indexType="btree">
-            <column name="customer_group_id"/>
-        </index>
-        <index referenceId="CATALOG_PRODUCT_INDEX_PRICE_TMP_WEBSITE_ID" indexType="btree">
-            <column name="website_id"/>
-        </index>
-        <index referenceId="CATALOG_PRODUCT_INDEX_PRICE_TMP_MIN_PRICE" indexType="btree">
-            <column name="min_price"/>
-        </index>
     </table>
     <table name="catalog_category_product_index_tmp" resource="default" engine="innodb"
            comment="Catalog Category Product Indexer temporary table">
diff --git a/app/code/Magento/CatalogSearch/Model/Indexer/Fulltext.php b/app/code/Magento/CatalogSearch/Model/Indexer/Fulltext.php
@@ -38,7 +38,7 @@ class Fulltext implements
     /**
      * Default batch size
      */
-    private const BATCH_SIZE = 100;
+    private const BATCH_SIZE = 1000;
 
     /**
      * @var array index structure
diff --git a/app/code/Magento/CatalogSearch/Model/Indexer/Fulltext/Action/DataProvider.php b/app/code/Magento/CatalogSearch/Model/Indexer/Fulltext/Action/DataProvider.php
@@ -204,7 +204,7 @@ public function getSearchableProducts(
         array $staticFields,
         $productIds = null,
         $lastProductId = 0,
-        $batch = 100
+        $batch = 1000
     ) {
         $select = $this->selectSearchableProducts->execute(
             (int) $storeId,
@@ -357,10 +357,11 @@ public function getProductAttributes($storeId, array $productIds, array $attribu
         $selects = [];
         $ifStoreValue = $this->connection->getCheckSql('t_store.value_id > 0', 't_store.value', 't_default.value');
         $linkField = $this->metadata->getLinkField();
-        $productLinkFieldsToEntityIdMap = $this->connection->fetchPairs(
+        $tmpTableName = 'index_fulltext_tmp_' . time();
+        $this->connection->query('CREATE TEMPORARY TABLE ' . $tmpTableName . ' ' .
             $this->connection->select()->from(
                 ['cpe' => $this->getTable('catalog_product_entity')],
-                [$linkField, 'entity_id']
+                $linkField != 'entity_id' ? [$linkField, 'entity_id'] : ['entity_id']
             )->where(
                 'cpe.entity_id IN (?)',
                 $productIds,
@@ -369,53 +370,45 @@ public function getProductAttributes($storeId, array $productIds, array $attribu
         );
         foreach ($attributeTypes as $backendType => $attributeIds) {
             if ($attributeIds) {
-                $tableName = $this->getTable('catalog_product_entity_' . $backendType);
-
-                $select = $this->connection->select()->from(
-                    ['t' => $tableName],
-                    [
-                        $linkField => 't.' . $linkField,
-                        'attribute_id' => 't.attribute_id',
-                        'value' => $this->unifyField($ifStoreValue, $backendType),
-                    ]
-                )->joinLeft(
-                    ['t_store' => $tableName],
-                    $this->connection->quoteInto(
-                        't.' . $linkField . '=t_store.' . $linkField .
-                        ' AND t.attribute_id=t_store.attribute_id' .
-                        ' AND t_store.store_id = ?',
-                        $storeId
-                    ),
-                    []
-                )->joinLeft(
-                    ['t_default' => $tableName],
-                    $this->connection->quoteInto(
-                        't.' . $linkField . '=t_default.' . $linkField .
-                        ' AND t.attribute_id=t_default.attribute_id' .
-                        ' AND t_default.store_id = ?',
-                        0
-                    ),
-                    []
-                )->where(
-                    't.attribute_id IN (?)',
-                    $attributeIds
-                )->where(
-                    't.' . $linkField . ' IN (?)',
-                    array_keys($productLinkFieldsToEntityIdMap)
-                )->distinct();
-                $selects[] = $select;
+                foreach ($attributeIds as $attributeId) {
+                    $tableName = $this->getTable('catalog_product_entity_' . $backendType);
+
+                    $select = $this->connection->select()->from(
+                        ['t' => $tmpTableName],
+                        [
+                            'entity_id' => 't.entity_id',
+                            $linkField => 't.' . $linkField,
+                            'value' => $this->unifyField($ifStoreValue, $backendType),
+                        ]
+                    )->joinLeft(
+                        ['t_default' => $tableName],
+                        't.' . $linkField . ' = t_default.' . $linkField
+                        . ' AND t_default.attribute_id = ' . (int) $attributeId
+                        . ' AND t_default.store_id = 0',
+                        ['attribute_id' => 't_default.attribute_id']
+                    )->joinLeft(
+                        ['t_store' => $tableName],
+                        't.' . $linkField . ' = t_store.' . $linkField
+                        . ' AND t_store.attribute_id = '  . (int) $attributeId
+                        . ' AND t_store.store_id = ' . (int) $storeId,
+
+                        []
+                    );
+                    $selects[] = $select;
+                }
+
             }
         }
 
         if ($selects) {
             $select = $this->connection->select()->union($selects, Select::SQL_UNION_ALL);
             $query = $this->connection->query($select);
             while ($row = $query->fetch()) {
-                $entityId = $productLinkFieldsToEntityIdMap[$row[$linkField]];
+                $entityId = $row['entity_id'];
                 $result[$entityId][$row['attribute_id']] = $row['value'];
             }
         }
-
+        $this->connection->dropTemporaryTable($tmpTableName);
         return $result;
     }
 
diff --git a/app/code/Magento/CatalogSearch/Model/Indexer/Fulltext/Action/Full.php b/app/code/Magento/CatalogSearch/Model/Indexer/Fulltext/Action/Full.php
@@ -260,7 +260,7 @@ public function __construct(
         $indexIteratorFactory = null,
         \Magento\Framework\EntityManager\MetadataPool $metadataPool = null,
         DataProvider $dataProvider = null,
-        $batchSize = 500,
+        $batchSize = 1000,
         ?DeploymentConfig $deploymentConfig = null
     ) {
         $this->resource = $resource;
diff --git a/app/code/Magento/CatalogSearch/Model/ResourceModel/Fulltext/Collection.php b/app/code/Magento/CatalogSearch/Model/ResourceModel/Fulltext/Collection.php
diff --git a/app/code/Magento/ConfigurableProduct/Model/ResourceModel/Product/Indexer/Price/Configurable.php b/app/code/Magento/ConfigurableProduct/Model/ResourceModel/Product/Indexer/Price/Configurable.php
diff --git a/dev/tests/integration/framework/Magento/TestFramework/Listener/ExtededTestdox.php b/dev/tests/integration/framework/Magento/TestFramework/Listener/ExtededTestdox.php
