MAGETWO-84529: [2.3] Correct flat indexing with staging and > 500 cats #12345

Author: Volodymyr Kublytskyi

app/code/Magento/Catalog/Model/Indexer/Category/Flat/AbstractAction.php

@@ -369,22 +369,55 @@ protected function getAttributeValues($entityIds, $storeId)
         }
         $values = [];
 
+        $linkIds = $this->getLinkIds($entityIds);
+        foreach ($linkIds as $linkId) {
+            $values[$linkId] = [];
+        }
+
         $attributes = $this->getAttributes();
         $attributesType = ['varchar', 'int', 'decimal', 'text', 'datetime'];
+        $linkField = $this->getCategoryMetadata()->getLinkField();
         foreach ($attributesType as $type) {
             foreach ($this->getAttributeTypeValues($type, $entityIds, $storeId) as $row) {
-                if (isset($row['entity_id']) && isset($row['attribute_id'])) {
+                if (isset($row[$linkField]) && isset($row['attribute_id'])) {
                     $attributeId = $row['attribute_id'];
                     if (isset($attributes[$attributeId])) {
                         $attributeCode = $attributes[$attributeId]['attribute_code'];
-                        $values[$row['entity_id']][$attributeCode] = $row['value'];
+                        $values[$row[$linkField]][$attributeCode] = $row['value'];
                     }
                 }
             }
         }
+
         return $values;
     }
 
+    /**
+     * Translate entity ids into link ids
+     *
+     * Used for rows with no EAV attributes set.
+     *
+     * @param array $entityIds
+     * @return array
+     */
+    private function getLinkIds(array $entityIds)
+    {
+        $linkField = $this->getCategoryMetadata()->getLinkField();
+        if ($linkField === 'entity_id') {
+            return $entityIds;
+        }
+
+        $select = $this->connection->select()->from(
+            ['e' => $this->connection->getTableName($this->getTableName('catalog_category_entity'))],
+            [$linkField]
+        )->where(
+            'e.entity_id IN (?)',
+            $entityIds
+        );
+
+        return $this->connection->fetchCol($select);
+    }
+
     /**
      * Return attribute values for given entities and store of specific attribute type
      *

app/code/Magento/Catalog/Model/Indexer/Category/Flat/Action/Full.php

@@ -64,18 +64,22 @@ protected function populateFlatTables(array $stores)
             }
             /** @TODO Do something with chunks */
             $categoriesIdsChunks = array_chunk($categoriesIds[$store->getRootCategoryId()], 500);
+
             foreach ($categoriesIdsChunks as $categoriesIdsChunk) {
                 $attributesData = $this->getAttributeValues($categoriesIdsChunk, $store->getId());
+                $linkField = $this->categoryMetadata->getLinkField();
+
                 $data = [];
                 foreach ($categories[$store->getRootCategoryId()] as $category) {
-                    if (!isset($attributesData[$category[$this->categoryMetadata->getLinkField()]])) {
+                    if (!isset($attributesData[$category[$linkField]])) {
                         continue;
                     }
                     $category['store_id'] = $store->getId();
                     $data[] = $this->prepareValuesToInsert(
-                        array_merge($category, $attributesData[$category[$this->categoryMetadata->getLinkField()]])
+                        array_merge($category, $attributesData[$category[$linkField]])
                     );
                 }
+
                 $this->connection->insertMultiple(
                     $this->addTemporaryTableSuffix($this->getMainStoreTable($store->getId())),
                     $data

app/code/Magento/Catalog/Model/Indexer/Category/Flat/Action/Rows.php

@@ -164,16 +164,22 @@ private function reindexStore(Store $store, array $entityIds, $useTempTable)
      */
     private function buildIndexData(Store $store, $categoriesIdsChunk, $attributesData)
     {
+        $linkField = $this->categoryMetadata->getLinkField();
+
         $data = [];
         foreach ($categoriesIdsChunk as $categoryId) {
             try {
+                $category = $this->categoryRepository->get($categoryId);
+                $categoryData = $category->getData();
+                $linkId = $categoryData[$linkField];
+
                 $categoryAttributesData = [];
-                if (isset($attributesData[$categoryId]) && is_array($attributesData[$categoryId])) {
-                    $categoryAttributesData = $attributesData[$categoryId];
+                if (isset($attributesData[$linkId]) && is_array($attributesData[$linkId])) {
+                    $categoryAttributesData = $attributesData[$linkId];
                 }
                 $categoryIndexData = $this->buildCategoryIndexData(
                     $store,
-                    $categoryId,
+                    $categoryData,
                     $categoryAttributesData
                 );
                 $data[] = $categoryIndexData;
@@ -186,18 +192,16 @@ private function buildIndexData(Store $store, $categoriesIdsChunk, $attributesDa
 
     /**
      * @param Store $store
-     * @param int $categoryId
+     * @param array $categoryData
      * @param array $categoryAttributesData
      * @return array
      * @throws NoSuchEntityException
      */
-    private function buildCategoryIndexData(Store $store, $categoryId, array $categoryAttributesData)
+    private function buildCategoryIndexData(Store $store, array $categoryData, array $categoryAttributesData)
     {
-        $category = $this->categoryRepository->get($categoryId);
-        $categoryAttributesData = [];
         $data = $this->prepareValuesToInsert(
             array_merge(
-                $category->getData(),
+                $categoryData,
                 $categoryAttributesData,
                 ['store_id' => $store->getId()]
             )