MAGETWO-47001: Error on import products - validation not works

Author: Leonid Poluyanov

app/code/Magento/CatalogImportExport/Model/Import/Product.php

@@ -137,6 +137,11 @@ class Product extends \Magento\ImportExport\Model\Import\Entity\AbstractEntity
      */
     const INVENTORY_USE_CONFIG_PREFIX = 'use_config_';
 
+    /**
+     * Url key attribute code
+     */
+    const URL_KEY = 'url_key';
+
     /**
      * Attribute cache
      *
@@ -233,6 +238,7 @@ class Product extends \Magento\ImportExport\Model\Import\Entity\AbstractEntity
         ValidatorInterface::ERROR_MEDIA_PATH_NOT_ACCESSIBLE => 'Imported resource (image) does not exist in the local media storage',
         ValidatorInterface::ERROR_MEDIA_URL_NOT_ACCESSIBLE => 'Imported resource (image) could not be downloaded from external resource due to timeout or access permissions',
         ValidatorInterface::ERROR_INVALID_WEIGHT => 'Product weight is invalid',
+        ValidatorInterface::ERROR_DUPLICATE_URL_KEY => 'Specified url key is already exist',
     ];
 
     /**
@@ -502,12 +508,24 @@ class Product extends \Magento\ImportExport\Model\Import\Entity\AbstractEntity
      */
     protected $categoryProcessor;
 
+    /** @var \Magento\Framework\App\Config\ScopeConfigInterface */
+    protected $scopeConfig;
+
+    /** @var \Magento\Catalog\Model\Product\Url */
+    protected $productUrl;
+
     /** @var array */
     protected $websitesCache = [];
 
     /** @var array */
     protected $categoriesCache = [];
 
+    /** @var array */
+    protected $productUrlSuffix = [];
+
+    /** @var array */
+    protected $productUrlKeys = [];
+
     /**
      * Instance of product tax class processor.
      *
@@ -561,6 +579,12 @@ class Product extends \Magento\ImportExport\Model\Import\Entity\AbstractEntity
      */
     protected $cachedImages = null;
 
+    /** @var array */
+    protected $urlKeys = [];
+
+    /** @var array */
+    protected $rowNumbers = [];
+
     /**
      * @param \Magento\Framework\Json\Helper\Data $jsonHelper
      * @param \Magento\ImportExport\Helper\Data $importExportData
@@ -596,6 +620,7 @@ class Product extends \Magento\ImportExport\Model\Import\Entity\AbstractEntity
      * @param ObjectRelationProcessor $objectRelationProcessor
      * @param TransactionManagerInterface $transactionManager
      * @param Product\TaxClassProcessor $taxClassProcessor
+     * @param \Magento\Framework\App\Config\ScopeConfigInterface $scopeConfig
      * @param array $data
      * @throws \Magento\Framework\Exception\LocalizedException
      *
@@ -636,6 +661,8 @@ public function __construct(
         ObjectRelationProcessor $objectRelationProcessor,
         TransactionManagerInterface $transactionManager,
         Product\TaxClassProcessor $taxClassProcessor,
+        \Magento\Framework\App\Config\ScopeConfigInterface $scopeConfig,
+        \Magento\Catalog\Model\Product\Url $productUrl,
         array $data = []
     ) {
         $this->_eventManager = $eventManager;
@@ -663,6 +690,8 @@ public function __construct(
         $this->objectRelationProcessor = $objectRelationProcessor;
         $this->transactionManager = $transactionManager;
         $this->taxClassProcessor = $taxClassProcessor;
+        $this->scopeConfig = $scopeConfig;
+        $this->productUrl = $productUrl;
         parent::__construct(
             $jsonHelper,
             $importExportData,
@@ -1288,13 +1317,16 @@ protected function getExistingImages($images)
         if (!$productMediaGalleryTableName) {
             $productMediaGalleryTableName = $resource->getTable('catalog_product_entity_media_gallery');
         }
+        $linkField = $this->metadataPool
+            ->getMetadata(\Magento\Catalog\Api\Data\ProductInterface::class)
+            ->getLinkField();
         $select = $this->_connection->select()->from(
             ['mg' => $resource->getTable('catalog_product_entity_media_gallery')],
             ['value' => 'mg.value']
         )->joinLeft(
             ['mgvte' => $resource->getTable('catalog_product_entity_media_gallery_value_to_entity')],
             '(mg.value_id = mgvte.value_id)',
-            ['entity_id' => 'mgvte.entity_id']
+            [$linkField => 'mgvte.' . $linkField]
         )->where(
             'mg.value IN(?)',
             $images
@@ -1999,11 +2031,8 @@ protected function _saveStockItem()
      */
     public function retrieveAttributeByCode($attrCode)
     {
-        if (!$this->_resource) {
-            $this->_resource = $this->_resourceFactory->create();
-        }
         if (!isset($this->_attributeCache[$attrCode])) {
-            $this->_attributeCache[$attrCode] = $this->_resource->getAttribute($attrCode);
+            $this->_attributeCache[$attrCode] = $this->getResource()->getAttribute($attrCode);
         }
         return $this->_attributeCache[$attrCode];
     }
@@ -2214,7 +2243,25 @@ public function validateRow(array $rowData, $rowNum)
         }
         // validate custom options
         $this->getOptionEntity()->validateRow($rowData, $rowNum);
-
+        if (!empty($rowData[self::URL_KEY]) || !empty($rowData[self::COL_NAME])) {
+            $urlKey = $this->getUrlKey($rowData);
+            $storeCodes = empty($rowData[self::COL_STORE_VIEW_CODE])
+                ? array_flip($this->storeResolver->getStoreCodeToId())
+                : explode($this->getMultipleValueSeparator(), $rowData[self::COL_STORE_VIEW_CODE]);
+            foreach ($storeCodes as $storeCode) {
+                $storeId = $this->storeResolver->getStoreCodeToId($storeCode);
+                $productUrlSuffix = $this->getProductUrlSuffix($storeId);
+                $urlPath = $urlKey . $productUrlSuffix;
+                if (empty($this->urlKeys[$storeId][$urlPath])
+                    || ($this->urlKeys[$storeId][$urlPath] == $rowData[self::COL_SKU])
+                ) {
+                    $this->urlKeys[$storeId][$urlPath] = $rowData[self::COL_SKU];
+                    $this->rowNumbers[$storeId][$urlPath] = $rowNum;
+                } else {
+                    $this->addRowError(ValidatorInterface::ERROR_DUPLICATE_URL_KEY, $rowNum);
+                }
+            }
+        }
         return !$this->getErrorAggregator()->isRowInvalid($rowNum);
     }
 
@@ -2319,7 +2366,79 @@ protected function _saveValidatedBunches()
             $this->validateRow($rowData, $source->key());
             $source->next();
         }
+        $this->checkUrlKeyDuplicates();
         $this->getOptionEntity()->validateAmbiguousData();
         return parent::_saveValidatedBunches();
     }
+
+    /**
+     * Check that url_keys are not assigned to other products in DB
+     *
+     * @return void
+     */
+    protected function checkUrlKeyDuplicates()
+    {
+        $resource = $this->getResource();
+        foreach ($this->urlKeys as $storeId => $urlKeys) {
+            $urlKeyDuplicates = $this->_connection->fetchAssoc(
+                $this->_connection->select()->from(
+                    ['url_rewrite' => $resource->getTable('url_rewrite')],
+                    ['request_path', 'store_id']
+                )->joinLeft(
+                    ['cpe' => $resource->getTable('catalog_product_entity')],
+                    "cpe.entity_id = url_rewrite.entity_id"
+                )->where('request_path IN (?)', array_keys($urlKeys))
+                    ->where('store_id IN (?)', $storeId)
+                    ->where('cpe.sku not in (?)', array_values($urlKeys))
+            );
+            foreach ($urlKeyDuplicates as $urlKey => $entityData) {
+                $rowNum = $this->rowNumbers[$entityData['store_id']][$entityData['request_path']];
+                $this->addRowError(ValidatorInterface::ERROR_DUPLICATE_URL_KEY, $rowNum);
+            }
+        }
+    }
+
+    /**
+     * Retrieve product rewrite suffix for store
+     *
+     * @param int $storeId
+     * @return string
+     */
+    protected function getProductUrlSuffix($storeId = null)
+    {
+        if (!isset($this->productUrlSuffix[$storeId])) {
+            $this->productUrlSuffix[$storeId] = $this->scopeConfig->getValue(
+                \Magento\CatalogUrlRewrite\Model\ProductUrlPathGenerator::XML_PATH_PRODUCT_URL_SUFFIX,
+                \Magento\Store\Model\ScopeInterface::SCOPE_STORE,
+                $storeId
+            );
+        }
+        return $this->productUrlSuffix[$storeId];
+    }
+
+    /**
+     * @param array $rowData
+     * @return string
+     */
+    protected function getUrlKey($rowData)
+    {
+        if (!empty($rowData[self::URL_KEY])) {
+            $this->productUrlKeys[$rowData[self::COL_SKU]] = $rowData[self::URL_KEY];
+        }
+        $urlKey = !empty($this->productUrlKeys[$rowData[self::COL_SKU]])
+            ? $this->productUrlKeys[$rowData[self::COL_SKU]]
+            : $this->productUrl->formatUrlKey($rowData[self::COL_NAME]);
+        return $urlKey;
+    }
+
+    /**
+     * @return Proxy\Product\ResourceModel
+     */
+    protected function getResource()
+    {
+        if (!$this->_resource) {
+            $this->_resource = $this->_resourceFactory->create();
+        }
+        return $this->_resource;
+    }
 }

app/code/Magento/CatalogImportExport/Model/Import/Product/RowValidatorInterface.php

@@ -75,6 +75,8 @@ interface RowValidatorInterface extends \Magento\Framework\Validator\ValidatorIn
 
     const ERROR_MEDIA_PATH_NOT_ACCESSIBLE = 'mediaPathNotAvailable';
 
+    const ERROR_DUPLICATE_URL_KEY = 'duplicatedUrlKey';
+
     /**
      * Value that means all entities (e.g. websites, groups etc.)
      */

app/code/Magento/CatalogImportExport/Test/Unit/Model/Import/ProductTest.php

@@ -154,6 +154,12 @@ class ProductTest extends \Magento\ImportExport\Test\Unit\Model\Import\AbstractI
      */
     protected $errorAggregator;
 
+    /** @var \Magento\Framework\App\Config\ScopeConfigInterface|\PHPUnit_Framework_MockObject_MockObject*/
+    protected $scopeConfig;
+
+    /** @var \Magento\Catalog\Model\Product\Url|\PHPUnit_Framework_MockObject_MockObject*/
+    protected $productUrl;
+
     /**
      * @SuppressWarnings(PHPMD.ExcessiveMethodLength)
      */
@@ -315,6 +321,14 @@ protected function setUp()
                 ->disableOriginalConstructor()
                 ->getMock();
 
+        $this->scopeConfig = $this->getMockBuilder('\Magento\Framework\App\Config\ScopeConfigInterface')
+            ->disableOriginalConstructor()
+            ->getMockForAbstractClass();
+
+        $this->productUrl = $this->getMockBuilder('\Magento\Catalog\Model\Product\Url')
+            ->disableOriginalConstructor()
+            ->getMock();
+
         $this->errorAggregator = $this->getErrorAggregatorObject();
 
         $this->data = [];
@@ -360,6 +374,8 @@ protected function setUp()
             $this->objectRelationProcessor,
             $this->transactionManager,
             $this->taxClassProcessor,
+            $this->scopeConfig,
+            $this->productUrl,
             $this->data
         );
     }

app/code/Magento/CatalogImportExport/composer.json

@@ -4,6 +4,7 @@
     "require": {
         "php": "~5.5.0|~5.6.0|~7.0.0",
         "magento/module-catalog": "100.0.*",
+        "magento/module-catalog-url-rewrite": "100.0.*",
         "magento/module-eav": "100.0.*",
         "magento/module-import-export": "100.0.*",
         "magento/module-store": "100.0.*",

dev/tests/integration/testsuite/Magento/Catalog/Model/ResourceModel/_files/product_simple.php

@@ -42,5 +42,5 @@
     ->setVisibility(\Magento\Catalog\Model\Product\Visibility::VISIBILITY_BOTH)
     ->setStatus(\Magento\Catalog\Model\Product\Attribute\Source\Status::STATUS_ENABLED)
     ->setUrlKey('url-key')
-    ->setUrlPath('url-key.html')
+    ->setUrlPath('url-key')
     ->save();