Skip to content

Commit

Permalink
MAGETWO-95789: [GitHub] Error in system.log about migrated attribute …
Browse files Browse the repository at this point in the history
…which is not included into attribute group #598
  • Loading branch information
victor-v-rad committed May 15, 2019
1 parent a280dc2 commit c5d7b65
Show file tree
Hide file tree
Showing 10 changed files with 339 additions and 23 deletions.
8 changes: 8 additions & 0 deletions etc/commerce-to-commerce/eav-document-groups.xml.dist
Original file line number Diff line number Diff line change
Expand Up @@ -66,4 +66,12 @@
<document>magento_rma_item_entity_varchar</document>
<document>magento_rma_item_form_attribute</document>
</group>
<group name="documents_attribute_set_leftover_values">
<document>catalog_product_entity_datetime</document>
<document>catalog_product_entity_decimal</document>
<document>catalog_product_entity_gallery</document>
<document>catalog_product_entity_int</document>
<document>catalog_product_entity_text</document>
<document>catalog_product_entity_varchar</document>
</group>
</groups>
8 changes: 8 additions & 0 deletions etc/opensource-to-commerce/eav-document-groups.xml.dist
Original file line number Diff line number Diff line change
Expand Up @@ -64,4 +64,12 @@
<document>magento_rma_item_entity_varchar</document>
<document>magento_rma_item_form_attribute</document>
</group>
<group name="documents_attribute_set_leftover_values">
<document>catalog_product_entity_datetime</document>
<document>catalog_product_entity_decimal</document>
<document>catalog_product_entity_gallery</document>
<document>catalog_product_entity_int</document>
<document>catalog_product_entity_text</document>
<document>catalog_product_entity_varchar</document>
</group>
</groups>
8 changes: 8 additions & 0 deletions etc/opensource-to-opensource/eav-document-groups.xml.dist
Original file line number Diff line number Diff line change
Expand Up @@ -53,4 +53,12 @@
<document>salesrule_product_attribute</document>
<document>weee_tax</document>
</group>
<group name="documents_attribute_set_leftover_values">
<document>catalog_product_entity_datetime</document>
<document>catalog_product_entity_decimal</document>
<document>catalog_product_entity_gallery</document>
<document>catalog_product_entity_int</document>
<document>catalog_product_entity_text</document>
<document>catalog_product_entity_varchar</document>
</group>
</groups>
36 changes: 34 additions & 2 deletions src/Migration/Step/PostProcessing/Data.php
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@
use Migration\App\ProgressBar;
use Migration\Logger\Manager as LogManager;
use Migration\Step\PostProcessing\Data\EavLeftoverDataCleaner;
use Migration\Step\PostProcessing\Data\AttributeSetLeftoverDataCleaner;
use Migration\Step\PostProcessing\Data\ProductsInRootCatalogCleaner;
use Migration\Step\PostProcessing\Data\DeletedRecordsCounter;

/**
* Class Data
Expand All @@ -26,24 +28,51 @@ class Data implements StageInterface
*/
private $eavLeftoverDataCleaner;

/**
* @var AttributeSetLeftoverDataCleaner
*/
private $attributeSetLeftoverDataCleaner;

/**
* @var ProductsInRootCatalogCleaner
*/
private $productsInRootCatalogCleaner;

/**
* @var DeletedRecordsCounter
*/
private $deletedRecordsCounter;

/**
* @var array
*/
private $documents = [];

/**
* Data constructor.
* @param ProgressBar\LogLevelProcessor $progressBar
* @param EavLeftoverDataCleaner $eavLeftoverDataCleaner
* @param AttributeSetLeftoverDataCleaner $attributeSetLeftoverDataCleaner
* @param ProductsInRootCatalogCleaner $productsInRootCatalogCleaner
* @param DeletedRecordsCounter $deletedRecordsCounter
*/
public function __construct(
ProgressBar\LogLevelProcessor $progressBar,
EavLeftoverDataCleaner $eavLeftoverDataCleaner,
ProductsInRootCatalogCleaner $productsInRootCatalogCleaner
AttributeSetLeftoverDataCleaner $attributeSetLeftoverDataCleaner,
ProductsInRootCatalogCleaner $productsInRootCatalogCleaner,
DeletedRecordsCounter $deletedRecordsCounter
) {
$this->progressBar = $progressBar;
$this->eavLeftoverDataCleaner = $eavLeftoverDataCleaner;
$this->attributeSetLeftoverDataCleaner = $attributeSetLeftoverDataCleaner;
$this->productsInRootCatalogCleaner = $productsInRootCatalogCleaner;
$this->deletedRecordsCounter = $deletedRecordsCounter;
$append = function ($document) {
$this->documents[] = $document;
};
array_map($append, $this->eavLeftoverDataCleaner->getDocuments());
array_map($append, $this->attributeSetLeftoverDataCleaner->getDocuments());
}

/**
Expand All @@ -52,8 +81,11 @@ public function __construct(
public function perform()
{
$this->progressBar->start($this->getIterationsCount(), LogManager::LOG_LEVEL_INFO);
$this->deletedRecordsCounter->count($this->documents);
$this->eavLeftoverDataCleaner->clean();
$this->attributeSetLeftoverDataCleaner->clean();
$this->productsInRootCatalogCleaner->clean();
$this->deletedRecordsCounter->saveDeleted($this->documents);
$this->progressBar->finish(LogManager::LOG_LEVEL_INFO);
return true;
}
Expand All @@ -65,6 +97,6 @@ public function perform()
*/
private function getIterationsCount()
{
return $this->eavLeftoverDataCleaner->getIterationsCount();
return count($this->documents);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
<?php
/**
* Copyright © Magento, Inc. All rights reserved.
* See COPYING.txt for license details.
*/
namespace Migration\Step\PostProcessing\Data;

use Migration\ResourceModel;
use Migration\App\ProgressBar;
use Migration\App\Progress;
use Migration\Logger\Manager as LogManager;
use \Migration\Step\PostProcessing\Model\AttributeSetLeftoverData as AttributeSetLeftoverDataModel;

/**
* Class AttributeSetLeftoverDataCleaner
*/
class AttributeSetLeftoverDataCleaner
{
/**
* @var ResourceModel\Destination
*/
private $destination;

/**
* @var ProgressBar\LogLevelProcessor
*/
private $progressBar;

/**
* @var Progress
*/
private $progress;

/**
* @var AttributeSetLeftoverDataModel
*/
private $attributeSetLeftoverDataModel;

/**
* @param ProgressBar\LogLevelProcessor $progressBar
* @param ResourceModel\Destination $destination
* @param Progress $progress
* @param AttributeSetLeftoverDataModel $attributeSetLeftoverDataModel
*/
public function __construct(
ProgressBar\LogLevelProcessor $progressBar,
ResourceModel\Destination $destination,
Progress $progress,
AttributeSetLeftoverDataModel $attributeSetLeftoverDataModel
) {
$this->destination = $destination;
$this->progressBar = $progressBar;
$this->progress = $progress;
$this->attributeSetLeftoverDataModel = $attributeSetLeftoverDataModel;
}

/**
* Records which are still in product entity tables
* but product attribute no longer exist in attribute set
*/
public function clean()
{
$entityValueIds = $this->attributeSetLeftoverDataModel->getLeftoverIds();
if (!$entityValueIds) {
return ;
}
foreach ($this->attributeSetLeftoverDataModel->getDocuments() as $document) {
$this->progressBar->advance(LogManager::LOG_LEVEL_INFO);
if (isset($entityValueIds[$document]) && $entityValueIds[$document]) {
$this->destination->deleteRecords($document, 'value_id', $entityValueIds[$document]);
}
}
}

/**
* Get documents
*
* @return array
*/
public function getDocuments()
{
return $this->attributeSetLeftoverDataModel->getDocuments();
}
}
81 changes: 81 additions & 0 deletions src/Migration/Step/PostProcessing/Data/DeletedRecordsCounter.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
<?php
/**
* Copyright © Magento, Inc. All rights reserved.
* See COPYING.txt for license details.
*/
namespace Migration\Step\PostProcessing\Data;

use Migration\ResourceModel;
use Migration\App\Progress;

/**
* Class DeletedRecordsCounter
*/
class DeletedRecordsCounter
{
/**
* @var ResourceModel\Destination
*/
private $destination;

/**
* @var array
*/
private $documentRecordsCount;

/**
* @var Progress
*/
private $progress;

/**
* @param ResourceModel\Destination $destination
* @param Progress $progress
*/
public function __construct(
ResourceModel\Destination $destination,
Progress $progress
) {
$this->destination = $destination;
$this->progress = $progress;
}

/**
* Count records in given documents
*
* @param $documents
*/
public function count($documents)
{
$documents = array_unique($documents);
foreach ($documents as $document) {
$recordsCount = $this->destination->getRecordsCount($document);
$this->documentRecordsCount[$document] = $recordsCount;
}
}

/**
* Compare current amount of records in given documents
* and save number of deleted records
*
* @param $documents
*/
public function saveDeleted($documents)
{
$documentsToSave = [];
$documents = array_unique($documents);
foreach ($documents as $document) {
$recordsCount = $this->destination->getRecordsCount($document);
if (isset($this->documentRecordsCount[$document])
&& $this->documentRecordsCount[$document] > $recordsCount
) {
$documentsToSave[$document] = $this->documentRecordsCount[$document] - $recordsCount;
}
}
$this->progress->saveProcessedEntities(
'PostProcessing',
'deletedDocumentRowsCount',
$documentsToSave
);
}
}
24 changes: 7 additions & 17 deletions src/Migration/Step/PostProcessing/Data/EavLeftoverDataCleaner.php
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,9 @@ class EavLeftoverDataCleaner
private $progress;

/**
* @var array
* @var EavLeftoverDataModel
*/
private $deletedDocumentRowsCount;
private $eavLeftoverDataModel;

/**
* @param ProgressBar\LogLevelProcessor $progressBar
Expand Down Expand Up @@ -65,29 +65,19 @@ public function clean()
if (!$attributeIds) {
return ;
}
foreach ($this->eavLeftoverDataModel->getDocumentsToCheck() as $document) {
foreach ($this->eavLeftoverDataModel->getDocuments() as $document) {
$this->progressBar->advance(LogManager::LOG_LEVEL_INFO);
$rowsCountBefore = $this->destination->getRecordsCount($document);
$this->destination->deleteRecords($document, 'attribute_id', $attributeIds);
$deletedCount = $rowsCountBefore - $this->destination->getRecordsCount($document);
if (!empty($deletedCount)) {
$this->deletedDocumentRowsCount[$document] = $deletedCount;
}
}
$this->progress->saveProcessedEntities(
'PostProcessing',
'deletedDocumentRowsCount',
$this->deletedDocumentRowsCount
);
}

/**
* Get iterations count
* Get documents
*
* @return int
* @return array
*/
public function getIterationsCount()
public function getDocuments()
{
return count($this->eavLeftoverDataModel->getDocumentsToCheck());
return $this->eavLeftoverDataModel->getDocuments(false);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -54,4 +54,14 @@ public function clean()
$productIds
);
}

/**
* Get documents
*
* @return array
*/
public function getDocuments()
{
return [$this->productsInRootCatalogModel->getCatalogCategoryProductDocument()];
}
}
Loading

0 comments on commit c5d7b65

Please sign in to comment.