Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve performance of "in" condition on some version of MySQl #27129

Merged
merged 20 commits into from
Sep 4, 2020
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions app/code/Magento/AdvancedSearch/Model/ResourceModel/Index.php
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ protected function _getCatalogProductPriceData($productIds = null)
['entity_id', 'customer_group_id', 'website_id', 'min_price']
);
if ($productIds) {
$select->where('entity_id IN (?)', $productIds);
$select->where('entity_id IN (?)', $productIds, \Zend_Db::BIGINT_TYPE);
}
$catalogProductIndexPriceSelect[] = $select;
}
Expand Down Expand Up @@ -177,7 +177,7 @@ public function getCategoryProductIndexData($storeId = null, $productIds = null)
);

if ($productIds) {
$select->where('product_id IN (?)', $productIds);
$select->where('product_id IN (?)', $productIds, \Zend_Db::BIGINT_TYPE);
}

$result = [];
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,7 @@ public function joinPrices($websiteId)
public function setOptionIdsFilter($optionIds)
{
if (!empty($optionIds)) {
$this->getSelect()->where('selection.option_id IN (?)', $optionIds);
$this->getSelect()->where('selection.option_id IN (?)', $optionIds, \Zend_Db::BIGINT_TYPE);
ihor-sviziev marked this conversation as resolved.
Show resolved Hide resolved
}
return $this;
}
Expand All @@ -229,7 +229,7 @@ public function setOptionIdsFilter($optionIds)
public function setSelectionIdsFilter($selectionIds)
{
if (!empty($selectionIds)) {
$this->getSelect()->where('selection.selection_id IN (?)', $selectionIds);
$this->getSelect()->where('selection.selection_id IN (?)', $selectionIds, \Zend_Db::BIGINT_TYPE);
}
return $this;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ private function fetch() : array
}

$linkCollection->getSelect()
->where($field . ' IN (?)', $this->parentIds);
->where($field . ' IN (?)', $this->parentIds, \Zend_Db::BIGINT_TYPE);

/** @var Selection $link */
foreach ($linkCollection as $link) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -414,7 +414,8 @@ private function getLinkIds(array $entityIds)
[$linkField]
)->where(
'e.entity_id IN (?)',
$entityIds
$entityIds,
\Zend_Db::BIGINT_TYPE
);

return $this->connection->fetchCol($select);
Expand Down Expand Up @@ -459,10 +460,12 @@ protected function getAttributeTypeValues($type, $entityIds, $storeId)
]
)->where(
"e.entity_id IN (?)",
$entityIds
$entityIds,
\Zend_Db::BIGINT_TYPE
)->where(
'def.store_id IN (?)',
[\Magento\Store\Model\Store::DEFAULT_STORE_ID, $storeId]
[\Magento\Store\Model\Store::DEFAULT_STORE_ID, $storeId],
\Zend_Db::BIGINT_TYPE
);

return $this->connection->fetchAll($select);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,8 @@ protected function filterIdsByStore(array $ids, $store)
"path = {$rootIdExpr} OR path = {$rootCatIdExpr} OR path like {$catIdExpr}"
)->where(
"entity_id IN (?)",
$ids
$ids,
\Zend_Db::BIGINT_TYPE
);

$resultIds = [];
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -502,10 +502,11 @@ protected function createAnchorSelect(Store $store)
[]
)->joinInner(
['cc2' => $temporaryTreeTable],
'cc2.parent_id = cc.entity_id AND cc.entity_id NOT IN (' . implode(
',',
$rootCatIds
) . ')',
$this->connection->quoteInto(
'cc2.parent_id = cc.entity_id AND cc.entity_id NOT IN (?)',
$rootCatIds,
\Zend_Db::BIGINT_TYPE
),
[]
)->joinInner(
['ccp' => $this->getTable('catalog_category_product')],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -282,7 +282,7 @@ private function reindexCategoriesBySelect(Select $basicSelect, $whereCondition,
$this->connection->delete($this->tableMaintainer->getMainTmpTable((int)$store->getId()));
$entityIds = $this->connection->fetchCol($query);
$resultSelect = clone $basicSelect;
$resultSelect->where($whereCondition, $entityIds);
$resultSelect->where($whereCondition, $entityIds, \Zend_Db::BIGINT_TYPE);
$this->connection->query(
$this->connection->insertFromSelect(
$resultSelect,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ private function getProductIdsWithParents(array $childProductIds): array
->select()
->from(['relation' => $this->getTable('catalog_product_relation')], [])
->distinct(true)
->where('child_id IN (?)', $childProductIds)
->where('child_id IN (?)', $childProductIds, \Zend_Db::BIGINT_TYPE)
->join(
['cpe' => $this->getTable('catalog_product_entity')],
'relation.parent_id = cpe.' . $fieldForParent,
Expand Down Expand Up @@ -215,7 +215,7 @@ protected function removeEntries()
protected function getNonAnchorCategoriesSelect(\Magento\Store\Model\Store $store)
{
$select = parent::getNonAnchorCategoriesSelect($store);
return $select->where('ccp.product_id IN (?)', $this->limitationByProducts);
return $select->where('ccp.product_id IN (?)', $this->limitationByProducts, \Zend_Db::BIGINT_TYPE);
}

/**
Expand All @@ -227,7 +227,7 @@ protected function getNonAnchorCategoriesSelect(\Magento\Store\Model\Store $stor
protected function getAnchorCategoriesSelect(\Magento\Store\Model\Store $store)
{
$select = parent::getAnchorCategoriesSelect($store);
return $select->where('ccp.product_id IN (?)', $this->limitationByProducts);
return $select->where('ccp.product_id IN (?)', $this->limitationByProducts, \Zend_Db::BIGINT_TYPE);
}

/**
Expand All @@ -239,7 +239,7 @@ protected function getAnchorCategoriesSelect(\Magento\Store\Model\Store $store)
protected function getAllProducts(\Magento\Store\Model\Store $store)
{
$select = parent::getAllProducts($store);
return $select->where('cp.entity_id IN (?)', $this->limitationByProducts);
return $select->where('cp.entity_id IN (?)', $this->limitationByProducts, \Zend_Db::BIGINT_TYPE);
}

/**
Expand All @@ -265,7 +265,7 @@ private function getCategoryIdsFromIndex(array $productIds): array
$storeCategories = $this->connection->fetchCol(
$this->connection->select()
->from($this->getIndexTable($store->getId()), ['category_id'])
->where('product_id IN (?)', $productIds)
->where('product_id IN (?)', $productIds, \Zend_Db::BIGINT_TYPE)
->distinct()
);
$categoryIds[] = $storeCategories;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -340,7 +340,9 @@ protected function _updateTemporaryTableByStoreValues(
[$attributeCode => 't.value']
);
if (!empty($changedIds)) {
$select->where($this->_connection->quoteInto('et.entity_id IN (?)', $changedIds));
$select->where(
$this->_connection->quoteInto('et.entity_id IN (?)', $changedIds, \Zend_Db::BIGINT_TYPE)
);
}
$sql = $select->crossUpdateFromSelect(['et' => $temporaryFlatTableName]);
$this->_connection->query($sql);
Expand All @@ -363,7 +365,9 @@ protected function _updateTemporaryTableByStoreValues(
[$columnName => $columnValue]
)->where($columnValue . ' IS NOT NULL');
if (!empty($changedIds)) {
$select->where($this->_connection->quoteInto('et.entity_id IN (?)', $changedIds));
$select->where(
$this->_connection->quoteInto('et.entity_id IN (?)', $changedIds, \Zend_Db::BIGINT_TYPE))
;
}
$sql = $select->crossUpdateFromSelect(['et' => $temporaryFlatTableName]);
$this->_connection->query($sql);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -233,7 +233,9 @@ protected function _fillTemporaryEntityTable($tableName, array $columns, array $
$select->from(['e' => $tableName], $columns);
$onDuplicate = false;
if (!empty($changedIds)) {
$select->where($this->_connection->quoteInto('e.entity_id IN (?)', $changedIds));
$select->where(
$this->_connection->quoteInto('e.entity_id IN (?)', $changedIds, \Zend_Db::BIGINT_TYPE)
);
$onDuplicate = true;
}
$sql = $select->insertFromSelect($temporaryEntityTable, $columns, $onDuplicate);
Expand Down Expand Up @@ -347,15 +349,19 @@ protected function _fillTemporaryTable(
}

if (!empty($changedIds)) {
$select->where($this->_connection->quoteInto('e.entity_id IN (?)', $changedIds));
$select->where(
$this->_connection->quoteInto('e.entity_id IN (?)', $changedIds, \Zend_Db::BIGINT_TYPE)
);
}

$sql = $select->insertFromSelect($temporaryTableName, $columns, true);
$this->_connection->query($sql);

if (count($valueColumns) > 1) {
if (!empty($changedIds)) {
$selectValue->where($this->_connection->quoteInto('e.entity_id IN (?)', $changedIds));
$selectValue->where(
$this->_connection->quoteInto('e.entity_id IN (?)', $changedIds, \Zend_Db::BIGINT_TYPE)
);
}
$sql = $selectValue->insertFromSelect($temporaryValueTableName, $valueColumns, true);
$this->_connection->query($sql);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -410,7 +410,7 @@ private function deleteIndexData(array $entityIds)
$select = $this->getConnection()->select()->from(
['index_price' => $this->tableMaintainer->getMainTable($dimensions)],
null
)->where('index_price.entity_id IN (?)', $entityIds);
)->where('index_price.entity_id IN (?)', $entityIds, \Zend_Db::BIGINT_TYPE);
$query = $select->deleteFromSelect('index_price');
$this->getConnection()->query($query);
}
Expand Down Expand Up @@ -519,7 +519,7 @@ private function getProductsTypes(array $changedIds = [])
['entity_id', 'type_id']
);
if ($changedIds) {
$select->where('entity_id IN (?)', $changedIds);
$select->where('entity_id IN (?)', $changedIds, \Zend_Db::BIGINT_TYPE);
}
$pairs = $this->getConnection()->fetchPairs($select);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ public function findAttributeSetIdsByProductIds(array $productIds)
->getSelect()
->reset(Select::COLUMNS)
->columns(ProductInterface::ATTRIBUTE_SET_ID)
->where('entity_id IN (?)', $productIds)
->where('entity_id IN (?)', $productIds, \Zend_Db::BIGINT_TYPE)
->group(ProductInterface::ATTRIBUTE_SET_ID);
$result = $collection->getConnection()->fetchCol($select);
return $result;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ public function get(array $skus)
->select()
->from($this->attributeResource->getTable($this->table));
return $this->attributeResource->getConnection()->fetchAll(
$select->where($this->getEntityLinkField() . ' IN (?)', $ids)
$select->where($this->getEntityLinkField() . ' IN (?)', $ids, \Zend_Db::BIGINT_TYPE)
->where('attribute_id = ?', $this->getAttributeId())
);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ public function get(array $ids)
{
$select = $this->tierpriceResource->getConnection()->select()->from($this->tierpriceResource->getMainTable());
return $this->tierpriceResource->getConnection()->fetchAll(
$select->where($this->getEntityLinkField() . ' IN (?)', $ids)
$select->where($this->getEntityLinkField() . ' IN (?)', $ids, \Zend_Db::BIGINT_TYPE)
);
}

Expand Down
14 changes: 11 additions & 3 deletions app/code/Magento/Catalog/Model/ResourceModel/AbstractResource.php
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ protected function _getLoadAttributesSelect($object, $table)
->select()
->from(['attr_table' => $table], [])
->where("attr_table.{$this->getLinkField()} = ?", $object->getData($this->getLinkField()))
->where('attr_table.store_id IN (?)', $storeIds);
->where('attr_table.store_id IN (?)', $storeIds, \Zend_Db::BIGINT_TYPE);

if ($setId) {
$select->join(
Expand Down Expand Up @@ -562,7 +562,11 @@ public function getAttributeRawValue($entityId, $attribute, $store)
if ($typedAttributes) {
foreach ($typedAttributes as $table => $_attributes) {
$defaultJoinCondition = [
$connection->quoteInto('default_value.attribute_id IN (?)', array_keys($_attributes)),
$connection->quoteInto(
'default_value.attribute_id IN (?)',
array_keys($_attributes),
\Zend_Db::BIGINT_TYPE
),
"default_value.{$this->getLinkField()} = e.{$this->getLinkField()}",
'default_value.store_id = 0',
];
Expand All @@ -589,7 +593,11 @@ public function getAttributeRawValue($entityId, $attribute, $store)
'store_value.attribute_id'
);
$joinCondition = [
$connection->quoteInto('store_value.attribute_id IN (?)', array_keys($_attributes)),
$connection->quoteInto(
'store_value.attribute_id IN (?)',
array_keys($_attributes),
\Zend_Db::BIGINT_TYPE
),
"store_value.{$this->getLinkField()} = e.{$this->getLinkField()}",
'store_value.store_id = :store_id',
];
Expand Down
7 changes: 4 additions & 3 deletions app/code/Magento/Catalog/Model/ResourceModel/Category.php
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,13 @@

namespace Magento\Catalog\Model\ResourceModel;

use Magento\Catalog\Api\Data\ProductInterface;
use Magento\Catalog\Model\Indexer\Category\Product\Processor;
use Magento\Catalog\Setup\CategorySetup;
use Magento\Framework\App\ObjectManager;
use Magento\Framework\DataObject;
use Magento\Framework\EntityManager\EntityManager;
use Magento\Catalog\Setup\CategorySetup;
use Magento\Framework\EntityManager\MetadataPool;
use Magento\Catalog\Api\Data\ProductInterface;

/**
* Resource model for category entity
Expand Down Expand Up @@ -666,7 +666,8 @@ public function findWhereAttributeIs($entityIdsFilter, $attribute, $expectedValu
'ci.value = :value'
)->where(
'ce.entity_id IN (?)',
$entityIdsFilter
$entityIdsFilter,
\Zend_Db::BIGINT_TYPE
);
$this->entitiesWhereAttributesIs[$entityIdsFilterHash][$attribute->getId()][$expectedValue] =
$this->getConnection()->fetchCol($selectEntities, $bind);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -294,7 +294,7 @@ protected function _loadNodes($parentNode = null, $recursionLevel = 0, $storeId
$inactiveCategories = $this->getInactiveCategoryIds();

if (!empty($inactiveCategories)) {
$select->where('main_table.entity_id NOT IN (?)', $inactiveCategories);
$select->where('main_table.entity_id NOT IN (?)', $inactiveCategories, \Zend_Db::BIGINT_TYPE);
}

// Allow extensions to modify select (e.g. add custom category attributes to select)
Expand Down Expand Up @@ -681,7 +681,8 @@ public function getAnchorsAbove(array $filterIds, $storeId = 0)
1
)->where(
'entity_id IN (?)',
$filterIds
$filterIds,
\Zend_Db::BIGINT_TYPE
);

return $this->getConnection()->fetchCol($select);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -170,10 +170,12 @@ protected function _getLoadAttributesSelect($table, $attributeIds = [])
['e.entity_id']
)->where(
"e.entity_id IN (?)",
array_keys($this->_itemsById)
array_keys($this->_itemsById),
\Zend_Db::BIGINT_TYPE
)->where(
't_d.attribute_id IN (?)',
$attributeIds
$attributeIds,
\Zend_Db::BIGINT_TYPE
)->joinLeft(
['t_s' => $table],
implode(' AND ', $joinCondition),
Expand All @@ -192,10 +194,12 @@ protected function _getLoadAttributesSelect($table, $attributeIds = [])
['e.entity_id']
)->where(
"e.entity_id IN (?)",
array_keys($this->_itemsById)
array_keys($this->_itemsById),
\Zend_Db::BIGINT_TYPE
)->where(
'attribute_id IN (?)',
$attributeIds
$attributeIds,
\Zend_Db::BIGINT_TYPE
)->where(
'store_id = ?',
$this->getDefaultStoreId()
Expand Down
8 changes: 5 additions & 3 deletions app/code/Magento/Catalog/Model/ResourceModel/Product.php
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,8 @@ public function getWebsiteIdsByProductIds($productIds)
['product_id', 'website_id']
)->where(
'product_id IN (?)',
$productIds
$productIds,
\Zend_Db::BIGINT_TYPE
);
$productsWebsites = [];
foreach ($this->getConnection()->fetchAll($select) as $productInfo) {
Expand Down Expand Up @@ -357,7 +358,7 @@ private function deleteSelectedEntityAttributeRows(DataObject $product, array $a
$entityId = $product->getData($entityIdField);
foreach ($backendTables as $backendTable => $attributes) {
$connection = $this->getConnection();
$where = $connection->quoteInto('attribute_id IN (?)', $attributes);
$where = $connection->quoteInto('attribute_id IN (?)', $attributes, \Zend_Db::BIGINT_TYPE);
$where .= $connection->quoteInto(" AND {$entityIdField} = ?", $entityId);
$connection->delete($backendTable, $where);
}
Expand Down Expand Up @@ -594,7 +595,8 @@ public function getProductsSku(array $productIds)
['entity_id', 'sku']
)->where(
'entity_id IN (?)',
$productIds
$productIds,
\Zend_Db::BIGINT_TYPE
);
return $this->getConnection()->fetchAll($select);
}
Expand Down
Loading