diff --git a/velox/functions/prestosql/ArrayConstructor.cpp b/velox/functions/prestosql/ArrayConstructor.cpp index 965fcaccb27e..67d239042dba 100644 --- a/velox/functions/prestosql/ArrayConstructor.cpp +++ b/velox/functions/prestosql/ArrayConstructor.cpp @@ -26,6 +26,24 @@ class ArrayConstructor : public exec::VectorFunction { return false; } + static bool shouldCopyRanges(const TypePtr& type) { + if (type->isPrimitiveType()) { + return false; + } + + if (type->isRow()) { + const auto& rowType = type->asRow(); + for (const auto& child : rowType.children()) { + if (shouldCopyRanges(child)) { + return true; + } + } + return false; + } + + return true; + } + void apply( const SelectivityVector& rows, std::vector& args, @@ -55,24 +73,55 @@ class ArrayConstructor : public exec::VectorFunction { } else { elementsResult->resize(baseOffset + numArgs * rows.countSelected()); - std::vector ranges; - ranges.reserve(rows.end()); + if (shouldCopyRanges(elementsResult->type())) { + std::vector ranges; + ranges.reserve(rows.end()); - vector_size_t offset = baseOffset; - rows.applyToSelected([&](vector_size_t row) { - rawSizes[row] = numArgs; - rawOffsets[row] = offset; - ranges.push_back({row, offset, 1}); - offset += numArgs; - }); + vector_size_t offset = baseOffset; + rows.applyToSelected([&](vector_size_t row) { + rawSizes[row] = numArgs; + rawOffsets[row] = offset; + ranges.push_back({row, offset, 1}); + offset += numArgs; + }); + + elementsResult->copyRanges(args[0].get(), ranges); + + for (int i = 1; i < numArgs; i++) { + for (auto& range : ranges) { + ++range.targetIndex; + } + elementsResult->copyRanges(args[i].get(), ranges); + } + } else { + SelectivityVector targetRows(elementsResult->size(), false); + std::vector toSourceRow(elementsResult->size()); + + vector_size_t offset = baseOffset; + rows.applyToSelected([&](vector_size_t row) { + rawSizes[row] = numArgs; + rawOffsets[row] = offset; + + targetRows.setValid(offset, true); + toSourceRow[offset] = row; + + offset += numArgs; + }); + targetRows.updateBounds(); + elementsResult->copy(args[0].get(), targetRows, toSourceRow.data()); - elementsResult->copyRanges(args[0].get(), ranges); + for (int i = 1; i < numArgs; i++) { + targetRows.clearAll(); + vector_size_t offset = baseOffset; + rows.applyToSelected([&](vector_size_t row) { + targetRows.setValid(offset + i, true); + toSourceRow[offset + i] = row; + offset += numArgs; + }); - for (int i = 1; i < numArgs; i++) { - for (auto& range : ranges) { - ++range.targetIndex; + targetRows.updateBounds(); + elementsResult->copy(args[i].get(), targetRows, toSourceRow.data()); } - elementsResult->copyRanges(args[i].get(), ranges); } } } diff --git a/velox/functions/prestosql/benchmarks/ArrayConstructorBenchmark.cpp b/velox/functions/prestosql/benchmarks/ArrayConstructorBenchmark.cpp new file mode 100644 index 000000000000..b9f3094f70b2 --- /dev/null +++ b/velox/functions/prestosql/benchmarks/ArrayConstructorBenchmark.cpp @@ -0,0 +1,94 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#include "velox/benchmarks/ExpressionBenchmarkBuilder.h" +#include "velox/functions/lib/LambdaFunctionUtil.h" +#include "velox/functions/lib/benchmarks/FunctionBenchmarkBase.h" +#include "velox/functions/prestosql/ArrayFunctions.h" +#include "velox/functions/prestosql/registration/RegistrationFunctions.h" + +using namespace facebook::velox; +using namespace facebook::velox::exec; +using namespace facebook::velox::functions; + +int main(int argc, char** argv) { + folly::init(&argc, &argv); + + functions::prestosql::registerArrayFunctions(); + + ExpressionBenchmarkBuilder benchmarkBuilder; + + auto* pool = benchmarkBuilder.pool(); + auto& vm = benchmarkBuilder.vectorMaker(); + + auto createSet = + [&](const TypePtr& type, bool withNulls, const VectorPtr& constantInput) { + VectorFuzzer::Options options; + options.vectorSize = 1'000; + options.nullRatio = withNulls ? 0.2 : 0.0; + + VectorFuzzer fuzzer(options, pool); + std::vector columns; + columns.push_back(fuzzer.fuzzFlat(type)); + columns.push_back(fuzzer.fuzzFlat(type)); + columns.push_back(fuzzer.fuzzFlat(type)); + columns.push_back( + BaseVector::createNullConstant(type, options.vectorSize, pool)); + columns.push_back( + BaseVector::wrapInConstant(options.vectorSize, 0, constantInput)); + + auto input = vm.rowVector({"c0", "c1", "c2", "n", "c"}, columns); + + benchmarkBuilder + .addBenchmarkSet( + fmt::format( + "array_constructor_{}_{}", + mapTypeKindToName(type->kind()), + withNulls ? "nulls" : "nullfree"), + input) + .addExpression("1", "array_constructor(c0)") + .addExpression("2", "array_constructor(c0, c1)") + .addExpression("3", "array_constructor(c0, c1, c2)") + .addExpression("2_null", "array_constructor(c0, c1, n)") + .addExpression("2_const", "array_constructor(c0, c1, c)"); + }; + + auto constantInteger = BaseVector::createConstant(INTEGER(), 11, 1, pool); + createSet(INTEGER(), true, constantInteger); + createSet(INTEGER(), false, constantInteger); + + auto constantRow = vm.rowVector({ + BaseVector::createConstant(INTEGER(), 11, 1, pool), + BaseVector::createConstant(DOUBLE(), 1.23, 1, pool), + }); + createSet(ROW({INTEGER(), DOUBLE()}), true, constantRow); + createSet(ROW({INTEGER(), DOUBLE()}), false, constantRow); + + auto constantArray = vm.arrayVector({{1, 2, 3, 4, 5}}); + createSet(ARRAY(INTEGER()), true, constantArray); + createSet(ARRAY(INTEGER()), false, constantArray); + + auto constantMap = vm.mapVector({{{1, 1.23}, {2, 2.34}}}); + createSet(MAP(INTEGER(), REAL()), true, constantMap); + createSet(MAP(INTEGER(), REAL()), false, constantMap); + + benchmarkBuilder.registerBenchmarks(); + + folly::runBenchmarks(); + return 0; +} diff --git a/velox/vector/ComplexVector.cpp b/velox/vector/ComplexVector.cpp index 14d3f7db3272..0adb8d85b006 100644 --- a/velox/vector/ComplexVector.cpp +++ b/velox/vector/ComplexVector.cpp @@ -231,17 +231,19 @@ void RowVector::copy( [&](auto row) { rawMappedIndices[row] = indices[toSourceRow[row]]; }); } - auto baseSource = decodedSource.base()->as(); - for (auto i = 0; i < childrenSize_; ++i) { - if (baseSource->childAt(i)) { - BaseVector::ensureWritable( - rows, type()->asRow().childAt(i), pool(), children_[i]); - children_[i]->copy( - baseSource->childAt(i)->loadedVector(), - nonNullRows, - rawMappedIndices ? rawMappedIndices : indices); - } else { - children_[i].reset(); + if (source->typeKind() != TypeKind::UNKNOWN) { + auto baseSource = decodedSource.base()->as(); + for (auto i = 0; i < childrenSize_; ++i) { + if (baseSource->childAt(i)) { + BaseVector::ensureWritable( + rows, type()->asRow().childAt(i), pool(), children_[i]); + children_[i]->copy( + baseSource->childAt(i)->loadedVector(), + nonNullRows, + rawMappedIndices ? rawMappedIndices : indices); + } else { + children_[i].reset(); + } } } } @@ -319,10 +321,13 @@ void RowVector::copyRanges( } } } - auto* rowSource = decoded.base()->as(); - for (int i = 0; i < children_.size(); ++i) { - children_[i]->copyRanges( - rowSource->childAt(i)->loadedVector(), baseRanges); + + if (source->typeKind() != TypeKind::UNKNOWN) { + auto* rowSource = decoded.base()->as(); + for (int i = 0; i < children_.size(); ++i) { + children_[i]->copyRanges( + rowSource->childAt(i)->loadedVector(), baseRanges); + } } } } diff --git a/velox/vector/FlatVector-inl.h b/velox/vector/FlatVector-inl.h index 58b30ebfbe34..40f827a8407a 100644 --- a/velox/vector/FlatVector-inl.h +++ b/velox/vector/FlatVector-inl.h @@ -218,19 +218,35 @@ void FlatVector::copyValuesAndNulls( } template -void FlatVector::copyValuesAndNulls( +void FlatVector::copyRanges( const BaseVector* source, - vector_size_t targetIndex, - vector_size_t sourceIndex, - vector_size_t count) { - if (count == 0) { - return; + const folly::Range& ranges) { + if constexpr (std::is_same_v) { + auto leaf = + source->wrappedVector()->asUnchecked>(); + if (BaseVector::pool_ != leaf->pool()) { + for (const auto& r : ranges) { + for (auto i = 0; i < r.count; ++i) { + if (source->isNullAt(r.sourceIndex + i)) { + this->setNull(r.targetIndex + i, true); + } else { + this->set( + r.targetIndex + i, + leaf->valueAt(source->wrappedIndex(r.sourceIndex + i))); + } + } + } + return; + } + + // We copy referencing the storage of 'source'. + acquireSharedStringBuffers(source); } + source = source->loadedVector(); VELOX_CHECK( BaseVector::compatibleKind(BaseVector::typeKind(), source->typeKind())); - VELOX_CHECK_GE(source->size(), sourceIndex + count); - VELOX_CHECK_GE(BaseVector::length_, targetIndex + count); + const uint64_t* sourceNulls = source->rawNulls(); uint64_t* rawNulls = const_cast(BaseVector::rawNulls_); if (source->mayHaveNulls()) { @@ -251,51 +267,85 @@ void FlatVector::copyValuesAndNulls( source->size()); } else if (source->typeKind() != TypeKind::UNKNOWN) { auto flat = source->asUnchecked>(); - if (Buffer::is_pod_like_v) { - memcpy( - &rawValues_[targetIndex], - &flat->rawValues()[sourceIndex], - count * sizeof(T)); - } else { - const T* srcValues = flat->rawValues(); - std::copy( - srcValues + sourceIndex, - srcValues + sourceIndex + count, - rawValues_ + targetIndex); + for (const auto& range : ranges) { + if (Buffer::is_pod_like_v) { + memcpy( + &rawValues_[range.targetIndex], + &flat->rawValues()[range.sourceIndex], + range.count * sizeof(T)); + } else { + const T* srcValues = flat->rawValues(); + std::copy( + srcValues + range.sourceIndex, + srcValues + range.sourceIndex + range.count, + rawValues_ + range.targetIndex); + } } } + if (rawNulls) { if (sourceNulls) { - bits::copyBits(sourceNulls, sourceIndex, rawNulls, targetIndex, count); + for (const auto& range : ranges) { + bits::copyBits( + sourceNulls, + range.sourceIndex, + rawNulls, + range.targetIndex, + range.count); + } } else { - bits::fillBits( - rawNulls, targetIndex, targetIndex + count, bits::kNotNull); + for (const auto& range : ranges) { + bits::fillBits( + rawNulls, + range.targetIndex, + range.targetIndex + range.count, + bits::kNotNull); + } } } } else if (source->isConstantEncoding()) { if (source->isNullAt(0)) { - bits::fillBits(rawNulls, targetIndex, targetIndex + count, bits::kNull); + for (const auto& range : ranges) { + bits::fillBits( + rawNulls, + range.targetIndex, + range.targetIndex + range.count, + bits::kNull); + } return; } auto constant = source->asUnchecked>(); T value = constant->valueAt(0); - for (auto row = targetIndex; row < targetIndex + count; ++row) { - rawValues_[row] = value; - } - if (rawNulls) { - bits::fillBits( - rawNulls, targetIndex, targetIndex + count, bits::kNotNull); + for (const auto& range : ranges) { + for (auto i = 0; i < range.count; ++i) { + rawValues_[range.targetIndex + i] = value; + } + if (rawNulls) { + bits::fillBits( + rawNulls, + range.targetIndex, + range.targetIndex + range.count, + bits::kNotNull); + } } } else { - auto sourceVector = source->asUnchecked>(); - for (int32_t i = 0; i < count; ++i) { - if (!source->isNullAt(sourceIndex + i)) { - rawValues_[targetIndex + i] = sourceVector->valueAt(sourceIndex + i); - if (rawNulls) { - bits::clearNull(rawNulls, targetIndex + i); + auto sourceVector = source->typeKind() != TypeKind::UNKNOWN + ? source->asUnchecked>() + : nullptr; + for (const auto& range : ranges) { + for (auto i = 0; i < range.count; ++i) { + auto row = range.targetIndex + i; + auto sourceRow = range.sourceIndex + i; + if (!source->isNullAt(sourceRow)) { + if (sourceVector) { + rawValues_[row] = sourceVector->valueAt(sourceRow); + } + if (rawNulls) { + bits::clearNull(rawNulls, row); + } + } else { + bits::setNull(rawNulls, row); } - } else { - bits::setNull(rawNulls, targetIndex + i); } } } diff --git a/velox/vector/FlatVector.cpp b/velox/vector/FlatVector.cpp index d6abd2ccccf4..f3ef70b72353 100644 --- a/velox/vector/FlatVector.cpp +++ b/velox/vector/FlatVector.cpp @@ -131,19 +131,15 @@ void FlatVector::copyValuesAndNulls( } template <> -void FlatVector::copyValuesAndNulls( +void FlatVector::copyRanges( const BaseVector* source, - vector_size_t targetIndex, - vector_size_t sourceIndex, - vector_size_t count) { - if (count == 0) { + const folly::Range& ranges) { + if (ranges.size() == 0) { return; } source = source->loadedVector(); VELOX_CHECK( BaseVector::compatibleKind(BaseVector::typeKind(), source->typeKind())); - VELOX_CHECK(source->size() >= sourceIndex + count); - VELOX_CHECK(BaseVector::length_ >= targetIndex + count); const uint64_t* sourceNulls = source->rawNulls(); auto rawValues = reinterpret_cast(rawValues_); @@ -151,49 +147,83 @@ void FlatVector::copyValuesAndNulls( if (source->mayHaveNulls()) { rawNulls = BaseVector::mutableRawNulls(); } + if (source->isFlatEncoding()) { if (source->typeKind() != TypeKind::UNKNOWN) { auto* sourceValues = source->asUnchecked>()->rawValues(); - bits::copyBits(sourceValues, sourceIndex, rawValues, targetIndex, count); + for (const auto& range : ranges) { + bits::copyBits( + sourceValues, + range.sourceIndex, + rawValues, + range.targetIndex, + range.count); + } } if (rawNulls) { if (sourceNulls) { - bits::copyBits(sourceNulls, sourceIndex, rawNulls, targetIndex, count); + for (const auto& range : ranges) { + bits::copyBits( + sourceNulls, + range.sourceIndex, + rawNulls, + range.targetIndex, + range.count); + } } else { - bits::fillBits( - rawNulls, targetIndex, targetIndex + count, bits::kNotNull); + for (const auto& range : ranges) { + bits::fillBits( + rawNulls, + range.targetIndex, + range.targetIndex + range.count, + bits::kNotNull); + } } } } else if (source->isConstantEncoding()) { auto constant = source->asUnchecked>(); if (constant->isNullAt(0)) { - bits::fillBits(rawNulls, targetIndex, targetIndex + count, bits::kNull); + for (const auto& range : ranges) { + bits::fillBits( + rawNulls, + range.targetIndex, + range.targetIndex + range.count, + bits::kNull); + } return; } bool value = constant->valueAt(0); - bits::fillBits(rawValues, targetIndex, targetIndex + count, value); - if (rawNulls) { + for (const auto& range : ranges) { bits::fillBits( - rawNulls, targetIndex, targetIndex + count, bits::kNotNull); + rawValues, range.targetIndex, range.targetIndex + range.count, value); + if (rawNulls) { + bits::fillBits( + rawNulls, + range.targetIndex, + range.targetIndex + range.count, + bits::kNotNull); + } } } else { auto sourceVector = source->typeKind() != TypeKind::UNKNOWN ? source->asUnchecked>() : nullptr; - for (int32_t i = 0; i < count; ++i) { - if (!source->isNullAt(sourceIndex + i)) { - if (sourceVector) { - bits::setBit( - rawValues, - targetIndex + i, - sourceVector->valueAt(sourceIndex + i)); - } - if (rawNulls) { - bits::clearNull(rawNulls, targetIndex + i); + for (const auto& range : ranges) { + for (int32_t i = 0; i < range.count; ++i) { + if (!source->isNullAt(range.sourceIndex + i)) { + if (sourceVector) { + bits::setBit( + rawValues, + range.targetIndex + i, + sourceVector->valueAt(range.sourceIndex + i)); + } + if (rawNulls) { + bits::clearNull(rawNulls, range.targetIndex + i); + } + } else { + bits::setNull(rawNulls, range.targetIndex + i); } - } else { - bits::setNull(rawNulls, targetIndex + i); } } } @@ -461,43 +491,6 @@ void FlatVector::copy( } } -template <> -void FlatVector::copy( - const BaseVector* source, - vector_size_t targetIndex, - vector_size_t sourceIndex, - vector_size_t count) { - if (count == 0) { - return; - } - BaseVector::copy(source, targetIndex, sourceIndex, count); -} - -template <> -void FlatVector::copyRanges( - const BaseVector* source, - const folly::Range& ranges) { - auto leaf = source->wrappedVector()->asUnchecked>(); - if (pool_ == leaf->pool()) { - // We copy referencing the storage of 'source'. - for (auto& r : ranges) { - copyValuesAndNulls(source, r.targetIndex, r.sourceIndex, r.count); - } - acquireSharedStringBuffers(source); - } else { - for (auto& r : ranges) { - for (auto i = 0; i < r.count; ++i) { - if (source->isNullAt(r.sourceIndex + i)) { - setNull(r.targetIndex + i, true); - } else { - set(r.targetIndex + i, - leaf->valueAt(source->wrappedIndex(r.sourceIndex + i))); - } - } - } - } -} - // For strings, we also verify if they point to valid memory locations inside // the string buffers. template <> diff --git a/velox/vector/FlatVector.h b/velox/vector/FlatVector.h index 10591708a4fd..5bb11450a92f 100644 --- a/velox/vector/FlatVector.h +++ b/velox/vector/FlatVector.h @@ -228,16 +228,13 @@ class FlatVector final : public SimpleVector { if (count == 0) { return; } - copyValuesAndNulls(source, targetIndex, sourceIndex, count); + BaseVector::CopyRange range{sourceIndex, targetIndex, count}; + copyRanges(source, folly::Range(&range, 1)); } void copyRanges( const BaseVector* source, - const folly::Range& ranges) override { - for (auto& range : ranges) { - copy(source, range.targetIndex, range.sourceIndex, range.count); - } - } + const folly::Range& ranges) override; void resize(vector_size_t newSize, bool setNotNull = true) override; @@ -449,12 +446,6 @@ class FlatVector final : public SimpleVector { const SelectivityVector& rows, const vector_size_t* toSourceRow); - void copyValuesAndNulls( - const BaseVector* source, - vector_size_t targetIndex, - vector_size_t sourceIndex, - vector_size_t count); - // Ensures that the values buffer has space for 'newSize' elements and is // mutable. Sets elements between the old and new sizes to 'initialValue' if // the new size > old size. @@ -508,18 +499,6 @@ void FlatVector::copy( const SelectivityVector& rows, const vector_size_t* toSourceRow); -template <> -void FlatVector::copy( - const BaseVector* source, - vector_size_t targetIndex, - vector_size_t sourceIndex, - vector_size_t count); - -template <> -void FlatVector::copyRanges( - const BaseVector* source, - const folly::Range& ranges); - template <> void FlatVector::validate( const VectorValidateOptions& options) const; @@ -531,11 +510,9 @@ void FlatVector::copyValuesAndNulls( const vector_size_t* toSourceRow); template <> -void FlatVector::copyValuesAndNulls( +void FlatVector::copyRanges( const BaseVector* source, - vector_size_t targetIndex, - vector_size_t sourceIndex, - vector_size_t count); + const folly::Range& ranges); template <> Buffer* FlatVector::getBufferWithSpace(vector_size_t size);