Skip to content

Commit

Permalink
[native]Add to config largest size class pages
Browse files Browse the repository at this point in the history
  • Loading branch information
xiaoxmeng committed Jun 25, 2024
1 parent 95da85b commit dbfb6ac
Show file tree
Hide file tree
Showing 4 changed files with 15 additions and 1 deletion.
1 change: 1 addition & 0 deletions presto-native-execution/presto_cpp/main/PrestoServer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -717,6 +717,7 @@ void PrestoServer::initializeVeloxMemory() {
options.memoryReclaimWaitMs = systemConfig->memoryReclaimWaitMs();
options.globalArbitrationEnabled =
systemConfig->memoryArbitratorGlobalArbitrationEnabled();
options.largestSizeClassPages = systemConfig->largestSizeClassPages();
options.arbitrationStateCheckCb = velox::exec::memoryArbitrationStateCheck;
}
memory::initializeMemoryManager(options);
Expand Down
5 changes: 5 additions & 0 deletions presto-native-execution/presto_cpp/main/common/Configs.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,7 @@ SystemConfig::SystemConfig() {
BOOL_PROP(kMemoryArbitratorGlobalArbitrationEnabled, false),
NUM_PROP(kQueryMemoryGb, 38),
NUM_PROP(kQueryReservedMemoryGb, 4),
NUM_PROP(kLargestSizeClassPages, 256),
BOOL_PROP(kEnableVeloxTaskLogging, false),
BOOL_PROP(kEnableVeloxExprSetLogging, false),
NUM_PROP(kLocalShuffleMaxPartitionBytes, 268435456),
Expand Down Expand Up @@ -652,6 +653,10 @@ std::chrono::duration<double> SystemConfig::cacheVeloxTtlCheckInterval() const {
optionalProperty(kCacheVeloxTtlCheckInterval).value());
}

int32_t SystemConfig::largestSizeClassPages() const {
return optionalProperty<int32_t>(kLargestSizeClassPages).value();
}

bool SystemConfig::enableRuntimeMetricsCollection() const {
return optionalProperty<bool>(kEnableRuntimeMetricsCollection).value();
}
Expand Down
8 changes: 8 additions & 0 deletions presto-native-execution/presto_cpp/main/common/Configs.h
Original file line number Diff line number Diff line change
Expand Up @@ -345,6 +345,12 @@ class SystemConfig : public ConfigBase {
"cache.velox.ttl-check-interval"};
static constexpr std::string_view kUseMmapAllocator{"use-mmap-allocator"};

/// Number of pages in largest size class in MallocAllocator. This is used to
/// optimize MmapAllocator performance for query workloads with large memory
/// allocation size.
static constexpr std::string_view kLargestSizeClassPages{
"largest-size-class-pages"};

static constexpr std::string_view kEnableRuntimeMetricsCollection{
"runtime-metrics-collection-enabled"};

Expand Down Expand Up @@ -740,6 +746,8 @@ class SystemConfig : public ConfigBase {

std::chrono::duration<double> cacheVeloxTtlCheckInterval() const;

int32_t largestSizeClassPages() const;

bool enableRuntimeMetricsCollection() const;

bool prestoNativeSidecar() const;
Expand Down
2 changes: 1 addition & 1 deletion presto-native-execution/velox
Submodule velox updated 87 files
+42 −10 velox/common/base/AsyncSource.h
+70 −0 velox/common/base/tests/AsyncSourceTest.cpp
+1 −0 velox/common/base/tests/StatsReporterTest.cpp
+1 −0 velox/common/memory/Memory.cpp
+3 −0 velox/common/memory/Memory.h
+12 −0 velox/common/memory/MemoryAllocator.cpp
+5 −1 velox/common/memory/MemoryAllocator.h
+10 −1 velox/common/memory/MemoryArbitrator.cpp
+9 −19 velox/common/memory/MemoryArbitrator.h
+2 −1 velox/common/memory/MmapAllocator.cpp
+2 −0 velox/common/memory/MmapAllocator.h
+46 −0 velox/common/memory/tests/MemoryAllocatorTest.cpp
+1 −1 velox/common/memory/tests/MockSharedArbitratorTest.cpp
+11 −2 velox/common/process/ThreadDebugInfo.cpp
+3 −0 velox/common/process/ThreadDebugInfo.h
+3 −0 velox/connectors/hive/HiveConnector.cpp
+28 −1 velox/connectors/hive/iceberg/tests/CMakeLists.txt
+371 −0 velox/connectors/hive/iceberg/tests/IcebergSplitReaderBenchmark.cpp
+132 −0 velox/connectors/hive/iceberg/tests/IcebergSplitReaderBenchmark.h
+67 −0 velox/connectors/hive/iceberg/tests/IcebergSplitReaderBenchmarkMain.cpp
+30 −0 velox/connectors/hive/iceberg/tests/IcebergSplitReaderBenchmarkTest.cpp
+5 −1 velox/docs/functions/presto/conversion.rst
+6 −0 velox/docs/functions/presto/datetime.rst
+6 −1 velox/docs/functions/spark/math.rst
+5 −0 velox/dwio/common/BufferedInput.h
+16 −19 velox/dwio/common/CacheInputStream.cpp
+4 −0 velox/dwio/common/CachedBufferedInput.h
+4 −0 velox/dwio/common/DirectBufferedInput.h
+36 −17 velox/dwio/common/Retry.h
+23 −10 velox/dwio/common/tests/RetryTests.cpp
+25 −0 velox/dwio/dwrf/RegisterDwrfReader.h
+25 −0 velox/dwio/dwrf/RegisterDwrfWriter.h
+0 −4 velox/dwio/dwrf/reader/DwrfReader.h
+13 −0 velox/dwio/dwrf/reader/ReaderBase.cpp
+118 −2 velox/dwio/dwrf/test/CacheInputTest.cpp
+0 −4 velox/dwio/dwrf/writer/Writer.h
+0 −2 velox/dwio/parquet/RegisterParquetReader.cpp
+0 −2 velox/dwio/parquet/RegisterParquetWriter.cpp
+1 −1 velox/examples/ScanAndSort.cpp
+9 −5 velox/examples/ScanOrc.cpp
+1 −1 velox/exec/HashBuild.cpp
+3 −3 velox/exec/HashProbe.cpp
+1 −1 velox/exec/MemoryReclaimer.cpp
+7 −0 velox/exec/fuzzer/AggregationFuzzer.cpp
+13 −1 velox/exec/fuzzer/AggregationFuzzerBase.cpp
+2 −0 velox/exec/fuzzer/AggregationFuzzerBase.h
+0 −1 velox/exec/fuzzer/FuzzerUtil.cpp
+0 −2 velox/exec/fuzzer/JoinFuzzer.cpp
+50 −2 velox/exec/fuzzer/MemoryArbitrationFuzzer.cpp
+0 −2 velox/exec/fuzzer/RowNumberFuzzer.cpp
+0 −9 velox/exec/fuzzer/WindowFuzzer.cpp
+0 −2 velox/exec/fuzzer/WriterFuzzer.cpp
+2 −2 velox/exec/tests/PrintPlanWithStatsTest.cpp
+0 −2 velox/exec/tests/utils/HiveConnectorTestBase.cpp
+16 −17 velox/expression/tests/CastExprTest.cpp
+23 −22 velox/functions/lib/KllSketch-inl.h
+26 −24 velox/functions/lib/KllSketch.h
+0 −1 velox/functions/prestosql/CMakeLists.txt
+47 −6 velox/functions/prestosql/DateTimeFunctions.h
+1 −1 velox/functions/prestosql/DateTimeImpl.h
+0 −97 velox/functions/prestosql/FromUnixTime.cpp
+17 −18 velox/functions/prestosql/InPredicate.cpp
+59 −34 velox/functions/prestosql/aggregates/ApproxPercentileAggregate.cpp
+31 −0 velox/functions/prestosql/aggregates/ArrayAggAggregate.cpp
+12 −2 velox/functions/prestosql/aggregates/RegisterAggregateFunctions.cpp
+4 −3 velox/functions/prestosql/aggregates/RegisterAggregateFunctions.h
+6 −3 velox/functions/prestosql/aggregates/SetAggregates.cpp
+49 −32 velox/functions/prestosql/aggregates/tests/ArrayAggTest.cpp
+6 −4 velox/functions/prestosql/fuzzer/AggregationFuzzerTest.cpp
+7 −0 velox/functions/prestosql/fuzzer/CMakeLists.txt
+398 −0 velox/functions/prestosql/fuzzer/MinMaxByResultVerifier.cpp
+118 −0 velox/functions/prestosql/fuzzer/MinMaxByResultVerifier.h
+1 −0 velox/functions/prestosql/fuzzer/WindowFuzzerTest.cpp
+18 −3 velox/functions/prestosql/registration/DateTimeFunctionsRegistration.cpp
+21 −0 velox/functions/prestosql/tests/DateTimeFunctionsTest.cpp
+114 −0 velox/functions/prestosql/tests/InPredicateTest.cpp
+11 −0 velox/functions/sparksql/Arithmetic.h
+9 −0 velox/functions/sparksql/LeastGreatest.cpp
+1 −0 velox/functions/sparksql/LeastGreatest.h
+2 −0 velox/functions/sparksql/RegisterArithmetic.cpp
+27 −1 velox/functions/sparksql/tests/ArithmeticTest.cpp
+16 −0 velox/functions/sparksql/tests/LeastGreatestTest.cpp
+32 −0 velox/type/Conversions.cpp
+18 −0 velox/type/Conversions.h
+27 −1 velox/type/tz/TimeZoneMap.cpp
+8 −4 velox/type/tz/TimeZoneMap.h
+14 −0 velox/type/tz/tests/TimeZoneMapTest.cpp

0 comments on commit dbfb6ac

Please sign in to comment.