diff --git a/backends-clickhouse/src-celeborn/main/scala/org/apache/spark/shuffle/CHCelebornColumnarBatchSerializer.scala b/backends-clickhouse/src-celeborn/main/scala/org/apache/spark/shuffle/CHCelebornColumnarBatchSerializer.scala index 72145f1b5f5c..c443ae568dab 100644 --- a/backends-clickhouse/src-celeborn/main/scala/org/apache/spark/shuffle/CHCelebornColumnarBatchSerializer.scala +++ b/backends-clickhouse/src-celeborn/main/scala/org/apache/spark/shuffle/CHCelebornColumnarBatchSerializer.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.shuffle -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.clickhouse.CHBackendSettings +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.vectorized.BlockOutputStream import org.apache.gluten.vectorized.CHStreamReader @@ -59,14 +59,14 @@ private class CHCelebornColumnarBatchSerializerInstance( with Logging { private lazy val conf = SparkEnv.get.conf - private lazy val gluten_conf = GlutenConfig.getConf + private lazy val gluten_conf = GlutenConfig.get private lazy val compressionCodec = GlutenShuffleUtils.getCompressionCodec(conf) private lazy val capitalizedCompressionCodec = compressionCodec.toUpperCase(Locale.ROOT) private lazy val compressionLevel = GlutenShuffleUtils.getCompressionLevel( conf, compressionCodec, - GlutenConfig.getConf.columnarShuffleCodecBackend.orNull) + GlutenConfig.get.columnarShuffleCodecBackend.orNull) override def deserializeStream(in: InputStream): DeserializationStream = { new DeserializationStream { diff --git a/backends-clickhouse/src-celeborn/main/scala/org/apache/spark/shuffle/CHCelebornColumnarShuffleWriter.scala b/backends-clickhouse/src-celeborn/main/scala/org/apache/spark/shuffle/CHCelebornColumnarShuffleWriter.scala index 2263cda7eafb..dc02fd40f01c 100644 --- a/backends-clickhouse/src-celeborn/main/scala/org/apache/spark/shuffle/CHCelebornColumnarShuffleWriter.scala +++ b/backends-clickhouse/src-celeborn/main/scala/org/apache/spark/shuffle/CHCelebornColumnarShuffleWriter.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.shuffle -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.clickhouse.CHBackendSettings +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.ColumnarNativeIterator import org.apache.gluten.memory.CHThreadGroup import org.apache.gluten.vectorized._ @@ -80,10 +80,10 @@ class CHCelebornColumnarShuffleWriter[K, V]( nativeBufferSize, capitalizedCompressionCodec, compressionLevel, - GlutenConfig.getConf.chColumnarShuffleSpillThreshold, + GlutenConfig.get.chColumnarShuffleSpillThreshold, CHBackendSettings.shuffleHashAlgorithm, celebornPartitionPusher, - GlutenConfig.getConf.chColumnarForceMemorySortShuffle + GlutenConfig.get.chColumnarForceMemorySortShuffle || ShuffleMode.SORT.name.equalsIgnoreCase(shuffleWriterType) ) diff --git a/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala index 6c7b2ab87664..082515b02c06 100644 --- a/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala +++ b/backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.delta -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.clickhouse.CHConf +import org.apache.gluten.config.GlutenConfig import org.apache.spark.SparkException import org.apache.spark.sql.Dataset @@ -68,7 +68,7 @@ class ClickhouseOptimisticTransaction( writeOptions: Option[DeltaOptions], isOptimize: Boolean, additionalConstraints: Seq[Constraint]): Seq[FileAction] = { - val nativeWrite = GlutenConfig.getConf.enableNativeWriter.getOrElse(false) + val nativeWrite = GlutenConfig.get.enableNativeWriter.getOrElse(false) if (writingMergeTree) { // TODO: update FallbackByBackendSettings for mergetree always return true val onePipeline = nativeWrite && CHConf.get.enableOnePipelineMergeTreeWrite diff --git a/backends-clickhouse/src-delta-32/test/scala/org/apache/spark/sql/execution/datasources/DeltaV1WritesSuite.scala b/backends-clickhouse/src-delta-32/test/scala/org/apache/spark/sql/execution/datasources/DeltaV1WritesSuite.scala index 1a90148df29e..6db41f56c53d 100644 --- a/backends-clickhouse/src-delta-32/test/scala/org/apache/spark/sql/execution/datasources/DeltaV1WritesSuite.scala +++ b/backends-clickhouse/src-delta-32/test/scala/org/apache/spark/sql/execution/datasources/DeltaV1WritesSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.execution.datasources -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{GlutenClickHouseWholeStageTransformerSuite, GlutenPlan, SortExecTransformer} import org.apache.spark.SparkConf import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat diff --git a/backends-clickhouse/src-iceberg/test/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergSuite.scala b/backends-clickhouse/src-iceberg/test/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergSuite.scala index 676061235ed0..86b1caeffcef 100644 --- a/backends-clickhouse/src-iceberg/test/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergSuite.scala +++ b/backends-clickhouse/src-iceberg/test/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergSuite.scala @@ -17,7 +17,7 @@ package org.apache.gluten.execution.iceberg import org.apache.gluten.execution.{GlutenClickHouseWholeStageTransformerSuite, IcebergScanTransformer} -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.SparkConf import org.apache.spark.sql.Row @@ -62,7 +62,7 @@ class ClickHouseIcebergSuite extends GlutenClickHouseWholeStageTransformerSuite val rightTable = "p_int_tb" withTable(leftTable, rightTable) { // Partition key of string type. - withSQLConf(GlutenConfig.GLUTEN_ENABLED_KEY -> "false") { + withSQLConf(GlutenConfig.GLUTEN_ENABLED.key -> "false") { // Gluten does not support write iceberg table. spark.sql( s""" @@ -84,7 +84,7 @@ class ClickHouseIcebergSuite extends GlutenClickHouseWholeStageTransformerSuite // Partition key of integer type. withSQLConf( - GlutenConfig.GLUTEN_ENABLED_KEY -> "false" + GlutenConfig.GLUTEN_ENABLED.key -> "false" ) { // Gluten does not support write iceberg table. spark.sql( @@ -145,7 +145,7 @@ class ClickHouseIcebergSuite extends GlutenClickHouseWholeStageTransformerSuite val rightTable = "p_int_tb" withTable(leftTable, rightTable) { // Partition key of string type. - withSQLConf(GlutenConfig.GLUTEN_ENABLED_KEY -> "false") { + withSQLConf(GlutenConfig.GLUTEN_ENABLED.key -> "false") { // Gluten does not support write iceberg table. spark.sql( s""" @@ -167,7 +167,7 @@ class ClickHouseIcebergSuite extends GlutenClickHouseWholeStageTransformerSuite // Partition key of integer type. withSQLConf( - GlutenConfig.GLUTEN_ENABLED_KEY -> "false" + GlutenConfig.GLUTEN_ENABLED.key -> "false" ) { // Gluten does not support write iceberg table. spark.sql( @@ -228,7 +228,7 @@ class ClickHouseIcebergSuite extends GlutenClickHouseWholeStageTransformerSuite val rightTable = "p_int_tb" withTable(leftTable, rightTable) { // Partition key of string type. - withSQLConf(GlutenConfig.GLUTEN_ENABLED_KEY -> "false") { + withSQLConf(GlutenConfig.GLUTEN_ENABLED.key -> "false") { // Gluten does not support write iceberg table. spark.sql( s""" @@ -250,7 +250,7 @@ class ClickHouseIcebergSuite extends GlutenClickHouseWholeStageTransformerSuite // Partition key of integer type. withSQLConf( - GlutenConfig.GLUTEN_ENABLED_KEY -> "false" + GlutenConfig.GLUTEN_ENABLED.key -> "false" ) { // Gluten does not support write iceberg table. spark.sql( @@ -350,7 +350,7 @@ class ClickHouseIcebergSuite extends GlutenClickHouseWholeStageTransformerSuite test("iceberg read mor table - delete and update") { withTable("iceberg_mor_tb") { - withSQLConf(GlutenConfig.GLUTEN_ENABLED_KEY -> "false") { + withSQLConf(GlutenConfig.GLUTEN_ENABLED.key -> "false") { spark.sql( """ |create table iceberg_mor_tb ( @@ -403,7 +403,7 @@ class ClickHouseIcebergSuite extends GlutenClickHouseWholeStageTransformerSuite // TODO: support merge-on-read mode ignore("iceberg read mor table - delete and update with merge-on-read mode") { withTable("iceberg_mor_tb") { - withSQLConf(GlutenConfig.GLUTEN_ENABLED_KEY -> "false") { + withSQLConf(GlutenConfig.GLUTEN_ENABLED.key -> "false") { spark.sql( """ |create table iceberg_mor_tb ( @@ -458,7 +458,7 @@ class ClickHouseIcebergSuite extends GlutenClickHouseWholeStageTransformerSuite test("iceberg read mor table - merge into") { withTable("iceberg_mor_tb", "merge_into_source_tb") { - withSQLConf(GlutenConfig.GLUTEN_ENABLED_KEY -> "false") { + withSQLConf(GlutenConfig.GLUTEN_ENABLED.key -> "false") { spark.sql( """ |create table iceberg_mor_tb ( @@ -531,7 +531,7 @@ class ClickHouseIcebergSuite extends GlutenClickHouseWholeStageTransformerSuite // TODO: support merge-on-read mode ignore("iceberg read mor table - merge into with merge-on-read mode") { withTable("iceberg_mor_tb", "merge_into_source_tb") { - withSQLConf(GlutenConfig.GLUTEN_ENABLED_KEY -> "false") { + withSQLConf(GlutenConfig.GLUTEN_ENABLED.key -> "false") { spark.sql( """ |create table iceberg_mor_tb ( diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHNativeExpressionEvaluator.java b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHNativeExpressionEvaluator.java index 2099ddbbf545..12b79a54ce27 100644 --- a/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHNativeExpressionEvaluator.java +++ b/backends-clickhouse/src/main/java/org/apache/gluten/vectorized/CHNativeExpressionEvaluator.java @@ -16,8 +16,8 @@ */ package org.apache.gluten.vectorized; -import org.apache.gluten.GlutenConfig; import org.apache.gluten.backendsapi.BackendsApiManager; +import org.apache.gluten.config.GlutenConfig; import org.apache.gluten.execution.ColumnarNativeIterator; import org.apache.gluten.memory.CHThreadGroup; import org.apache.gluten.utils.ConfigUtil; diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala index 97d862e13c06..924f3b31704e 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHBackend.scala @@ -17,10 +17,10 @@ package org.apache.gluten.backendsapi.clickhouse import org.apache.gluten.GlutenBuildInfo._ -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi._ import org.apache.gluten.columnarbatch.CHBatch import org.apache.gluten.component.Component.BuildInfo +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.WriteFilesExecTransformer import org.apache.gluten.expression.WindowFunctionsBuilder import org.apache.gluten.extension.ValidationResult @@ -282,11 +282,11 @@ object CHBackendSettings extends BackendSettingsApi with Logging { } override def supportSortExec(): Boolean = { - GlutenConfig.getConf.enableColumnarSort + GlutenConfig.get.enableColumnarSort } override def supportSortMergeJoinExec(): Boolean = { - GlutenConfig.getConf.enableColumnarSortMergeJoin + GlutenConfig.get.enableColumnarSortMergeJoin } override def supportWindowExec(windowFunctions: Seq[NamedExpression]): Boolean = { @@ -391,7 +391,7 @@ object CHBackendSettings extends BackendSettingsApi with Logging { } override def enableNativeWriteFiles(): Boolean = { - GlutenConfig.getConf.enableNativeWriter.getOrElse(false) + GlutenConfig.get.enableNativeWriter.getOrElse(false) } override def supportCartesianProductExec(): Boolean = true diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHConf.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHConf.scala index 83cc75bc0ac7..14bcf5d01153 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHConf.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHConf.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.backendsapi.clickhouse -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.SparkConf import org.apache.spark.sql.internal.SQLConf diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala index 878e27a5b8c2..0ceb5cd0eaf6 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.backendsapi.clickhouse -import org.apache.gluten.GlutenNumaBindingInfo import org.apache.gluten.backendsapi.IteratorApi +import org.apache.gluten.config.GlutenNumaBindingInfo import org.apache.gluten.execution._ import org.apache.gluten.expression.ConverterUtils import org.apache.gluten.logging.LogLevelUtil diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala index b93c002561f0..f80db0aee69b 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala @@ -16,9 +16,9 @@ */ package org.apache.gluten.backendsapi.clickhouse -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.ListenerApi import org.apache.gluten.columnarbatch.CHBatch +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.CHBroadcastBuildSideCache import org.apache.gluten.execution.datasource.GlutenFormatFactory import org.apache.gluten.expression.UDFMappings diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHRuleApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHRuleApi.scala index 05a50396fca3..e6ee3f79ee32 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHRuleApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHRuleApi.scala @@ -16,9 +16,9 @@ */ package org.apache.gluten.backendsapi.clickhouse -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.RuleApi import org.apache.gluten.columnarbatch.CHBatch +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.extension._ import org.apache.gluten.extension.columnar._ import org.apache.gluten.extension.columnar.MiscColumnarRules.{RemoveGlutenTableCacheColumnarToRow, RemoveTopmostColumnarToRow, RewriteSubqueryBroadcast} diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala index 635228881566..679ac4435c72 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.backendsapi.clickhouse -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.{BackendsApiManager, SparkPlanExecApi} +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.exception.{GlutenException, GlutenNotSupportException} import org.apache.gluten.execution._ import org.apache.gluten.expression._ @@ -452,12 +452,12 @@ class CHSparkPlanExecApi extends SparkPlanExecApi with Logging { val readBatchNumRows = metrics("avgReadBatchNumRows") val numOutputRows = metrics("numOutputRows") val dataSize = metrics("dataSize") - if (GlutenConfig.getConf.isUseCelebornShuffleManager) { + if (GlutenConfig.get.isUseCelebornShuffleManager) { val clazz = ClassUtils.getClass("org.apache.spark.shuffle.CHCelebornColumnarBatchSerializer") val constructor = clazz.getConstructor(classOf[SQLMetric], classOf[SQLMetric], classOf[SQLMetric]) constructor.newInstance(readBatchNumRows, numOutputRows, dataSize).asInstanceOf[Serializer] - } else if (GlutenConfig.getConf.isUseUniffleShuffleManager) { + } else if (GlutenConfig.get.isUseUniffleShuffleManager) { throw new UnsupportedOperationException("temporarily uniffle not support ch ") } else { new CHColumnarBatchSerializer(readBatchNumRows, numOutputRows, dataSize) diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHValidatorApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHValidatorApi.scala index 29d26410b9b0..c2b52d591978 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHValidatorApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHValidatorApi.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.backendsapi.clickhouse -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.ValidatorApi +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.expression.ExpressionConverter import org.apache.gluten.extension.ValidationResult import org.apache.gluten.substrait.SubstraitContext @@ -95,7 +95,7 @@ class CHValidatorApi extends ValidatorApi with AdaptiveSparkPlanHelper with Logg } case rangePartitoning: RangePartitioning => if ( - GlutenConfig.getConf.enableColumnarSort && + GlutenConfig.get.enableColumnarSort && RangePartitionerBoundsGenerator.supportedOrderings(rangePartitoning, child) ) { None diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/extension/CommonSubexpressionEliminateRule.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/extension/CommonSubexpressionEliminateRule.scala index a944f55450d4..ffe3c1af2339 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/extension/CommonSubexpressionEliminateRule.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/extension/CommonSubexpressionEliminateRule.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.extension -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.internal.Logging import org.apache.spark.sql.SparkSession @@ -39,8 +39,8 @@ class CommonSubexpressionEliminateRule(spark: SparkSession) extends Rule[Logical override def apply(plan: LogicalPlan): LogicalPlan = { val newPlan = if ( - plan.resolved && GlutenConfig.getConf.enableGluten - && GlutenConfig.getConf.enableCommonSubexpressionEliminate && !plan.fastEquals(lastPlan) + plan.resolved && GlutenConfig.get.enableGluten + && GlutenConfig.get.enableCommonSubexpressionEliminate && !plan.fastEquals(lastPlan) ) { lastPlan = plan visitPlan(plan) diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/extension/CountDistinctWithoutExpand.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/extension/CountDistinctWithoutExpand.scala index 82051baeebc7..c396eb1677e7 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/extension/CountDistinctWithoutExpand.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/extension/CountDistinctWithoutExpand.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.extension -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, Count, CountDistinct} import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan @@ -32,9 +32,7 @@ import org.apache.spark.sql.catalyst.trees.TreePattern.AGGREGATE_EXPRESSION */ object CountDistinctWithoutExpand extends Rule[LogicalPlan] { def apply(plan: LogicalPlan): LogicalPlan = { - if ( - GlutenConfig.getConf.enableGluten && GlutenConfig.getConf.enableCountDistinctWithoutExpand - ) { + if (GlutenConfig.get.enableGluten && GlutenConfig.get.enableCountDistinctWithoutExpand) { plan.transformAllExpressionsWithPruning(_.containsPattern(AGGREGATE_EXPRESSION)) { case ae: AggregateExpression if ae.isDistinct && ae.aggregateFunction.isInstanceOf[Count] && diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/extension/ExtendedColumnPruning.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/extension/ExtendedColumnPruning.scala index f2a0a549bc7e..53b22f217f7b 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/extension/ExtendedColumnPruning.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/extension/ExtendedColumnPruning.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.extension -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.internal.Logging import org.apache.spark.sql.SparkSession @@ -36,7 +36,7 @@ object ExtendedGeneratorNestedColumnAliasing { def unapply(plan: LogicalPlan): Option[LogicalPlan] = plan match { case pj @ Project(projectList, f @ Filter(condition, g: Generate)) if canPruneGenerator(g.generator) && - GlutenConfig.getConf.enableExtendedColumnPruning && + GlutenConfig.get.enableExtendedColumnPruning && (SQLConf.get.nestedPruningOnExpressions || SQLConf.get.nestedSchemaPruningEnabled) => val attrToExtractValues = getAttributeToExtractValues(projectList ++ g.generator.children :+ condition, Seq.empty) diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/extension/FallbackBroadcastHashJoinRules.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/extension/FallbackBroadcastHashJoinRules.scala index 6a788617a6c4..b46925dfd018 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/extension/FallbackBroadcastHashJoinRules.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/extension/FallbackBroadcastHashJoinRules.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.extension -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.extension.columnar.FallbackTags import org.apache.spark.sql.SparkSession @@ -37,7 +37,7 @@ import scala.util.control.Breaks.{break, breakable} // queryStagePrepRules. case class FallbackBroadcastHashJoinPrepQueryStage(session: SparkSession) extends Rule[SparkPlan] { override def apply(plan: SparkPlan): SparkPlan = { - val glutenConf: GlutenConfig = GlutenConfig.getConf + val glutenConf: GlutenConfig = GlutenConfig.get plan.foreach { case bhj: BroadcastHashJoinExec => val buildSidePlan = bhj.buildSide match { @@ -106,8 +106,8 @@ case class FallbackBroadcastHashJoinPrepQueryStage(session: SparkSession) extend case Some(exchange @ BroadcastExchangeExec(mode, child)) => val isTransformable = if ( - !GlutenConfig.getConf.enableColumnarBroadcastExchange || - !GlutenConfig.getConf.enableColumnarBroadcastJoin + !GlutenConfig.get.enableColumnarBroadcastExchange || + !GlutenConfig.get.enableColumnarBroadcastJoin ) { ValidationResult.failed( "columnar broadcast exchange is disabled or " + @@ -146,12 +146,12 @@ case class FallbackBroadcastHashJoinPrepQueryStage(session: SparkSession) extend case class FallbackBroadcastHashJoin(session: SparkSession) extends Rule[SparkPlan] { private val enableColumnarBroadcastJoin: Boolean = - GlutenConfig.getConf.enableColumnarBroadcastJoin && - GlutenConfig.getConf.enableColumnarBroadcastExchange + GlutenConfig.get.enableColumnarBroadcastJoin && + GlutenConfig.get.enableColumnarBroadcastExchange private val enableColumnarBroadcastNestedLoopJoin: Boolean = - GlutenConfig.getConf.broadcastNestedLoopJoinTransformerTransformerEnabled && - GlutenConfig.getConf.enableColumnarBroadcastExchange + GlutenConfig.get.broadcastNestedLoopJoinTransformerTransformerEnabled && + GlutenConfig.get.enableColumnarBroadcastExchange override def apply(plan: SparkPlan): SparkPlan = { plan.foreachUp { diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/extension/RewriteDateTimestampComparisonRule.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/extension/RewriteDateTimestampComparisonRule.scala index fa8a37ffa2be..6873c06348a1 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/extension/RewriteDateTimestampComparisonRule.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/extension/RewriteDateTimestampComparisonRule.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.extension -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.internal.Logging import org.apache.spark.sql.SparkSession @@ -56,8 +56,8 @@ class RewriteDateTimestampComparisonRule(spark: SparkSession) override def apply(plan: LogicalPlan): LogicalPlan = { if ( plan.resolved && - GlutenConfig.getConf.enableGluten && - GlutenConfig.getConf.enableRewriteDateTimestampComparison + GlutenConfig.get.enableGluten && + GlutenConfig.get.enableRewriteDateTimestampComparison ) { visitPlan(plan) } else { diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/extension/RewriteToDateExpresstionRule.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/extension/RewriteToDateExpresstionRule.scala index 6e8486330465..51fd4b53bba4 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/extension/RewriteToDateExpresstionRule.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/extension/RewriteToDateExpresstionRule.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.extension -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.internal.Logging import org.apache.spark.sql.SparkSession @@ -42,8 +42,8 @@ class RewriteToDateExpresstionRule(spark: SparkSession) extends Rule[LogicalPlan override def apply(plan: LogicalPlan): LogicalPlan = { if ( plan.resolved && - GlutenConfig.getConf.enableGluten && - GlutenConfig.getConf.enableCHRewriteDateConversion + GlutenConfig.get.enableGluten && + GlutenConfig.get.enableCHRewriteDateConversion ) { visitPlan(plan) } else { diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/extension/WriteFilesWithBucketValue.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/extension/WriteFilesWithBucketValue.scala index 8ab78dcff9bc..d489c24a806b 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/extension/WriteFilesWithBucketValue.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/extension/WriteFilesWithBucketValue.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.extension -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.sql.catalyst.expressions.{Alias, BitwiseAnd, Expression, HiveHash, Literal, Pmod} import org.apache.spark.sql.catalyst.plans.physical.HashPartitioning @@ -34,8 +34,8 @@ object WriteFilesWithBucketValue extends Rule[SparkPlan] { override def apply(plan: SparkPlan): SparkPlan = { if ( - GlutenConfig.getConf.enableGluten - && GlutenConfig.getConf.enableNativeWriter.getOrElse(false) + GlutenConfig.get.enableGluten + && GlutenConfig.get.enableNativeWriter.getOrElse(false) ) { plan.transformDown { case writeFiles: WriteFilesExec if writeFiles.bucketSpec.isDefined => diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/vectorized/CHColumnarBatchSerializer.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/vectorized/CHColumnarBatchSerializer.scala index 370d93d7e7fb..a83ce4ff6507 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/vectorized/CHColumnarBatchSerializer.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/vectorized/CHColumnarBatchSerializer.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.vectorized -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.clickhouse.CHBackendSettings +import org.apache.gluten.config.GlutenConfig import org.apache.spark.SparkEnv import org.apache.spark.internal.Logging @@ -61,9 +61,9 @@ private class CHColumnarBatchSerializerInstance( GlutenShuffleUtils.getCompressionLevel( conf, compressionCodec, - GlutenConfig.getConf.columnarShuffleCodecBackend.orNull) + GlutenConfig.get.columnarShuffleCodecBackend.orNull) - private val useColumnarShuffle: Boolean = GlutenConfig.getConf.isUseColumnarShuffleManager + private val useColumnarShuffle: Boolean = GlutenConfig.get.isUseColumnarShuffleManager override def deserializeStream(in: InputStream): DeserializationStream = { // Don't use GlutenConfig in this method. It will execute in non task Thread. diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/rpc/GlutenDriverEndpoint.scala b/backends-clickhouse/src/main/scala/org/apache/spark/rpc/GlutenDriverEndpoint.scala index a061a620d209..58eb542e4141 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/rpc/GlutenDriverEndpoint.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/rpc/GlutenDriverEndpoint.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.rpc -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.SparkEnv import org.apache.spark.internal.Logging diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/shuffle/CHColumnarShuffleWriter.scala b/backends-clickhouse/src/main/scala/org/apache/spark/shuffle/CHColumnarShuffleWriter.scala index b0595fd05dd7..283f85598340 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/shuffle/CHColumnarShuffleWriter.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/shuffle/CHColumnarShuffleWriter.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.shuffle -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.clickhouse.CHBackendSettings +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.ColumnarNativeIterator import org.apache.gluten.memory.CHThreadGroup import org.apache.gluten.vectorized._ @@ -51,17 +51,17 @@ class CHColumnarShuffleWriter[K, V]( .map(_.getAbsolutePath) .mkString(",") private val subDirsPerLocalDir = blockManager.diskBlockManager.subDirsPerLocalDir - private val splitSize = GlutenConfig.getConf.maxBatchSize + private val splitSize = GlutenConfig.get.maxBatchSize private val compressionCodec = GlutenShuffleUtils.getCompressionCodec(conf) private val capitalizedCompressionCodec = compressionCodec.toUpperCase(Locale.ROOT) private val compressionLevel = GlutenShuffleUtils.getCompressionLevel( conf, compressionCodec, - GlutenConfig.getConf.columnarShuffleCodecBackend.orNull) - private val maxSortBufferSize = GlutenConfig.getConf.chColumnarMaxSortBufferSize - private val forceMemorySortShuffle = GlutenConfig.getConf.chColumnarForceMemorySortShuffle - private val spillThreshold = GlutenConfig.getConf.chColumnarShuffleSpillThreshold + GlutenConfig.get.columnarShuffleCodecBackend.orNull) + private val maxSortBufferSize = GlutenConfig.get.chColumnarMaxSortBufferSize + private val forceMemorySortShuffle = GlutenConfig.get.chColumnarForceMemorySortShuffle + private val spillThreshold = GlutenConfig.get.chColumnarShuffleSpillThreshold private val jniWrapper = new CHShuffleSplitterJniWrapper // Are we in the process of stopping? Because map tasks can call stop() with success = true // and then call stop() with success = false if they get an exception, we want to make sure diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/catalyst/CHAggregateFunctionRewriteRule.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/catalyst/CHAggregateFunctionRewriteRule.scala index b11e1e2bb306..adf34aa05674 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/catalyst/CHAggregateFunctionRewriteRule.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/catalyst/CHAggregateFunctionRewriteRule.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.catalyst -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.expressions.Cast @@ -35,8 +35,8 @@ case class CHAggregateFunctionRewriteRule(spark: SparkSession) extends Rule[Logi case a: Aggregate => a.transformExpressions { case avgExpr @ AggregateExpression(avg: Average, _, _, _, _) - if GlutenConfig.getConf.enableCastAvgAggregateFunction && - GlutenConfig.getConf.enableColumnarHashAgg && + if GlutenConfig.get.enableCastAvgAggregateFunction && + GlutenConfig.get.enableColumnarHashAgg && !avgExpr.isDistinct && isDataTypeNeedConvert(avg.child.dataType) => AggregateExpression( avg.copy(child = Cast(avg.child, DoubleType)), diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/CHParquetWriterInjects.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/CHParquetWriterInjects.scala index 1e4d0b8841c9..ae7c9b51b967 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/CHParquetWriterInjects.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/CHParquetWriterInjects.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.execution.datasources.v1 -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.sql.internal.SQLConf @@ -34,11 +34,11 @@ class CHParquetWriterInjects extends CHFormatWriterInjects { sparkOptions.put(SQLConf.PARQUET_COMPRESSION.key, compressionCodec) val blockSize = options.getOrElse( GlutenConfig.PARQUET_BLOCK_SIZE, - GlutenConfig.getConf.columnarParquetWriteBlockSize.toString) + GlutenConfig.get.columnarParquetWriteBlockSize.toString) sparkOptions.put(GlutenConfig.PARQUET_BLOCK_SIZE, blockSize) val blockRows = options.getOrElse( GlutenConfig.PARQUET_BLOCK_ROWS, - GlutenConfig.getConf.columnarParquetWriteBlockRows.toString) + GlutenConfig.get.columnarParquetWriteBlockRows.toString) sparkOptions.put(GlutenConfig.PARQUET_BLOCK_ROWS, blockRows) sparkOptions } diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/utils/CHExecUtil.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/utils/CHExecUtil.scala index 21d113248821..df70f9781757 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/utils/CHExecUtil.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/utils/CHExecUtil.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.execution.utils -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.clickhouse.CHBackendSettings +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.expression.ConverterUtils import org.apache.gluten.row.SparkRowInfo import org.apache.gluten.vectorized._ @@ -32,7 +32,7 @@ import org.apache.spark.shuffle.utils.RangePartitionerBoundsGenerator import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.{Attribute, BindReferences, BoundReference, UnsafeProjection, UnsafeRow} import org.apache.spark.sql.catalyst.expressions.codegen.LazilyGeneratedOrdering -import org.apache.spark.sql.catalyst.plans.physical.{SinglePartition, _} +import org.apache.spark.sql.catalyst.plans.physical._ import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec import org.apache.spark.sql.execution.metric.{SQLMetric, SQLShuffleWriteMetricsReporter} @@ -239,7 +239,7 @@ object CHExecUtil extends Logging { private def buildPartitioningOptions(nativePartitioning: NativePartitioning): IteratorOptions = { val options = new IteratorOptions - options.setBufferSize(GlutenConfig.getConf.maxBatchSize) + options.setBufferSize(GlutenConfig.get.maxBatchSize) options.setName(nativePartitioning.getShortName) options.setPartitionNum(nativePartitioning.getNumPartitions) options.setExpr(new String(nativePartitioning.getExprList)) @@ -345,8 +345,8 @@ object CHExecUtil extends Logging { val rddWithPartitionKey: RDD[Product2[Int, ColumnarBatch]] = if ( - GlutenConfig.getConf.isUseColumnarShuffleManager - || GlutenConfig.getConf.isUseCelebornShuffleManager + GlutenConfig.get.isUseColumnarShuffleManager + || GlutenConfig.get.isUseCelebornShuffleManager ) { newPartitioning match { case _ => diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/RunTPCHTest.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/RunTPCHTest.scala index 46f93f15014e..6b8aee8b3520 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/RunTPCHTest.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/RunTPCHTest.scala @@ -18,6 +18,7 @@ package org.apache.gluten import org.apache.gluten.backendsapi.clickhouse.RuntimeConfig import org.apache.gluten.benchmarks.GenTPCHTableScripts +import org.apache.gluten.config.GlutenConfig import org.apache.spark.sql.SparkSession import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/affinity/CHUTAffinity.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/affinity/CHUTAffinity.scala index ee8ec686fab2..caa6a8d55eaf 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/affinity/CHUTAffinity.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/affinity/CHUTAffinity.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.affinity -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.clickhouse.CHBackendSettings +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.softaffinity.AffinityManager import org.apache.spark.affinity.MixedAffinity diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDeltaParquetWriteSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDeltaParquetWriteSuite.scala index 866c41c510fe..86a81aa0edbd 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDeltaParquetWriteSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseDeltaParquetWriteSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.SparkConf import org.apache.spark.gluten.delta.DeltaStatsUtils diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseExcelFormatSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseExcelFormatSuite.scala index 7c444d572b69..76afd602cefa 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseExcelFormatSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseExcelFormatSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.clickhouse.CHConf +import org.apache.gluten.config.GlutenConfig import org.apache.spark.SparkConf import org.apache.spark.sql.{functions, DataFrame, Row} diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseJoinSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseJoinSuite.scala index 7a9abd3bad6f..9a3e14270d87 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseJoinSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseJoinSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.clickhouse.CHConf +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.utils.UTSystemParameters import org.apache.spark.SparkConf diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeExceptionSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeExceptionSuite.scala index cac1a8c5b346..3db268403719 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeExceptionSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeExceptionSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.utils.{TestExceptionUtils, UTSystemParameters} import org.apache.spark.SparkConf diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeLibSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeLibSuite.scala index 0221f06bd681..77af49ca999d 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeLibSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeLibSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.exception.GlutenException import org.apache.gluten.utils.UTSystemParameters diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseSyntheticDataSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseSyntheticDataSuite.scala index bd739a291e51..ac874bf88a01 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseSyntheticDataSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseSyntheticDataSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.utils.UTSystemParameters import org.apache.spark.SparkConf diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSAbstractSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSAbstractSuite.scala index f5b97e6b5ef1..70ce5e6eb49a 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSAbstractSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSAbstractSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig import org.apache.gluten.benchmarks.GenTPCDSTableScripts +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.utils.{Arm, UTSystemParameters} import org.apache.spark.SparkConf diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHAbstractSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHAbstractSuite.scala index fbaebdecaaf6..08f743bb918a 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHAbstractSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHAbstractSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.utils.UTSystemParameters import org.apache.spark.SparkConf diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHNullableSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHNullableSuite.scala index d5e1156ba9d2..b881fc4a1220 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHNullableSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHNullableSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.SparkConf import org.apache.spark.sql.catalyst.expressions.Alias diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseWholeStageTransformerSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseWholeStageTransformerSuite.scala index d7b277edfac5..cef8b9c776d8 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseWholeStageTransformerSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseWholeStageTransformerSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.clickhouse.RuntimeConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.utils.UTSystemParameters import org.apache.spark.{SPARK_VERSION_SHORT, SparkConf} diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala index 39b5421f5d68..e9df41ace6e7 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.utils.UTSystemParameters import org.apache.spark.SparkConf diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/compatibility/GlutenClickhouseFunctionSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/compatibility/GlutenClickhouseFunctionSuite.scala index 8f2658ef8f24..a7c828453be6 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/compatibility/GlutenClickhouseFunctionSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/compatibility/GlutenClickhouseFunctionSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.execution.compatibility -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{GlutenClickHouseTPCHAbstractSuite, ProjectExecTransformer} import org.apache.gluten.utils.UTSystemParameters diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseHiveTableSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseHiveTableSuite.scala index 14d6ff53649f..0fee52108b1c 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseHiveTableSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseHiveTableSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.execution.hive -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{FileSourceScanExecTransformer, GlutenClickHouseWholeStageTransformerSuite, ProjectExecTransformer, TransformSupport} import org.apache.gluten.test.AllDataTypesWithComplexType import org.apache.gluten.utils.UTSystemParameters diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseNativeWriteTableSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseNativeWriteTableSuite.scala index d77779f95e33..4de1168ce31c 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseNativeWriteTableSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseNativeWriteTableSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.execution.hive -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.clickhouse.RuntimeConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.GlutenClickHouseWholeStageTransformerSuite import org.apache.gluten.test.AllDataTypesWithComplexType.genTestData import org.apache.gluten.utils.UTSystemParameters diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseTableAfterRestart.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseTableAfterRestart.scala index 59bdb4738303..9379ef835d30 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseTableAfterRestart.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseTableAfterRestart.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.execution.hive -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.clickhouse.{CHConf, RuntimeConfig, RuntimeSettings} +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.GlutenClickHouseTPCHAbstractSuite import org.apache.spark.SparkConf diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeCacheDataSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeCacheDataSuite.scala index 8722c676cf31..68cc6ac1c13e 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeCacheDataSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeCacheDataSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.execution.mergetree -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.clickhouse.{CHConf, RuntimeConfig} +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{FileSourceScanExecTransformer, GlutenClickHouseTPCHAbstractSuite} import org.apache.spark.SparkConf diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeOptimizeSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeOptimizeSuite.scala index 42ff6d91360f..23ee77675a4d 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeOptimizeSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeOptimizeSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.execution.mergetree -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.clickhouse.{CHConf, RuntimeConfig, RuntimeSettings} +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{FileSourceScanExecTransformer, GlutenClickHouseTPCHAbstractSuite} import org.apache.spark.SparkConf diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreePathBasedWriteSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreePathBasedWriteSuite.scala index cec4bdc105b2..4ef77c24e312 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreePathBasedWriteSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreePathBasedWriteSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.execution.mergetree -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.clickhouse.{CHConf, RuntimeSettings} +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution._ import org.apache.gluten.utils.Arm diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteOnHDFSSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteOnHDFSSuite.scala index 413448730042..f32a72083253 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteOnHDFSSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteOnHDFSSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.execution.mergetree -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.clickhouse.{CHConf, RuntimeConfig, RuntimeSettings} +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{FileSourceScanExecTransformer, GlutenClickHouseTPCHAbstractSuite} import org.apache.spark.SparkConf diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteOnHDFSWithRocksDBMetaSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteOnHDFSWithRocksDBMetaSuite.scala index d689ad7ba1ca..c7506fe198d7 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteOnHDFSWithRocksDBMetaSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteOnHDFSWithRocksDBMetaSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.execution.mergetree -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.clickhouse.{CHConf, RuntimeConfig, RuntimeSettings} +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{FileSourceScanExecTransformer, GlutenClickHouseTPCHAbstractSuite} import org.apache.spark.SparkConf diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteOnS3Suite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteOnS3Suite.scala index edd41b6bea08..3715bae9ec2d 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteOnS3Suite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteOnS3Suite.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.execution.mergetree -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.clickhouse.{CHConf, RuntimeConfig} +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{BasicScanExecTransformer, FileSourceScanExecTransformer, GlutenClickHouseTPCHAbstractSuite} import org.apache.spark.SparkConf diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteSuite.scala index 5f1d531e0dcf..286d739b7027 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.execution.mergetree -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.clickhouse.{CHConf, RuntimeSettings} +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution._ import org.apache.gluten.utils.Arm diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteTaskNotSerializableSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteTaskNotSerializableSuite.scala index b1885874f5ae..22a0541d99a1 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteTaskNotSerializableSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteTaskNotSerializableSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.execution.mergetree -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.clickhouse.CHConf +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.GlutenClickHouseTPCHAbstractSuite import org.apache.spark.SparkConf diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/tpch/GlutenClickHouseTPCHColumnarShuffleParquetAQESuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/tpch/GlutenClickHouseTPCHColumnarShuffleParquetAQESuite.scala index b4186fee66aa..4055b93edd99 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/tpch/GlutenClickHouseTPCHColumnarShuffleParquetAQESuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/tpch/GlutenClickHouseTPCHColumnarShuffleParquetAQESuite.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.execution.tpch -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution._ import org.apache.gluten.utils.Arm diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/tpch/GlutenClickHouseTPCHSaltNullParquetSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/tpch/GlutenClickHouseTPCHSaltNullParquetSuite.scala index 6feb1e880622..5c54dbc304ba 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/tpch/GlutenClickHouseTPCHSaltNullParquetSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/tpch/GlutenClickHouseTPCHSaltNullParquetSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.execution.tpch -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.clickhouse.CHConf +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution._ import org.apache.gluten.execution.GlutenPlan diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/s3/S3AuthSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/s3/S3AuthSuite.scala index 32c64c78a69a..b286551bba01 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/s3/S3AuthSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/s3/S3AuthSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.s3 -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.sql.SparkSession import org.apache.spark.sql.SparkSession.Builder diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/test/GlutenSQLTestUtils.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/test/GlutenSQLTestUtils.scala index 9888baf9aac1..d2be942459c5 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/test/GlutenSQLTestUtils.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/test/GlutenSQLTestUtils.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.test -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.SparkFunSuite import org.apache.spark.sql.delta.DeltaLog diff --git a/backends-clickhouse/src/test/scala/org/apache/spark/gluten/NativeWriteChecker.scala b/backends-clickhouse/src/test/scala/org/apache/spark/gluten/NativeWriteChecker.scala index 631461b6a2f4..384780e7d296 100644 --- a/backends-clickhouse/src/test/scala/org/apache/spark/gluten/NativeWriteChecker.scala +++ b/backends-clickhouse/src/test/scala/org/apache/spark/gluten/NativeWriteChecker.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.gluten -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.GlutenClickHouseWholeStageTransformerSuite import org.apache.spark.sql.{Dataset, Row} diff --git a/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHAggAndShuffleBenchmark.scala b/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHAggAndShuffleBenchmark.scala index 358b785187a0..e914fb74a935 100644 --- a/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHAggAndShuffleBenchmark.scala +++ b/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHAggAndShuffleBenchmark.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.execution.benchmarks -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{FileSourceScanExecTransformer, ProjectExecTransformer, WholeStageTransformer} import org.apache.gluten.sql.shims.SparkShimLoader diff --git a/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHSqlBasedBenchmark.scala b/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHSqlBasedBenchmark.scala index 0467ae2814fa..297dffabbf74 100644 --- a/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHSqlBasedBenchmark.scala +++ b/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHSqlBasedBenchmark.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.execution.benchmarks -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.jni.JniLibLoader import org.apache.gluten.utils.UTSystemParameters diff --git a/backends-velox/src-celeborn/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala b/backends-velox/src-celeborn/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala index a4a97d43de08..c2792ab80418 100644 --- a/backends-velox/src-celeborn/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala +++ b/backends-velox/src-celeborn/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala @@ -16,9 +16,9 @@ */ package org.apache.spark.shuffle -import org.apache.gluten.GlutenConfig -import org.apache.gluten.GlutenConfig.{GLUTEN_RSS_SORT_SHUFFLE_WRITER, GLUTEN_SORT_SHUFFLE_WRITER} import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.config.GlutenConfig +import org.apache.gluten.config.GlutenConfig.{GLUTEN_RSS_SORT_SHUFFLE_WRITER, GLUTEN_SORT_SHUFFLE_WRITER} import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators import org.apache.gluten.runtime.Runtimes import org.apache.gluten.utils.ArrowAbiUtil @@ -85,12 +85,12 @@ private class CelebornColumnarBatchSerializerInstance( null // uncompressed } val compressionCodecBackend = - GlutenConfig.getConf.columnarShuffleCodecBackend.orNull - val shuffleWriterType = GlutenConfig.getConf.celebornShuffleWriterType + GlutenConfig.get.columnarShuffleCodecBackend.orNull + val shuffleWriterType = GlutenConfig.get.celebornShuffleWriterType .replace(GLUTEN_SORT_SHUFFLE_WRITER, GLUTEN_RSS_SORT_SHUFFLE_WRITER) val jniWrapper = ShuffleReaderJniWrapper.create(runtime) - val batchSize = GlutenConfig.getConf.maxBatchSize - val bufferSize = GlutenConfig.getConf.columnarShuffleReaderBufferSize + val batchSize = GlutenConfig.get.maxBatchSize + val bufferSize = GlutenConfig.get.columnarShuffleReaderBufferSize val handle = jniWrapper .make( cSchema.memoryAddress(), diff --git a/backends-velox/src-celeborn/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarShuffleWriter.scala b/backends-velox/src-celeborn/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarShuffleWriter.scala index 165d68785d4b..f20c3e679416 100644 --- a/backends-velox/src-celeborn/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarShuffleWriter.scala +++ b/backends-velox/src-celeborn/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarShuffleWriter.scala @@ -16,9 +16,9 @@ */ package org.apache.spark.shuffle -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.columnarbatch.ColumnarBatches +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.memory.memtarget.{MemoryTarget, Spiller, Spillers} import org.apache.gluten.runtime.Runtimes import org.apache.gluten.vectorized._ @@ -127,7 +127,7 @@ class VeloxCelebornColumnarShuffleWriter[K, V]( compressionLevel, compressionBufferSize, bufferCompressThreshold, - GlutenConfig.getConf.columnarShuffleCompressionMode, + GlutenConfig.get.columnarShuffleCompressionMode, conf.get(SHUFFLE_SORT_INIT_BUFFER_SIZE).toInt, conf.get(SHUFFLE_SORT_USE_RADIXSORT), clientPushBufferMaxSize, @@ -138,7 +138,7 @@ class VeloxCelebornColumnarShuffleWriter[K, V]( GlutenShuffleUtils.getStartPartitionId(dep.nativePartitioning, context.partitionId), "celeborn", shuffleWriterType, - GlutenConfig.getConf.columnarShuffleReallocThreshold + GlutenConfig.get.columnarShuffleReallocThreshold ) runtime .memoryManager() diff --git a/backends-velox/src-uniffle/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java b/backends-velox/src-uniffle/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java index 7cbf452f76f6..dafc9d5adcd0 100644 --- a/backends-velox/src-uniffle/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java +++ b/backends-velox/src-uniffle/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java @@ -16,9 +16,10 @@ */ package org.apache.spark.shuffle.writer; -import org.apache.gluten.GlutenConfig; import org.apache.gluten.backendsapi.BackendsApiManager; import org.apache.gluten.columnarbatch.ColumnarBatches; +import org.apache.gluten.config.GlutenConfig; +import org.apache.gluten.config.GlutenConfig$; import org.apache.gluten.memory.memtarget.MemoryTarget; import org.apache.gluten.memory.memtarget.Spiller; import org.apache.gluten.memory.memtarget.Spillers; @@ -64,8 +65,9 @@ public class VeloxUniffleColumnarShuffleWriter extends RssShuffleWriter extends RssShuffleWriter null)); + GlutenConfig$.MODULE$.get().columnarShuffleCodecBackend().getOrElse(() -> null)); compressionBufferSize = GlutenShuffleUtils.getSortEvictBufferSize(sparkConf, compressionCodec); } @@ -158,7 +160,7 @@ protected void writeImpl(Iterator> records) { compressionLevel, compressionBufferSize, compressThreshold, - GlutenConfig.getConf().columnarShuffleCompressionMode(), + GlutenConfig$.MODULE$.get().columnarShuffleCompressionMode(), (int) (long) sparkConf.get(package$.MODULE$.SHUFFLE_SORT_INIT_BUFFER_SIZE()), (boolean) sparkConf.get(package$.MODULE$.SHUFFLE_SORT_USE_RADIXSORT()), bufferSize, diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala index 4d66c26913a3..4a19f4abcca2 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala @@ -17,10 +17,10 @@ package org.apache.gluten.backendsapi.velox import org.apache.gluten.GlutenBuildInfo._ -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi._ import org.apache.gluten.columnarbatch.VeloxBatch import org.apache.gluten.component.Component.BuildInfo +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.exception.GlutenNotSupportException import org.apache.gluten.execution.WriteFilesExecTransformer import org.apache.gluten.expression.WindowFunctionsBuilder @@ -107,9 +107,7 @@ object VeloxBackendSettings extends BackendSettingsApi { val filteredRootPaths = distinctRootPaths(rootPaths) if (filteredRootPaths.nonEmpty) { val resolvedPaths = - if ( - GlutenConfig.getConf.enableHdfsViewfs && filteredRootPaths.head.startsWith("viewfs") - ) { + if (GlutenConfig.get.enableHdfsViewfs && filteredRootPaths.head.startsWith("viewfs")) { // Convert the viewfs path to hdfs path. filteredRootPaths.map { viewfsPath => @@ -149,7 +147,7 @@ object VeloxBackendSettings extends BackendSettingsApi { def isCharType(stringType: StringType, metadata: Metadata): Boolean = { val charTypePattern = "char\\((\\d+)\\)".r - GlutenConfig.getConf.forceOrcCharTypeScanFallbackEnabled && charTypePattern + GlutenConfig.get.forceOrcCharTypeScanFallbackEnabled && charTypePattern .findFirstIn( CharVarcharUtils .getRawTypeString(metadata) @@ -162,7 +160,7 @@ object VeloxBackendSettings extends BackendSettingsApi { val typeValidator: PartialFunction[StructField, String] = { // Parquet timestamp is not fully supported yet case StructField(_, TimestampType, _, _) - if GlutenConfig.getConf.forceParquetTimestampTypeScanFallbackEnabled => + if GlutenConfig.get.forceParquetTimestampTypeScanFallbackEnabled => "TimestampType(force fallback)" } val parquetOptions = new ParquetOptions(CaseInsensitiveMap(properties), SQLConf.get) @@ -174,7 +172,7 @@ object VeloxBackendSettings extends BackendSettingsApi { } case DwrfReadFormat => None case OrcReadFormat => - if (!GlutenConfig.getConf.veloxOrcScanEnabled) { + if (!GlutenConfig.get.veloxOrcScanEnabled) { Some(s"Velox ORC scan is turned off, ${GlutenConfig.VELOX_ORC_SCAN_ENABLED.key}") } else { val typeValidator: PartialFunction[StructField, String] = { @@ -319,7 +317,7 @@ object VeloxBackendSettings extends BackendSettingsApi { def validateFileFormat(): Option[String] = { format match { case _: ParquetFileFormat => None // Parquet is directly supported - case h: HiveFileFormat if GlutenConfig.getConf.enableHiveFileFormatWriter => + case h: HiveFileFormat if GlutenConfig.get.enableHiveFileFormatWriter => validateHiveFileFormat(h) // Parquet via Hive SerDe case _ => Some( @@ -379,7 +377,7 @@ object VeloxBackendSettings extends BackendSettingsApi { override def supportSortExec(): Boolean = true override def supportSortMergeJoinExec(): Boolean = { - GlutenConfig.getConf.enableColumnarSortMergeJoin + GlutenConfig.get.enableColumnarSortMergeJoin } override def supportWindowGroupLimitExec(rankLikeFunction: Expression): Boolean = { @@ -460,9 +458,9 @@ object VeloxBackendSettings extends BackendSettingsApi { } override def supportColumnarShuffleExec(): Boolean = { - GlutenConfig.getConf.enableColumnarShuffle && (GlutenConfig.getConf.isUseColumnarShuffleManager - || GlutenConfig.getConf.isUseCelebornShuffleManager - || GlutenConfig.getConf.isUseUniffleShuffleManager) + GlutenConfig.get.enableColumnarShuffle && (GlutenConfig.get.isUseColumnarShuffleManager + || GlutenConfig.get.isUseCelebornShuffleManager + || GlutenConfig.get.isUseUniffleShuffleManager) } override def enableJoinKeysRewrite(): Boolean = false @@ -515,7 +513,7 @@ object VeloxBackendSettings extends BackendSettingsApi { override def alwaysFailOnMapExpression(): Boolean = true override def requiredChildOrderingForWindow(): Boolean = { - GlutenConfig.getConf.veloxColumnarWindowType.equals("streaming") + GlutenConfig.get.veloxColumnarWindowType.equals("streaming") } override def requiredChildOrderingForWindowGroupLimit(): Boolean = false @@ -525,13 +523,13 @@ object VeloxBackendSettings extends BackendSettingsApi { override def allowDecimalArithmetic: Boolean = true override def enableNativeWriteFiles(): Boolean = { - GlutenConfig.getConf.enableNativeWriter.getOrElse( + GlutenConfig.get.enableNativeWriter.getOrElse( SparkShimLoader.getSparkShims.enableNativeWriteFilesByDefault() ) } override def enableNativeArrowReadFiles(): Boolean = { - GlutenConfig.getConf.enableNativeArrowReader + GlutenConfig.get.enableNativeArrowReader } override def shouldRewriteCount(): Boolean = { diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala index 26bc108c15d9..c06a8cfc9880 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.backendsapi.velox -import org.apache.gluten.GlutenNumaBindingInfo import org.apache.gluten.backendsapi.{BackendsApiManager, IteratorApi} +import org.apache.gluten.config.GlutenNumaBindingInfo import org.apache.gluten.execution._ import org.apache.gluten.iterator.Iterators import org.apache.gluten.metrics.{IMetrics, IteratorMetricsJniWrapper} diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala index 175e34177a5d..15d1afe1bd82 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala @@ -16,10 +16,10 @@ */ package org.apache.gluten.backendsapi.velox -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.ListenerApi import org.apache.gluten.columnarbatch.ArrowBatches.{ArrowJavaBatch, ArrowNativeBatch} import org.apache.gluten.columnarbatch.VeloxBatch +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.datasource.GlutenFormatFactory import org.apache.gluten.expression.UDFMappings import org.apache.gluten.init.NativeBackendInitializer diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxRuleApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxRuleApi.scala index 460547bc7154..72d769c999e8 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxRuleApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxRuleApi.scala @@ -16,9 +16,9 @@ */ package org.apache.gluten.backendsapi.velox -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.RuleApi import org.apache.gluten.columnarbatch.VeloxBatch +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.extension._ import org.apache.gluten.extension.columnar._ import org.apache.gluten.extension.columnar.MiscColumnarRules.{RemoveGlutenTableCacheColumnarToRow, RemoveTopmostColumnarToRow, RewriteSubqueryBroadcast} diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index 5975a20a267d..ba3293715177 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.backendsapi.velox -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.SparkPlanExecApi +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.exception.GlutenNotSupportException import org.apache.gluten.execution._ import org.apache.gluten.expression._ @@ -345,8 +345,8 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { plan match { case shuffle: ColumnarShuffleExchangeExec if !shuffle.useSortBasedShuffle && - GlutenConfig.getConf.veloxResizeBatchesShuffleInput => - val range = GlutenConfig.getConf.veloxResizeBatchesShuffleInputRange + GlutenConfig.get.veloxResizeBatchesShuffleInput => + val range = GlutenConfig.get.veloxResizeBatchesShuffleInputRange val appendBatches = VeloxResizeBatchesExec(shuffle.child, range.min, range.max) shuffle.withNewChildren(Seq(appendBatches)) @@ -541,12 +541,12 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { /** Determine whether to use sort-based shuffle based on shuffle partitioning and output. */ override def useSortBasedShuffle(partitioning: Partitioning, output: Seq[Attribute]): Boolean = { - val conf = GlutenConfig.getConf + val conf = GlutenConfig.get lazy val isCelebornSortBasedShuffle = conf.isUseCelebornShuffleManager && conf.celebornShuffleWriterType == GlutenConfig.GLUTEN_SORT_SHUFFLE_WRITER partitioning != SinglePartition && - (partitioning.numPartitions >= GlutenConfig.getConf.columnarShuffleSortPartitionsThreshold || - output.size >= GlutenConfig.getConf.columnarShuffleSortColumnsThreshold) || + (partitioning.numPartitions >= GlutenConfig.get.columnarShuffleSortPartitionsThreshold || + output.size >= GlutenConfig.get.columnarShuffleSortColumnsThreshold) || isCelebornSortBasedShuffle } @@ -599,7 +599,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { val deserializeTime = metrics("deserializeTime") val readBatchNumRows = metrics("avgReadBatchNumRows") val decompressTime = metrics("decompressTime") - if (GlutenConfig.getConf.isUseCelebornShuffleManager) { + if (GlutenConfig.get.isUseCelebornShuffleManager) { val clazz = ClassUtils.getClass("org.apache.spark.shuffle.CelebornColumnarBatchSerializer") val constructor = clazz.getConstructor(classOf[StructType], classOf[SQLMetric], classOf[SQLMetric]) @@ -728,7 +728,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { // ISOControl characters, refer java.lang.Character.isISOControl(int) val isoControlStr = (('\u0000' to '\u001F') ++ ('\u007F' to '\u009F')).toList.mkString // scalastyle:on nonascii - if (GlutenConfig.getConf.castFromVarcharAddTrimNode && c.child.dataType == StringType) { + if (GlutenConfig.get.castFromVarcharAddTrimNode && c.child.dataType == StringType) { val trimStr = c.dataType match { case BinaryType | _: ArrayType | _: MapType | _: StructType | _: UserDefinedType[_] => None @@ -770,7 +770,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { } override def rewriteSpillPath(path: String): String = { - val fs = GlutenConfig.getConf.veloxSpillFileSystem + val fs = GlutenConfig.get.veloxSpillFileSystem fs match { case "local" => path diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/ColumnarPartialProjectExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/ColumnarPartialProjectExec.scala index 576f3a2cb205..056d19e55070 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/ColumnarPartialProjectExec.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/ColumnarPartialProjectExec.scala @@ -16,9 +16,9 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.columnarbatch.{ColumnarBatches, VeloxColumnarBatches} +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.expression.{ArrowProjection, ExpressionUtils} import org.apache.gluten.extension.ValidationResult import org.apache.gluten.extension.columnar.transition.Convention @@ -138,7 +138,7 @@ case class ColumnarPartialProjectExec(original: ProjectExec, child: SparkPlan)( } override protected def doValidateInternal(): ValidationResult = { - if (!GlutenConfig.getConf.enableColumnarPartialProject) { + if (!GlutenConfig.get.enableColumnarPartialProject) { return ValidationResult.failed("Config disable this feature") } if (UDFAttrNotExists) { @@ -164,9 +164,7 @@ case class ColumnarPartialProjectExec(original: ProjectExec, child: SparkPlan)( return ValidationResult.failed("Contains expression not supported") } if ( - ExpressionUtils.hasComplexExpressions( - original, - GlutenConfig.getConf.fallbackExpressionsThreshold) + ExpressionUtils.hasComplexExpressions(original, GlutenConfig.get.fallbackExpressionsThreshold) ) { return ValidationResult.failed("Fallback by complex expression") } diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala index 9cdcf854db8b..ab67448c3dee 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala @@ -16,9 +16,9 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.columnarbatch.ColumnarBatches +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.iterator.Iterators import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators import org.apache.gluten.runtime.Runtimes @@ -48,7 +48,7 @@ case class RowToVeloxColumnarExec(child: SparkPlan) extends RowToColumnarExecBas val numInputRows = longMetric("numInputRows") val numOutputBatches = longMetric("numOutputBatches") val convertTime = longMetric("convertTime") - val numRows = GlutenConfig.getConf.maxBatchSize + val numRows = GlutenConfig.get.maxBatchSize // This avoids calling `schema` in the RDD closure, so that we don't need to include the entire // plan (this) in the closure. val localSchema = schema @@ -68,7 +68,7 @@ case class RowToVeloxColumnarExec(child: SparkPlan) extends RowToColumnarExecBas val numInputRows = longMetric("numInputRows") val numOutputBatches = longMetric("numOutputBatches") val convertTime = longMetric("convertTime") - val numRows = GlutenConfig.getConf.maxBatchSize + val numRows = GlutenConfig.get.maxBatchSize val mode = BroadcastUtils.getBroadcastMode(outputPartitioning) val relation = child.executeBroadcast() BroadcastUtils.sparkToVeloxUnsafe( diff --git a/backends-velox/src/main/scala/org/apache/gluten/extension/BloomFilterMightContainJointRewriteRule.scala b/backends-velox/src/main/scala/org/apache/gluten/extension/BloomFilterMightContainJointRewriteRule.scala index 56a3d86a9038..7b209ad605e1 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/extension/BloomFilterMightContainJointRewriteRule.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/extension/BloomFilterMightContainJointRewriteRule.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.extension -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.expression.VeloxBloomFilterMightContain import org.apache.gluten.expression.aggregate.VeloxBloomFilterAggregate import org.apache.gluten.sql.shims.SparkShimLoader @@ -27,7 +27,7 @@ import org.apache.spark.sql.execution.SparkPlan case class BloomFilterMightContainJointRewriteRule(spark: SparkSession) extends Rule[SparkPlan] { override def apply(plan: SparkPlan): SparkPlan = { - if (!GlutenConfig.getConf.enableNativeBloomFilter) { + if (!GlutenConfig.get.enableNativeBloomFilter) { return plan } val out = plan.transformWithSubqueries { diff --git a/backends-velox/src/main/scala/org/apache/gluten/extension/FlushableHashAggregateRule.scala b/backends-velox/src/main/scala/org/apache/gluten/extension/FlushableHashAggregateRule.scala index 2e5390697795..5346b110f678 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/extension/FlushableHashAggregateRule.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/extension/FlushableHashAggregateRule.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.extension -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution._ import org.apache.spark.sql.SparkSession @@ -34,7 +34,7 @@ import org.apache.spark.sql.execution.exchange.ShuffleExchangeLike case class FlushableHashAggregateRule(session: SparkSession) extends Rule[SparkPlan] { import FlushableHashAggregateRule._ override def apply(plan: SparkPlan): SparkPlan = { - if (!GlutenConfig.getConf.enableVeloxFlushablePartialAggregation) { + if (!GlutenConfig.get.enableVeloxFlushablePartialAggregation) { return plan } plan.transformUpWithPruning(_.containsPattern(EXCHANGE)) { diff --git a/backends-velox/src/main/scala/org/apache/gluten/extension/HLLRewriteRule.scala b/backends-velox/src/main/scala/org/apache/gluten/extension/HLLRewriteRule.scala index 8ceee3d573b9..e15fb053308f 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/extension/HLLRewriteRule.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/extension/HLLRewriteRule.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.extension -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.expression.aggregate.HLLAdapter import org.apache.spark.sql.SparkSession @@ -33,8 +33,8 @@ case class HLLRewriteRule(spark: SparkSession) extends Rule[LogicalPlan] { case a: Aggregate => a.transformExpressionsWithPruning(_.containsPattern(AGGREGATE_EXPRESSION)) { case aggExpr @ AggregateExpression(hll: HyperLogLogPlusPlus, _, _, _, _) - if GlutenConfig.getConf.enableNativeHyperLogLogAggregateFunction && - GlutenConfig.getConf.enableColumnarHashAgg && + if GlutenConfig.get.enableNativeHyperLogLogAggregateFunction && + GlutenConfig.get.enableColumnarHashAgg && isSupportedDataType(hll.child.dataType) => val hllAdapter = HLLAdapter( hll.child, diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoader.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoader.scala index 28a40f4b3d8a..1fc0cc913ea6 100755 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoader.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoader.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.utils -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.exception.GlutenException import org.apache.gluten.jni.JniLibLoader diff --git a/backends-velox/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala b/backends-velox/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala index cd035e3202d2..3af742f339cf 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.vectorized -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.iterator.ClosableIterator import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators import org.apache.gluten.runtime.Runtimes @@ -97,9 +97,9 @@ private class ColumnarBatchSerializerInstance( null // uncompressed } val compressionCodecBackend = - GlutenConfig.getConf.columnarShuffleCodecBackend.orNull - val batchSize = GlutenConfig.getConf.maxBatchSize - val bufferSize = GlutenConfig.getConf.columnarShuffleReaderBufferSize + GlutenConfig.get.columnarShuffleCodecBackend.orNull + val batchSize = GlutenConfig.get.maxBatchSize + val bufferSize = GlutenConfig.get.columnarShuffleReaderBufferSize val runtime = Runtimes.contextInstance(BackendsApiManager.getBackendName, "ShuffleReader") val jniWrapper = ShuffleReaderJniWrapper.create(runtime) val shuffleReaderHandle = jniWrapper.make( diff --git a/backends-velox/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala b/backends-velox/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala index bb84e3066b5c..8581bc5c7cd0 100644 --- a/backends-velox/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala +++ b/backends-velox/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala @@ -16,9 +16,9 @@ */ package org.apache.spark.shuffle -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.columnarbatch.ColumnarBatches +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.memory.memtarget.{MemoryTarget, Spiller, Spillers} import org.apache.gluten.runtime.Runtimes import org.apache.gluten.vectorized._ @@ -63,8 +63,8 @@ class ColumnarShuffleWriter[K, V]( .mkString(",") private lazy val nativeBufferSize = { - val bufferSize = GlutenConfig.getConf.shuffleWriterBufferSize - val maxBatchSize = GlutenConfig.getConf.maxBatchSize + val bufferSize = GlutenConfig.get.shuffleWriterBufferSize + val maxBatchSize = GlutenConfig.get.maxBatchSize if (bufferSize > maxBatchSize) { logInfo( s"${GlutenConfig.SHUFFLE_WRITER_BUFFER_SIZE.key} ($bufferSize) exceeds max " + @@ -75,9 +75,9 @@ class ColumnarShuffleWriter[K, V]( } } - private val nativeMergeBufferSize = GlutenConfig.getConf.maxBatchSize + private val nativeMergeBufferSize = GlutenConfig.get.maxBatchSize - private val nativeMergeThreshold = GlutenConfig.getConf.columnarShuffleMergeThreshold + private val nativeMergeThreshold = GlutenConfig.get.columnarShuffleMergeThreshold private val compressionCodec = if (conf.getBoolean(SHUFFLE_COMPRESS.key, SHUFFLE_COMPRESS.defaultValue.get)) { @@ -87,7 +87,7 @@ class ColumnarShuffleWriter[K, V]( } private val compressionCodecBackend = - GlutenConfig.getConf.columnarShuffleCodecBackend.orNull + GlutenConfig.get.columnarShuffleCodecBackend.orNull private val compressionLevel = GlutenShuffleUtils.getCompressionLevel(conf, compressionCodec, compressionCodecBackend) @@ -96,9 +96,9 @@ class ColumnarShuffleWriter[K, V]( GlutenShuffleUtils.getSortEvictBufferSize(conf, compressionCodec) private val bufferCompressThreshold = - GlutenConfig.getConf.columnarShuffleCompressionThreshold + GlutenConfig.get.columnarShuffleCompressionThreshold - private val reallocThreshold = GlutenConfig.getConf.columnarShuffleReallocThreshold + private val reallocThreshold = GlutenConfig.get.columnarShuffleReallocThreshold private val runtime = Runtimes.contextInstance(BackendsApiManager.getBackendName, "ShuffleWriter") @@ -149,7 +149,7 @@ class ColumnarShuffleWriter[K, V]( compressionLevel, sortEvictBufferSize, bufferCompressThreshold, - GlutenConfig.getConf.columnarShuffleCompressionMode, + GlutenConfig.get.columnarShuffleCompressionMode, conf.get(SHUFFLE_SORT_INIT_BUFFER_SIZE).toInt, conf.get(SHUFFLE_SORT_USE_RADIXSORT), dataTmp.getAbsolutePath, diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala b/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala index 64ad105c7f7c..20ca55203e10 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala @@ -16,9 +16,9 @@ */ package org.apache.spark.sql.execution -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.columnarbatch.ColumnarBatches +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{RowToVeloxColumnarExec, VeloxColumnarToRowExec} import org.apache.gluten.iterator.Iterators import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators @@ -108,14 +108,15 @@ class ColumnarCachedBatchSerializer extends CachedBatchSerializer with SQLConfHe // `convertColumnarBatchToCachedBatch`, but the inside ColumnarBatch is not arrow-based. // See: `InMemoryRelation.apply()`. // So we should disallow columnar input if using vanilla Spark columnar scan. - val noVanillaSparkColumnarScan = conf.getConf(GlutenConfig.COLUMNAR_FILESCAN_ENABLED) || - !conf.getConf(GlutenConfig.VANILLA_VECTORIZED_READERS_ENABLED) - conf.getConf(GlutenConfig.GLUTEN_ENABLED) && validateSchema( + val noVanillaSparkColumnarScan = + GlutenConfig.get.getConf(GlutenConfig.COLUMNAR_FILESCAN_ENABLED) || + !GlutenConfig.get.getConf(GlutenConfig.VANILLA_VECTORIZED_READERS_ENABLED) + GlutenConfig.get.getConf(GlutenConfig.GLUTEN_ENABLED) && validateSchema( schema) && noVanillaSparkColumnarScan } override def supportsColumnarOutput(schema: StructType): Boolean = { - conf.getConf(GlutenConfig.GLUTEN_ENABLED) && validateSchema(schema) + GlutenConfig.get.getConf(GlutenConfig.GLUTEN_ENABLED) && validateSchema(schema) } override def convertInternalRowToCachedBatch( diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxParquetWriterInjects.scala b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxParquetWriterInjects.scala index b6d4f6557b82..a75fec02853a 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxParquetWriterInjects.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxParquetWriterInjects.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.execution.datasources.velox -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.sql.internal.SQLConf @@ -33,11 +33,11 @@ class VeloxParquetWriterInjects extends VeloxFormatWriterInjects { sparkOptions.put(SQLConf.PARQUET_COMPRESSION.key, compressionCodec) val blockSize = options.getOrElse( GlutenConfig.PARQUET_BLOCK_SIZE, - GlutenConfig.getConf.columnarParquetWriteBlockSize.toString) + GlutenConfig.get.columnarParquetWriteBlockSize.toString) sparkOptions.put(GlutenConfig.PARQUET_BLOCK_SIZE, blockSize) val blockRows = options.getOrElse( GlutenConfig.PARQUET_BLOCK_ROWS, - GlutenConfig.getConf.columnarParquetWriteBlockRows.toString) + GlutenConfig.get.columnarParquetWriteBlockRows.toString) sparkOptions.put(GlutenConfig.PARQUET_BLOCK_ROWS, blockRows) options .get(GlutenConfig.PARQUET_GZIP_WINDOW_SIZE) diff --git a/backends-velox/src/test/java/org/apache/gluten/test/MockVeloxBackend.java b/backends-velox/src/test/java/org/apache/gluten/test/MockVeloxBackend.java index 254e8e2830f4..efb3903d2324 100644 --- a/backends-velox/src/test/java/org/apache/gluten/test/MockVeloxBackend.java +++ b/backends-velox/src/test/java/org/apache/gluten/test/MockVeloxBackend.java @@ -16,7 +16,8 @@ */ package org.apache.gluten.test; -import org.apache.gluten.GlutenConfig; +import org.apache.gluten.config.GlutenConfig; +import org.apache.gluten.config.GlutenConfig$; import com.codahale.metrics.MetricRegistry; import org.apache.spark.SparkConf; @@ -71,7 +72,7 @@ public Object ask(Object message) throws Exception { private static SparkConf newSparkConf() { final SparkConf conf = new SparkConf(); conf.set(GlutenConfig.SPARK_OFFHEAP_SIZE_KEY(), "1g"); - conf.set(GlutenConfig.COLUMNAR_VELOX_CONNECTOR_IO_THREADS(), "0"); + conf.set(GlutenConfig$.MODULE$.COLUMNAR_VELOX_CONNECTOR_IO_THREADS().key(), "0"); return conf; } } diff --git a/backends-velox/src/test/scala/org/apache/gluten/benchmarks/NativeBenchmarkPlanGenerator.scala b/backends-velox/src/test/scala/org/apache/gluten/benchmarks/NativeBenchmarkPlanGenerator.scala index 1e378d16f14b..410af2ea0911 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/benchmarks/NativeBenchmarkPlanGenerator.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/benchmarks/NativeBenchmarkPlanGenerator.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.benchmarks -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{VeloxWholeStageTransformerSuite, WholeStageTransformer} import org.apache.spark.SparkConf diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/ArrowCsvScanSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/ArrowCsvScanSuite.scala index 116813fcf6da..374fa543af10 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/ArrowCsvScanSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/ArrowCsvScanSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.datasource.ArrowCSVFileFormat import org.apache.spark.SparkConf diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala index 0f94b8648e71..542593abb196 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/FallbackSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.SparkConf import org.apache.spark.sql.execution.{ColumnarBroadcastExchangeExec, ColumnarShuffleExchangeExec, SparkPlan} diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala index 989def88e70c..c4a2a6ca6bcc 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.expression.VeloxDummyExpression import org.apache.gluten.sql.shims.SparkShimLoader @@ -1969,11 +1969,11 @@ class MiscOperatorSuite extends VeloxWholeStageTransformerSuite with AdaptiveSpa } test("test 'spark.gluten.enabled'") { - withSQLConf(GlutenConfig.GLUTEN_ENABLED_KEY -> "true") { + withSQLConf(GlutenConfig.GLUTEN_ENABLED.key -> "true") { runQueryAndCompare("select * from lineitem limit 1") { checkGlutenOperatorMatch[FileSourceScanExecTransformer] } - withSQLConf(GlutenConfig.GLUTEN_ENABLED_KEY -> "false") { + withSQLConf(GlutenConfig.GLUTEN_ENABLED.key -> "false") { runQueryAndCompare("select * from lineitem limit 1") { checkSparkOperatorMatch[FileSourceScanExec] } diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala index a0c1f1ec000b..0d0607d3b93e 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.{SparkConf, SparkException} import org.apache.spark.sql.catalyst.optimizer.NullPropagation diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala index cf0bf272bc5b..67682306314d 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.extension.columnar.validator.FallbackInjects import org.apache.spark.SparkConf diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxColumnarCacheSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxColumnarCacheSuite.scala index 6786a47a4e94..e9151ad84ab6 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxColumnarCacheSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxColumnarCacheSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.utils.PlanUtil import org.apache.spark.SparkConf diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxMetricsSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxMetricsSuite.scala index a7ad8514f93a..c5c5ee22f6ca 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxMetricsSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxMetricsSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.spark.SparkConf diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxRoughCostModel2Suite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxRoughCostModel2Suite.scala index e29d3082a959..cf61a7323665 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxRoughCostModel2Suite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxRoughCostModel2Suite.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.SparkConf import org.apache.spark.sql.execution.ProjectExec diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxRoughCostModelSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxRoughCostModelSuite.scala index 86566dd0cde6..95831c1145fb 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxRoughCostModelSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxRoughCostModelSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.SparkConf import org.apache.spark.sql.execution.ProjectExec diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxScanSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxScanSuite.scala index 51a5925b0658..525eca940727 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxScanSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxScanSuite.scala @@ -16,9 +16,9 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.velox.VeloxBackendSettings import org.apache.gluten.benchmarks.RandomParquetDataGenerator +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.utils.VeloxFileSystemValidationJniWrapper import org.apache.spark.SparkConf diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxTPCHSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxTPCHSuite.scala index 44ffae45ad6f..e88446032a2c 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxTPCHSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxTPCHSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.SparkConf import org.apache.spark.sql.{DataFrame, Row, TestUtils} diff --git a/backends-velox/src/test/scala/org/apache/gluten/planner/VeloxRasSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/planner/VeloxRasSuite.scala index 885b549d0def..6304e06ad4d9 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/planner/VeloxRasSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/planner/VeloxRasSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.planner -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.extension.columnar.enumerated.EnumeratedTransform import org.apache.gluten.extension.columnar.enumerated.planner.GlutenOptimization import org.apache.gluten.extension.columnar.enumerated.planner.cost.{LegacyCoster, LongCostModel} diff --git a/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetReadSuite.scala b/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetReadSuite.scala index 383786d64760..72b6ad74a2ef 100644 --- a/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetReadSuite.scala +++ b/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetReadSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.execution -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{BasicScanExecTransformer, VeloxWholeStageTransformerSuite} import org.apache.spark.SparkConf diff --git a/backends-velox/src/test/scala/org/apache/spark/sql/execution/benchmark/ColumnarTableCacheBenchmark.scala b/backends-velox/src/test/scala/org/apache/spark/sql/execution/benchmark/ColumnarTableCacheBenchmark.scala index 46fc1b662ea5..d6b6c49563c8 100644 --- a/backends-velox/src/test/scala/org/apache/spark/sql/execution/benchmark/ColumnarTableCacheBenchmark.scala +++ b/backends-velox/src/test/scala/org/apache/spark/sql/execution/benchmark/ColumnarTableCacheBenchmark.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.execution.benchmark -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.benchmark.Benchmark import org.apache.spark.storage.StorageLevel @@ -33,7 +33,7 @@ object ColumnarTableCacheBenchmark extends SqlBasedBenchmark { private def doBenchmark(name: String, cardinality: Long)(f: => Unit): Unit = { val benchmark = new Benchmark(name, cardinality, output = output) - val flag = if (spark.sessionState.conf.getConf(GlutenConfig.COLUMNAR_TABLE_CACHE_ENABLED)) { + val flag = if (GlutenConfig.get.getConf(GlutenConfig.COLUMNAR_TABLE_CACHE_ENABLED)) { "enable" } else { "disable" diff --git a/backends-velox/src/test/scala/org/apache/spark/sql/execution/benchmark/StreamingAggregateBenchmark.scala b/backends-velox/src/test/scala/org/apache/spark/sql/execution/benchmark/StreamingAggregateBenchmark.scala index 77370d059caa..73752925f76e 100644 --- a/backends-velox/src/test/scala/org/apache/spark/sql/execution/benchmark/StreamingAggregateBenchmark.scala +++ b/backends-velox/src/test/scala/org/apache/spark/sql/execution/benchmark/StreamingAggregateBenchmark.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.execution.benchmark -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.benchmark.Benchmark import org.apache.spark.sql.internal.SQLConf diff --git a/gluten-arrow/src/main/java/org/apache/gluten/init/NativeBackendInitializer.java b/gluten-arrow/src/main/java/org/apache/gluten/init/NativeBackendInitializer.java index fe1ebc763415..232ec5d7a8ab 100644 --- a/gluten-arrow/src/main/java/org/apache/gluten/init/NativeBackendInitializer.java +++ b/gluten-arrow/src/main/java/org/apache/gluten/init/NativeBackendInitializer.java @@ -16,7 +16,7 @@ */ package org.apache.gluten.init; -import org.apache.gluten.GlutenConfig; +import org.apache.gluten.config.GlutenConfig; import org.apache.gluten.utils.ConfigUtil; import org.apache.spark.util.SparkShutdownManagerUtil; diff --git a/gluten-arrow/src/main/java/org/apache/gluten/memory/arrow/alloc/ManagedAllocationListener.java b/gluten-arrow/src/main/java/org/apache/gluten/memory/arrow/alloc/ManagedAllocationListener.java index a76c0aabee3b..ca8dcc5c14e9 100644 --- a/gluten-arrow/src/main/java/org/apache/gluten/memory/arrow/alloc/ManagedAllocationListener.java +++ b/gluten-arrow/src/main/java/org/apache/gluten/memory/arrow/alloc/ManagedAllocationListener.java @@ -16,7 +16,7 @@ */ package org.apache.gluten.memory.arrow.alloc; -import org.apache.gluten.GlutenConfig; +import org.apache.gluten.config.GlutenConfig$; import org.apache.gluten.memory.SimpleMemoryUsageRecorder; import org.apache.gluten.memory.memtarget.MemoryTarget; @@ -29,7 +29,7 @@ public class ManagedAllocationListener implements AllocationListener, AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger(ManagedAllocationListener.class); - public static long BLOCK_SIZE = GlutenConfig.getConf().memoryReservationBlockSize(); + public static long BLOCK_SIZE = GlutenConfig$.MODULE$.get().memoryReservationBlockSize(); private final MemoryTarget target; private final SimpleMemoryUsageRecorder sharedUsage; diff --git a/gluten-arrow/src/main/java/org/apache/gluten/memory/listener/ReservationListeners.java b/gluten-arrow/src/main/java/org/apache/gluten/memory/listener/ReservationListeners.java index 9d63a8601b4d..b17365a06df2 100644 --- a/gluten-arrow/src/main/java/org/apache/gluten/memory/listener/ReservationListeners.java +++ b/gluten-arrow/src/main/java/org/apache/gluten/memory/listener/ReservationListeners.java @@ -16,7 +16,7 @@ */ package org.apache.gluten.memory.listener; -import org.apache.gluten.GlutenConfig; +import org.apache.gluten.config.GlutenConfig$; import org.apache.gluten.memory.MemoryUsageStatsBuilder; import org.apache.gluten.memory.SimpleMemoryUsageRecorder; import org.apache.gluten.memory.memtarget.*; @@ -44,8 +44,8 @@ public static ReservationListener create( private static ReservationListener create0( String name, Spiller spiller, Map mutableStats) { // Memory target. - final double overAcquiredRatio = GlutenConfig.getConf().memoryOverAcquiredRatio(); - final long reservationBlockSize = GlutenConfig.getConf().memoryReservationBlockSize(); + final double overAcquiredRatio = GlutenConfig$.MODULE$.get().memoryOverAcquiredRatio(); + final long reservationBlockSize = GlutenConfig$.MODULE$.get().memoryReservationBlockSize(); final TaskMemoryManager tmm = TaskResources.getLocalTaskContext().taskMemoryManager(); final TreeMemoryTarget consumer = MemoryTargets.newConsumer( diff --git a/gluten-arrow/src/main/scala/org/apache/gluten/memory/NativeMemoryManager.scala b/gluten-arrow/src/main/scala/org/apache/gluten/memory/NativeMemoryManager.scala index e65d3c4d9589..f2db33d8e120 100644 --- a/gluten-arrow/src/main/scala/org/apache/gluten/memory/NativeMemoryManager.scala +++ b/gluten-arrow/src/main/scala/org/apache/gluten/memory/NativeMemoryManager.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.memory -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.exception.GlutenException import org.apache.gluten.memory.listener.ReservationListeners import org.apache.gluten.memory.memtarget.{KnownNameAndStats, MemoryTarget, Spiller, Spillers} diff --git a/gluten-arrow/src/main/scala/org/apache/gluten/runtime/Runtime.scala b/gluten-arrow/src/main/scala/org/apache/gluten/runtime/Runtime.scala index 8741c12474e4..f4fb2fa42c19 100644 --- a/gluten-arrow/src/main/scala/org/apache/gluten/runtime/Runtime.scala +++ b/gluten-arrow/src/main/scala/org/apache/gluten/runtime/Runtime.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.runtime -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.exception.GlutenException import org.apache.gluten.memory.NativeMemoryManager import org.apache.gluten.utils.ConfigUtil diff --git a/gluten-arrow/src/main/scala/org/apache/gluten/utils/DebugUtil.scala b/gluten-arrow/src/main/scala/org/apache/gluten/utils/DebugUtil.scala index f9bb7478e7d0..3b24515e7446 100644 --- a/gluten-arrow/src/main/scala/org/apache/gluten/utils/DebugUtil.scala +++ b/gluten-arrow/src/main/scala/org/apache/gluten/utils/DebugUtil.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.utils -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.TaskContext @@ -27,23 +27,23 @@ object DebugUtil { // if specify stageId and partitionId, then only do that partition for that stage def saveInputToFile(): Boolean = { def taskIdMatches = - GlutenConfig.getConf.benchmarkTaskId.nonEmpty && - GlutenConfig.getConf.benchmarkTaskId + GlutenConfig.get.benchmarkTaskId.nonEmpty && + GlutenConfig.get.benchmarkTaskId .split(",") .map(_.toLong) .contains(TaskContext.get().taskAttemptId()) def partitionIdMatches = - TaskContext.get().stageId() == GlutenConfig.getConf.benchmarkStageId && - (GlutenConfig.getConf.benchmarkPartitionId.isEmpty || - GlutenConfig.getConf.benchmarkPartitionId + TaskContext.get().stageId() == GlutenConfig.get.benchmarkStageId && + (GlutenConfig.get.benchmarkPartitionId.isEmpty || + GlutenConfig.get.benchmarkPartitionId .split(",") .map(_.toInt) .contains(TaskContext.get().partitionId())) val saveInput = taskIdMatches || partitionIdMatches if (saveInput) { - if (GlutenConfig.getConf.benchmarkSaveDir.isEmpty) { + if (GlutenConfig.get.benchmarkSaveDir.isEmpty) { throw new IllegalArgumentException(GlutenConfig.BENCHMARK_SAVE_DIR.key + " is not set.") } } diff --git a/gluten-celeborn/src-celeborn/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java b/gluten-celeborn/src-celeborn/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java index 5e14531d6f19..9ce8be7eb942 100644 --- a/gluten-celeborn/src-celeborn/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java +++ b/gluten-celeborn/src-celeborn/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java @@ -16,8 +16,8 @@ */ package org.apache.spark.shuffle.gluten.celeborn; -import org.apache.gluten.GlutenConfig; import org.apache.gluten.backendsapi.BackendsApiManager; +import org.apache.gluten.config.GlutenConfig$; import org.apache.gluten.exception.GlutenException; import com.google.common.base.Preconditions; @@ -216,7 +216,7 @@ public ShuffleHandle registerShuffle( if (dependency instanceof ColumnarShuffleDependency) { if (fallbackPolicyRunner.applyAllFallbackPolicy( lifecycleManager, dependency.partitioner().numPartitions())) { - if (GlutenConfig.getConf().enableCelebornFallback()) { + if (GlutenConfig$.MODULE$.get().enableCelebornFallback()) { logger.warn("Fallback to ColumnarShuffleManager!"); columnarShuffleIds.add(shuffleId); return columnarShuffleManager().registerShuffle(shuffleId, dependency); diff --git a/gluten-celeborn/src-celeborn/main/scala/org/apache/spark/shuffle/CelebornColumnarShuffleWriter.scala b/gluten-celeborn/src-celeborn/main/scala/org/apache/spark/shuffle/CelebornColumnarShuffleWriter.scala index 42e939e4420d..7b61cd99f4e6 100644 --- a/gluten-celeborn/src-celeborn/main/scala/org/apache/spark/shuffle/CelebornColumnarShuffleWriter.scala +++ b/gluten-celeborn/src-celeborn/main/scala/org/apache/spark/shuffle/CelebornColumnarShuffleWriter.scala @@ -18,7 +18,8 @@ package org.apache.spark.shuffle import org.apache.celeborn.client.ShuffleClient import org.apache.celeborn.common.CelebornConf -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig + import org.apache.spark._ import org.apache.spark.internal.Logging import org.apache.spark.internal.config.SHUFFLE_COMPRESS @@ -52,8 +53,8 @@ abstract class CelebornColumnarShuffleWriter[K, V]( protected val mapId: Int = context.partitionId() protected lazy val nativeBufferSize: Int = { - val bufferSize = GlutenConfig.getConf.shuffleWriterBufferSize - val maxBatchSize = GlutenConfig.getConf.maxBatchSize + val bufferSize = GlutenConfig.get.shuffleWriterBufferSize + val maxBatchSize = GlutenConfig.get.maxBatchSize if (bufferSize > maxBatchSize) { logInfo( s"${GlutenConfig.SHUFFLE_WRITER_BUFFER_SIZE.key} ($bufferSize) exceeds max " + @@ -95,13 +96,13 @@ abstract class CelebornColumnarShuffleWriter[K, V]( GlutenShuffleUtils.getCompressionLevel( conf, customizedCompressionCodec, - GlutenConfig.getConf.columnarShuffleCodecBackend.orNull) + GlutenConfig.get.columnarShuffleCodecBackend.orNull) protected val compressionBufferSize: Int = GlutenShuffleUtils.getSortEvictBufferSize(conf, customizedCompressionCodec) protected val bufferCompressThreshold: Int = - GlutenConfig.getConf.columnarShuffleCompressionThreshold + GlutenConfig.get.columnarShuffleCompressionThreshold // Are we in the process of stopping? Because map tasks can call stop() with success = true // and then call stop() with success = false if they get an exception, we want to make sure diff --git a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/DynamicOffHeapSizingMemoryTarget.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/DynamicOffHeapSizingMemoryTarget.java index ea18a05d0c60..2233e87f158d 100644 --- a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/DynamicOffHeapSizingMemoryTarget.java +++ b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/DynamicOffHeapSizingMemoryTarget.java @@ -16,7 +16,7 @@ */ package org.apache.gluten.memory.memtarget; -import org.apache.gluten.GlutenConfig; +import org.apache.gluten.config.GlutenConfig$; import org.apache.spark.annotation.Experimental; import org.slf4j.Logger; @@ -30,7 +30,7 @@ public class DynamicOffHeapSizingMemoryTarget implements MemoryTarget { private final MemoryTarget delegated; // When dynamic off-heap sizing is enabled, the off-heap should be sized for the total usable // memory, so we can use it as the max memory we will use. - private static final long MAX_MEMORY_IN_BYTES = GlutenConfig.getConf().offHeapMemorySize(); + private static final long MAX_MEMORY_IN_BYTES = GlutenConfig$.MODULE$.get().offHeapMemorySize(); private static final AtomicLong USED_OFFHEAP_BYTES = new AtomicLong(); public DynamicOffHeapSizingMemoryTarget(MemoryTarget delegated) { diff --git a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java index c6f5b59de8c2..0dc262855f87 100644 --- a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java +++ b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java @@ -16,7 +16,7 @@ */ package org.apache.gluten.memory.memtarget; -import org.apache.gluten.GlutenConfig; +import org.apache.gluten.config.GlutenConfig$; import org.apache.gluten.memory.MemoryUsageStatsBuilder; import org.apache.gluten.memory.memtarget.spark.TreeMemoryConsumers; @@ -50,7 +50,7 @@ public static MemoryTarget overAcquire( @Experimental public static MemoryTarget dynamicOffHeapSizingIfEnabled(MemoryTarget memoryTarget) { - if (GlutenConfig.getConf().dynamicOffHeapSizingEnabled()) { + if (GlutenConfig$.MODULE$.get().dynamicOffHeapSizingEnabled()) { return new DynamicOffHeapSizingMemoryTarget(memoryTarget); } @@ -63,7 +63,7 @@ public static TreeMemoryTarget newConsumer( Spiller spiller, Map virtualChildren) { final TreeMemoryConsumers.Factory factory = TreeMemoryConsumers.factory(tmm); - if (GlutenConfig.getConf().memoryIsolation()) { + if (GlutenConfig$.MODULE$.get().memoryIsolation()) { return TreeMemoryTargets.newChild(factory.isolatedRoot(), name, spiller, virtualChildren); } final TreeMemoryTarget root = factory.legacyRoot(); diff --git a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/ThrowOnOomMemoryTarget.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/ThrowOnOomMemoryTarget.java index 67edb713a7ce..f517a2cc44cf 100644 --- a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/ThrowOnOomMemoryTarget.java +++ b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/ThrowOnOomMemoryTarget.java @@ -16,7 +16,7 @@ */ package org.apache.gluten.memory.memtarget; -import org.apache.gluten.GlutenConfig$; +import org.apache.gluten.config.GlutenConfig$; import org.apache.spark.memory.SparkMemoryUtil; import org.apache.spark.sql.internal.SQLConf; diff --git a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumers.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumers.java index a11a4a3e4a19..56b43c3161d9 100644 --- a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumers.java +++ b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumers.java @@ -16,7 +16,7 @@ */ package org.apache.gluten.memory.memtarget.spark; -import org.apache.gluten.GlutenConfig; +import org.apache.gluten.config.GlutenConfig$; import org.apache.gluten.memory.memtarget.Spillers; import org.apache.gluten.memory.memtarget.TreeMemoryTarget; @@ -77,7 +77,7 @@ public TreeMemoryTarget legacyRoot() { *

See GLUTEN-3030 */ public TreeMemoryTarget isolatedRoot() { - return ofCapacity(GlutenConfig.getConf().conservativeTaskOffHeapMemorySize()); + return ofCapacity(GlutenConfig$.MODULE$.get().conservativeTaskOffHeapMemorySize()); } } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala index 4b27cc75bd70..2cd524c32741 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala @@ -17,8 +17,9 @@ package org.apache.gluten import org.apache.gluten.GlutenBuildInfo._ -import org.apache.gluten.GlutenConfig._ import org.apache.gluten.component.Component +import org.apache.gluten.config.GlutenConfig +import org.apache.gluten.config.GlutenConfig._ import org.apache.gluten.events.GlutenBuildInfoEvent import org.apache.gluten.exception.GlutenException import org.apache.gluten.extension.GlutenSessionExtensions diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/GlutenSessionExtensions.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/GlutenSessionExtensions.scala index 5b3e97f356ca..4e7b2a034cd6 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/GlutenSessionExtensions.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/GlutenSessionExtensions.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.extension -import org.apache.gluten.GlutenConfig import org.apache.gluten.component.Component +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.extension.injector.Injector import org.apache.spark.internal.Logging @@ -32,7 +32,7 @@ private[gluten] class GlutenSessionExtensions injector.control.disableOn { session => val glutenEnabledGlobally = session.conf - .get(GlutenConfig.GLUTEN_ENABLED_KEY, GlutenConfig.GLUTEN_ENABLED_BY_DEFAULT.toString) + .get(GlutenConfig.GLUTEN_ENABLED.key, GlutenConfig.GLUTEN_ENABLED.defaultValueString) .toBoolean val disabled = !glutenEnabledGlobally logDebug(s"Gluten is disabled by variable: glutenEnabledGlobally: $glutenEnabledGlobally") diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ColumnarRuleApplier.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ColumnarRuleApplier.scala index bf7b84c9b316..3e9bd72c4507 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ColumnarRuleApplier.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ColumnarRuleApplier.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.extension.columnar -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.extension.util.AdaptiveContext import org.apache.spark.sql.SparkSession diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ColumnarRuleExecutor.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ColumnarRuleExecutor.scala index eea698e078bd..e10d9bd0b443 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ColumnarRuleExecutor.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ColumnarRuleExecutor.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.extension.columnar -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.logging.LogLevelUtil import org.apache.gluten.metrics.GlutenTimeMetric @@ -57,7 +57,7 @@ object ColumnarRuleExecutor { override def apply(plan: SparkPlan): SparkPlan = { val (out, millisTime) = GlutenTimeMetric.recordMillisTime(delegate.apply(plan)) - logOnLevel(GlutenConfig.getConf.transformPlanLogLevel, message(plan, out, millisTime)) + logOnLevel(GlutenConfig.get.transformPlanLogLevel, message(plan, out, millisTime)) out } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/injector/GlutenInjector.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/injector/GlutenInjector.scala index efe584d44126..11172a9b3636 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/injector/GlutenInjector.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/injector/GlutenInjector.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.extension.injector -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.extension.GlutenColumnarRule import org.apache.gluten.extension.columnar.ColumnarRuleApplier import org.apache.gluten.extension.columnar.ColumnarRuleApplier.ColumnarRuleCall diff --git a/gluten-core/src/main/scala/org/apache/gluten/softaffinity/SoftAffinityManager.scala b/gluten-core/src/main/scala/org/apache/gluten/softaffinity/SoftAffinityManager.scala index 754e42b81c4d..f755af8d367f 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/softaffinity/SoftAffinityManager.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/softaffinity/SoftAffinityManager.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.softaffinity -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.hash.ConsistentHash import org.apache.gluten.logging.LogLevelUtil import org.apache.gluten.softaffinity.strategy.{ConsistentHashSoftAffinityStrategy, ExecutorNode} @@ -116,7 +116,7 @@ abstract class AffinityManager extends LogLevelUtil with Logging { totalRegisteredExecutors.addAndGet(1) } logOnLevel( - GlutenConfig.getConf.softAffinityLogLevel, + GlutenConfig.get.softAffinityLogLevel, s"After adding executor ${execHostId._1} on host ${execHostId._2}, " + s"idForExecutors is ${idForExecutors.mkString(",")}, " + s"sortedIdForExecutors is ${sortedIdForExecutors.mkString(",")}, " + @@ -149,7 +149,7 @@ abstract class AffinityManager extends LogLevelUtil with Logging { totalRegisteredExecutors.addAndGet(-1) } logOnLevel( - GlutenConfig.getConf.softAffinityLogLevel, + GlutenConfig.get.softAffinityLogLevel, s"After removing executor $execId, " + s"idForExecutors is ${idForExecutors.mkString(",")}, " + s"sortedIdForExecutors is ${sortedIdForExecutors.mkString(",")}, " + @@ -196,7 +196,7 @@ abstract class AffinityManager extends LogLevelUtil with Logging { (originalValues ++ value) } logOnLevel( - GlutenConfig.getConf.softAffinityLogLevel, + GlutenConfig.get.softAffinityLogLevel, s"update host for $key: ${values.mkString(",")}") duplicateReadingInfos.put(key, values) } @@ -279,7 +279,7 @@ abstract class AffinityManager extends LogLevelUtil with Logging { if (!hosts.isEmpty) { rand.shuffle(hosts) logOnLevel( - GlutenConfig.getConf.softAffinityLogLevel, + GlutenConfig.get.softAffinityLogLevel, s"get host for $f: ${hosts.distinct.mkString(",")}") } hosts.distinct.toSeq diff --git a/gluten-core/src/main/scala/org/apache/spark/softaffinity/SoftAffinity.scala b/gluten-core/src/main/scala/org/apache/spark/softaffinity/SoftAffinity.scala index 66e4456d46d6..f0f16bae76f5 100644 --- a/gluten-core/src/main/scala/org/apache/spark/softaffinity/SoftAffinity.scala +++ b/gluten-core/src/main/scala/org/apache/spark/softaffinity/SoftAffinity.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.softaffinity -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.logging.LogLevelUtil import org.apache.gluten.softaffinity.{AffinityManager, SoftAffinityManager} @@ -66,7 +66,7 @@ abstract class Affinity(val manager: AffinityManager) extends LogLevelUtil with val locations = manager.askExecutors(filePath) if (locations.nonEmpty) { logOnLevel( - GlutenConfig.getConf.softAffinityLogLevel, + GlutenConfig.get.softAffinityLogLevel, s"SAMetrics=File $filePath - the expected executors are ${locations.mkString("_")} ") locations.map { case (executor, host) => toTaskLocation(host, executor) } } else { diff --git a/gluten-core/src/main/scala/org/apache/spark/task/TaskResources.scala b/gluten-core/src/main/scala/org/apache/spark/task/TaskResources.scala index 2f609b026db3..ba5c53614fc4 100644 --- a/gluten-core/src/main/scala/org/apache/spark/task/TaskResources.scala +++ b/gluten-core/src/main/scala/org/apache/spark/task/TaskResources.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.task -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.task.TaskListener import org.apache.spark.{TaskContext, TaskFailedReason, TaskKilledException, UnknownReason} diff --git a/gluten-core/src/test/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumerTest.java b/gluten-core/src/test/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumerTest.java index 6cb38fe8d5d3..a31a1d6d350e 100644 --- a/gluten-core/src/test/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumerTest.java +++ b/gluten-core/src/test/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumerTest.java @@ -16,7 +16,7 @@ */ package org.apache.gluten.memory.memtarget.spark; -import org.apache.gluten.GlutenConfig; +import org.apache.gluten.config.GlutenConfig; import org.apache.gluten.memory.memtarget.MemoryTarget; import org.apache.gluten.memory.memtarget.Spiller; import org.apache.gluten.memory.memtarget.Spillers; diff --git a/gluten-iceberg/src-iceberg/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesNode.java b/gluten-iceberg/src-iceberg/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesNode.java index 398bdbdb5f03..63794bf1fd60 100644 --- a/gluten-iceberg/src-iceberg/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesNode.java +++ b/gluten-iceberg/src-iceberg/main/java/org/apache/gluten/substrait/rel/IcebergLocalFilesNode.java @@ -16,7 +16,7 @@ */ package org.apache.gluten.substrait.rel; -import org.apache.gluten.GlutenConfig; +import org.apache.gluten.config.GlutenConfig$; import io.substrait.proto.ReadRel; import org.apache.iceberg.DeleteFile; @@ -64,7 +64,7 @@ protected void processFileBuilder(ReadRel.LocalFiles.FileOrFiles.Builder fileBui ReadRel.LocalFiles.FileOrFiles.ParquetReadOptions parquetReadOptions = ReadRel.LocalFiles.FileOrFiles.ParquetReadOptions.newBuilder() .setEnableRowGroupMaxminIndex( - GlutenConfig.getConf().enableParquetRowGroupMaxMinIndex()) + GlutenConfig$.MODULE$.get().enableParquetRowGroupMaxMinIndex()) .build(); icebergBuilder.setParquet(parquetReadOptions); break; @@ -104,7 +104,7 @@ protected void processFileBuilder(ReadRel.LocalFiles.FileOrFiles.Builder fileBui ReadRel.LocalFiles.FileOrFiles.ParquetReadOptions parquetReadOptions = ReadRel.LocalFiles.FileOrFiles.ParquetReadOptions.newBuilder() .setEnableRowGroupMaxminIndex( - GlutenConfig.getConf().enableParquetRowGroupMaxMinIndex()) + GlutenConfig$.MODULE$.get().enableParquetRowGroupMaxMinIndex()) .build(); deleteFileBuilder.setParquet(parquetReadOptions); break; diff --git a/gluten-iceberg/src-iceberg/test/scala/org/apache/gluten/execution/IcebergSuite.scala b/gluten-iceberg/src-iceberg/test/scala/org/apache/gluten/execution/IcebergSuite.scala index cb630a147539..8b992f575b1d 100644 --- a/gluten-iceberg/src-iceberg/test/scala/org/apache/gluten/execution/IcebergSuite.scala +++ b/gluten-iceberg/src-iceberg/test/scala/org/apache/gluten/execution/IcebergSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.SparkConf import org.apache.spark.sql.Row @@ -64,7 +64,7 @@ abstract class IcebergSuite extends WholeStageTransformerSuite { val rightTable = "p_int_tb" withTable(leftTable, rightTable) { // Partition key of string type. - withSQLConf(GlutenConfig.GLUTEN_ENABLED_KEY -> "false") { + withSQLConf(GlutenConfig.GLUTEN_ENABLED.key -> "false") { // Gluten does not support write iceberg table. spark.sql(s""" |create table $leftTable(id int, name string, p string) @@ -85,7 +85,7 @@ abstract class IcebergSuite extends WholeStageTransformerSuite { // Partition key of integer type. withSQLConf( - GlutenConfig.GLUTEN_ENABLED_KEY -> "false" + GlutenConfig.GLUTEN_ENABLED.key -> "false" ) { // Gluten does not support write iceberg table. spark.sql(s""" @@ -144,7 +144,7 @@ abstract class IcebergSuite extends WholeStageTransformerSuite { val rightTable = "p_int_tb" withTable(leftTable, rightTable) { // Partition key of string type. - withSQLConf(GlutenConfig.GLUTEN_ENABLED_KEY -> "false") { + withSQLConf(GlutenConfig.GLUTEN_ENABLED.key -> "false") { // Gluten does not support write iceberg table. spark.sql(s""" |create table $leftTable(id int, name string, p int) @@ -165,7 +165,7 @@ abstract class IcebergSuite extends WholeStageTransformerSuite { // Partition key of integer type. withSQLConf( - GlutenConfig.GLUTEN_ENABLED_KEY -> "false" + GlutenConfig.GLUTEN_ENABLED.key -> "false" ) { // Gluten does not support write iceberg table. spark.sql(s""" @@ -224,7 +224,7 @@ abstract class IcebergSuite extends WholeStageTransformerSuite { val rightTable = "p_int_tb" withTable(leftTable, rightTable) { // Partition key of string type. - withSQLConf(GlutenConfig.GLUTEN_ENABLED_KEY -> "false") { + withSQLConf(GlutenConfig.GLUTEN_ENABLED.key -> "false") { // Gluten does not support write iceberg table. spark.sql(s""" |create table $leftTable(id int, name string, p int) @@ -245,7 +245,7 @@ abstract class IcebergSuite extends WholeStageTransformerSuite { // Partition key of integer type. withSQLConf( - GlutenConfig.GLUTEN_ENABLED_KEY -> "false" + GlutenConfig.GLUTEN_ENABLED.key -> "false" ) { // Gluten does not support write iceberg table. spark.sql(s""" @@ -339,7 +339,7 @@ abstract class IcebergSuite extends WholeStageTransformerSuite { test("iceberg read mor table - delete and update") { withTable("iceberg_mor_tb") { - withSQLConf(GlutenConfig.GLUTEN_ENABLED_KEY -> "false") { + withSQLConf(GlutenConfig.GLUTEN_ENABLED.key -> "false") { spark.sql(""" |create table iceberg_mor_tb ( | id int, @@ -391,7 +391,7 @@ abstract class IcebergSuite extends WholeStageTransformerSuite { test("iceberg read mor table - merge into") { withTable("iceberg_mor_tb", "merge_into_source_tb") { - withSQLConf(GlutenConfig.GLUTEN_ENABLED_KEY -> "false") { + withSQLConf(GlutenConfig.GLUTEN_ENABLED.key -> "false") { spark.sql(""" |create table iceberg_mor_tb ( | id int, diff --git a/gluten-substrait/src/main/java/org/apache/gluten/substrait/rel/LocalFilesNode.java b/gluten-substrait/src/main/java/org/apache/gluten/substrait/rel/LocalFilesNode.java index 9513f497602a..0bb1c3f79438 100644 --- a/gluten-substrait/src/main/java/org/apache/gluten/substrait/rel/LocalFilesNode.java +++ b/gluten-substrait/src/main/java/org/apache/gluten/substrait/rel/LocalFilesNode.java @@ -16,7 +16,7 @@ */ package org.apache.gluten.substrait.rel; -import org.apache.gluten.GlutenConfig; +import org.apache.gluten.config.GlutenConfig$; import org.apache.gluten.expression.ConverterUtils; import io.substrait.proto.NamedStruct; @@ -198,7 +198,7 @@ public ReadRel.LocalFiles toProtobuf() { ReadRel.LocalFiles.FileOrFiles.ParquetReadOptions parquetReadOptions = ReadRel.LocalFiles.FileOrFiles.ParquetReadOptions.newBuilder() .setEnableRowGroupMaxminIndex( - GlutenConfig.getConf().enableParquetRowGroupMaxMinIndex()) + GlutenConfig$.MODULE$.get().enableParquetRowGroupMaxMinIndex()) .build(); fileBuilder.setParquet(parquetReadOptions); break; @@ -225,15 +225,15 @@ public ReadRel.LocalFiles toProtobuf() { .setHeader(Long.parseLong(header)) .setEscape(escape) .setNullValue(nullValue) - .setMaxBlockSize(GlutenConfig.getConf().textInputMaxBlockSize()) - .setEmptyAsDefault(GlutenConfig.getConf().textIputEmptyAsDefault()) + .setMaxBlockSize(GlutenConfig$.MODULE$.get().textInputMaxBlockSize()) + .setEmptyAsDefault(GlutenConfig$.MODULE$.get().textIputEmptyAsDefault()) .build(); fileBuilder.setText(textReadOptions); break; case JsonReadFormat: ReadRel.LocalFiles.FileOrFiles.JsonReadOptions jsonReadOptions = ReadRel.LocalFiles.FileOrFiles.JsonReadOptions.newBuilder() - .setMaxBlockSize(GlutenConfig.getConf().textInputMaxBlockSize()) + .setMaxBlockSize(GlutenConfig$.MODULE$.get().textInputMaxBlockSize()) .build(); fileBuilder.setJson(jsonReadOptions); break; diff --git a/gluten-substrait/src/main/java/org/apache/gluten/test/TestStats.java b/gluten-substrait/src/main/java/org/apache/gluten/test/TestStats.java index 37d9e09bb5d8..0a694a89e271 100644 --- a/gluten-substrait/src/main/java/org/apache/gluten/test/TestStats.java +++ b/gluten-substrait/src/main/java/org/apache/gluten/test/TestStats.java @@ -16,7 +16,7 @@ */ package org.apache.gluten.test; -import org.apache.gluten.GlutenConfig; +import org.apache.gluten.config.GlutenConfig$; import java.util.*; @@ -37,7 +37,7 @@ public class TestStats { public static int offloadGlutenTestNumber = 0; private static boolean enabled() { - return GlutenConfig.getConf().collectUtStats(); + return GlutenConfig$.MODULE$.get().collectUtStats(); } public static void reset() { diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala b/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala index 052348aee4b6..0c220ada64f9 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.backendsapi -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.extension.ValidationResult import org.apache.gluten.extension.columnar.transition.Convention import org.apache.gluten.substrait.rel.LocalFilesNode @@ -68,7 +68,7 @@ trait BackendSettingsApi { false } def supportColumnarShuffleExec(): Boolean = { - GlutenConfig.getConf.enableColumnarShuffle + GlutenConfig.get.enableColumnarShuffle } def enableJoinKeysRewrite(): Boolean = true def supportHashBuildJoinTypeOnLeft: JoinType => Boolean = { diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/IteratorApi.scala b/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/IteratorApi.scala index 11211bd0da91..4e9b939debe5 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/IteratorApi.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/IteratorApi.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.backendsapi -import org.apache.gluten.GlutenNumaBindingInfo +import org.apache.gluten.config.GlutenNumaBindingInfo import org.apache.gluten.execution.{BaseGlutenPartition, BasicScanExecTransformer, WholeStageTransformContext} import org.apache.gluten.metrics.IMetrics import org.apache.gluten.substrait.plan.PlanNode diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala b/gluten-substrait/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala index 0d1296999a73..3b3ad4aa5c61 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.expression.{ConverterUtils, ExpressionConverter} import org.apache.gluten.extension.ValidationResult import org.apache.gluten.sql.shims.SparkShimLoader @@ -52,7 +52,7 @@ trait BasicScanExecTransformer extends LeafTransformSupport with BaseDataSource val fileFormat: ReadFileFormat def getRootFilePaths: Seq[String] = { - if (GlutenConfig.getConf.scanFileSchemeValidationEnabled) { + if (GlutenConfig.get.scanFileSchemeValidationEnabled) { getRootPathsInternal } else { Seq.empty diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/execution/BroadcastNestedLoopJoinExecTransformer.scala b/gluten-substrait/src/main/scala/org/apache/gluten/execution/BroadcastNestedLoopJoinExecTransformer.scala index 1e0c3d5a7e70..9f12cdefeeb3 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/execution/BroadcastNestedLoopJoinExecTransformer.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/execution/BroadcastNestedLoopJoinExecTransformer.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.extension.ValidationResult import org.apache.gluten.metrics.MetricsUpdater import org.apache.gluten.substrait.{JoinParams, SubstraitContext} @@ -185,7 +185,7 @@ abstract class BroadcastNestedLoopJoinExecTransformer( } override protected def doValidateInternal(): ValidationResult = { - if (!GlutenConfig.getConf.broadcastNestedLoopJoinTransformerTransformerEnabled) { + if (!GlutenConfig.get.broadcastNestedLoopJoinTransformerTransformerEnabled) { return ValidationResult.failed( s"Config ${GlutenConfig.BROADCAST_NESTED_LOOP_JOIN_TRANSFORMER_ENABLED.key} not enabled") } diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/execution/GlutenWholeStageColumnarRDD.scala b/gluten-substrait/src/main/scala/org/apache/gluten/execution/GlutenWholeStageColumnarRDD.scala index de5fc85f356d..a54bbfad4e54 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/execution/GlutenWholeStageColumnarRDD.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/execution/GlutenWholeStageColumnarRDD.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.metrics.{GlutenTimeMetric, IMetrics} import org.apache.spark.{Partition, SparkContext, SparkException, TaskContext} @@ -59,7 +59,7 @@ class GlutenWholeStageColumnarRDD( updateInputMetrics: InputMetricsWrapper => Unit, updateNativeMetrics: IMetrics => Unit) extends RDD[ColumnarBatch](sc, rdds.getDependencies) { - private val numaBindingInfo = GlutenConfig.getConf.numaBindingInfo + private val numaBindingInfo = GlutenConfig.get.numaBindingInfo override def compute(split: Partition, context: TaskContext): Iterator[ColumnarBatch] = { diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/execution/HashAggregateExecBaseTransformer.scala b/gluten-substrait/src/main/scala/org/apache/gluten/execution/HashAggregateExecBaseTransformer.scala index 9a28af801d83..12ac9f8fcf83 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/execution/HashAggregateExecBaseTransformer.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/execution/HashAggregateExecBaseTransformer.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.exception.GlutenNotSupportException import org.apache.gluten.expression._ import org.apache.gluten.extension.ValidationResult @@ -53,7 +53,7 @@ abstract class HashAggregateExecBaseTransformer( BackendsApiManager.getMetricsApiInstance.genHashAggregateTransformerMetrics(sparkContext) protected def isCapableForStreamingAggregation: Boolean = { - if (!conf.getConf(GlutenConfig.COLUMNAR_PREFER_STREAMING_AGGREGATE)) { + if (!GlutenConfig.get.getConf(GlutenConfig.COLUMNAR_PREFER_STREAMING_AGGREGATE)) { return false } if (groupingExpressions.isEmpty) { diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala b/gluten-substrait/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala index fc38d0c3ffa3..de18305cec38 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/execution/WholeStageTransformer.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.execution -import org.apache.gluten.{GlutenConfig, GlutenNumaBindingInfo} import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.config.{GlutenConfig, GlutenNumaBindingInfo} import org.apache.gluten.exception.{GlutenException, GlutenNotSupportException} import org.apache.gluten.expression._ import org.apache.gluten.extension.ValidationResult @@ -62,7 +62,7 @@ case class WholeStageTransformContext(root: PlanNode, substraitContext: Substrai * Since https://github.com/apache/incubator-gluten/pull/2185. */ trait ValidatablePlan extends GlutenPlan with LogLevelUtil { - protected def glutenConf: GlutenConfig = GlutenConfig.getConf + protected def glutenConf: GlutenConfig = GlutenConfig.get protected lazy val enableNativeValidation = glutenConf.enableNativeValidation @@ -214,7 +214,7 @@ case class WholeStageTransformer(child: SparkPlan, materializeInput: Boolean = f val serializableHadoopConf: SerializableConfiguration = new SerializableConfiguration( sparkContext.hadoopConfiguration) - val numaBindingInfo: GlutenNumaBindingInfo = GlutenConfig.getConf.numaBindingInfo + val numaBindingInfo: GlutenNumaBindingInfo = GlutenConfig.get.numaBindingInfo @transient private var wholeStageTransformerContext: Option[WholeStageTransformContext] = None @@ -271,11 +271,12 @@ case class WholeStageTransformer(child: SparkPlan, materializeInput: Boolean = f override def nodeName: String = s"WholeStageCodegenTransformer ($transformStageId)" override def verboseStringWithOperatorId(): String = { - val nativePlan = if (conf.getConf(GlutenConfig.INJECT_NATIVE_PLAN_STRING_TO_EXPLAIN)) { - s"Native Plan:\n${nativePlanString()}" - } else { - "" - } + val nativePlan = + if (GlutenConfig.get.getConf(GlutenConfig.INJECT_NATIVE_PLAN_STRING_TO_EXPLAIN)) { + s"Native Plan:\n${nativePlanString()}" + } else { + "" + } super.verboseStringWithOperatorId() ++ nativePlan } @@ -315,7 +316,7 @@ case class WholeStageTransformer(child: SparkPlan, materializeInput: Boolean = f def doWholeStageTransform(): WholeStageTransformContext = { val context = generateWholeStageTransformContext() - if (conf.getConf(GlutenConfig.CACHE_WHOLE_STAGE_TRANSFORMER_CONTEXT)) { + if (GlutenConfig.get.getConf(GlutenConfig.CACHE_WHOLE_STAGE_TRANSFORMER_CONTEXT)) { wholeStageTransformerContext = Some(context) } context @@ -360,7 +361,7 @@ case class WholeStageTransformer(child: SparkPlan, materializeInput: Boolean = f }( t => logOnLevel( - GlutenConfig.getConf.substraitPlanLogLevel, + GlutenConfig.get.substraitPlanLogLevel, s"$nodeName generating the substrait plan took: $t ms.")) val inputRDDs = new ColumnarInputRDDsWrapper(columnarInputRDDs) // Check if BatchScan exists. @@ -376,7 +377,7 @@ case class WholeStageTransformer(child: SparkPlan, materializeInput: Boolean = f val allScanPartitions = basicScanExecTransformers.map(_.getPartitions.toIndexedSeq) val allScanSplitInfos = getSplitInfosFromPartitions(basicScanExecTransformers, allScanPartitions) - if (GlutenConfig.getConf.enableHdfsViewfs) { + if (GlutenConfig.get.enableHdfsViewfs) { allScanSplitInfos.foreach { splitInfos => splitInfos.foreach { diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/execution/WholeStageZippedPartitionsRDD.scala b/gluten-substrait/src/main/scala/org/apache/gluten/execution/WholeStageZippedPartitionsRDD.scala index 32cf23d437a8..1bb19cfd8b98 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/execution/WholeStageZippedPartitionsRDD.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/execution/WholeStageZippedPartitionsRDD.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenNumaBindingInfo import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.config.GlutenNumaBindingInfo import org.apache.gluten.metrics.{GlutenTimeMetric, IMetrics} import org.apache.spark.{Partition, SparkConf, SparkContext, TaskContext} diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/execution/WindowExecTransformer.scala b/gluten-substrait/src/main/scala/org/apache/gluten/execution/WindowExecTransformer.scala index 28d780992492..7bdb29f0ee95 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/execution/WindowExecTransformer.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/execution/WindowExecTransformer.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.expression._ import org.apache.gluten.extension.ValidationResult import org.apache.gluten.metrics.MetricsUpdater @@ -88,7 +88,7 @@ case class WindowExecTransformer( val windowParametersStr = new StringBuffer("WindowParameters:") // isStreaming: 1 for streaming, 0 for sort val isStreaming: Int = - if (GlutenConfig.getConf.veloxColumnarWindowType.equals("streaming")) 1 else 0 + if (GlutenConfig.get.veloxColumnarWindowType.equals("streaming")) 1 else 0 windowParametersStr .append("isStreaming=") diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/execution/WriteFilesExecTransformer.scala b/gluten-substrait/src/main/scala/org/apache/gluten/execution/WriteFilesExecTransformer.scala index 726dbdc3ef30..b5cfb617b2b6 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/execution/WriteFilesExecTransformer.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/execution/WriteFilesExecTransformer.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.expression.ConverterUtils import org.apache.gluten.extension.ValidationResult import org.apache.gluten.metrics.MetricsUpdater @@ -118,7 +118,7 @@ case class WriteFilesExecTransformer( } private def getFinalChildOutput: Seq[Attribute] = { - val metadataExclusionList = conf + val metadataExclusionList = GlutenConfig.get .getConf(GlutenConfig.NATIVE_WRITE_FILES_COLUMN_METADATA_EXCLUSION_LIST) .split(",") .map(_.trim) diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala b/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala index f6cc01f8ac83..f9b5a2446171 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.expression -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.expression.ExpressionNames._ import org.apache.gluten.sql.shims.SparkShimLoader @@ -339,7 +339,7 @@ object ExpressionMappings { ) def expressionsMap: Map[Class[_], String] = { - val blacklist = GlutenConfig.getConf.expressionBlacklist + val blacklist = GlutenConfig.get.expressionBlacklist val filtered = (defaultExpressionsMap ++ toMap( BackendsApiManager.getSparkPlanExecApiInstance.extraExpressionMappings)).filterNot( kv => blacklist.contains(kv._2)) diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/expression/UDFMappings.scala b/gluten-substrait/src/main/scala/org/apache/gluten/expression/UDFMappings.scala index 3b64c5117fbb..948f6f4d163f 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/expression/UDFMappings.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/expression/UDFMappings.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.expression -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.SparkConf import org.apache.spark.internal.Logging diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/CollapseProjectExecTransformer.scala b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/CollapseProjectExecTransformer.scala index bfb926706cf7..39c1d650aef2 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/CollapseProjectExecTransformer.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/CollapseProjectExecTransformer.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.extension.columnar -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.ProjectExecTransformer import org.apache.spark.sql.catalyst.expressions.{Alias, CreateNamedStruct, NamedExpression} @@ -27,7 +27,7 @@ import org.apache.spark.sql.execution.SparkPlan object CollapseProjectExecTransformer extends Rule[SparkPlan] { override def apply(plan: SparkPlan): SparkPlan = { - if (!GlutenConfig.getConf.enableColumnarProjectCollapse) { + if (!GlutenConfig.get.enableColumnarProjectCollapse) { return plan } plan.transformUp { diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/FallbackRules.scala b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/FallbackRules.scala index 432ecd1584d8..8c8bcaf12855 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/FallbackRules.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/FallbackRules.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.extension.columnar -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.rules.Rule @@ -27,7 +27,7 @@ import org.apache.spark.sql.execution.joins._ case class FallbackOnANSIMode(session: SparkSession) extends Rule[SparkPlan] { override def apply(plan: SparkPlan): SparkPlan = { - if (GlutenConfig.getConf.enableAnsiMode) { + if (GlutenConfig.get.enableAnsiMode) { plan.foreach(FallbackTags.add(_, "does not support ansi mode")) } plan @@ -35,7 +35,7 @@ case class FallbackOnANSIMode(session: SparkSession) extends Rule[SparkPlan] { } case class FallbackMultiCodegens(session: SparkSession) extends Rule[SparkPlan] { - lazy val glutenConf: GlutenConfig = GlutenConfig.getConf + lazy val glutenConf: GlutenConfig = GlutenConfig.get lazy val physicalJoinOptimize = glutenConf.enablePhysicalJoinOptimize lazy val optimizeLevel: Integer = glutenConf.physicalJoinOptimizationThrottle @@ -47,7 +47,7 @@ case class FallbackMultiCodegens(session: SparkSession) extends Rule[SparkPlan] case plan: ShuffledHashJoinExec => if ((count + 1) >= optimizeLevel) return true plan.children.exists(existsMultiCodegens(_, count + 1)) - case plan: SortMergeJoinExec if GlutenConfig.getConf.forceShuffledHashJoin => + case plan: SortMergeJoinExec if GlutenConfig.get.forceShuffledHashJoin => if ((count + 1) >= optimizeLevel) return true plan.children.exists(existsMultiCodegens(_, count + 1)) case _ => false diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/MergeTwoPhasesHashBaseAggregate.scala b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/MergeTwoPhasesHashBaseAggregate.scala index a034a3229a88..16d4c0ebe644 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/MergeTwoPhasesHashBaseAggregate.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/MergeTwoPhasesHashBaseAggregate.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.extension.columnar -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.internal.Logging import org.apache.spark.sql.SparkSession @@ -38,11 +38,11 @@ case class MergeTwoPhasesHashBaseAggregate(session: SparkSession) extends Rule[SparkPlan] with Logging { - val glutenConf: GlutenConfig = GlutenConfig.getConf + val glutenConf: GlutenConfig = GlutenConfig.get val scanOnly: Boolean = glutenConf.enableScanOnly val enableColumnarHashAgg: Boolean = !scanOnly && glutenConf.enableColumnarHashAgg - val replaceSortAggWithHashAgg: Boolean = GlutenConfig.getConf.forceToUseHashAgg - val mergeTwoPhasesAggEnabled: Boolean = GlutenConfig.getConf.mergeTwoPhasesAggEnabled + val replaceSortAggWithHashAgg: Boolean = GlutenConfig.get.forceToUseHashAgg + val mergeTwoPhasesAggEnabled: Boolean = GlutenConfig.get.mergeTwoPhasesAggEnabled private def isPartialAgg(partialAgg: BaseAggregateExec, finalAgg: BaseAggregateExec): Boolean = { // TODO: now it can not support to merge agg which there are the filters in the aggregate exprs. diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/RemoveNativeWriteFilesSortAndProject.scala b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/RemoveNativeWriteFilesSortAndProject.scala index ac35ac83bb21..7fce082f8a95 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/RemoveNativeWriteFilesSortAndProject.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/RemoveNativeWriteFilesSortAndProject.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.extension.columnar -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{ProjectExecTransformer, SortExecTransformer, WriteFilesExecTransformer} import org.apache.spark.internal.Logging @@ -32,7 +32,7 @@ import org.apache.spark.sql.execution.{ProjectExec, SortExec, SparkPlan} */ case class RemoveNativeWriteFilesSortAndProject() extends Rule[SparkPlan] { override def apply(plan: SparkPlan): SparkPlan = { - if (!conf.getConf(GlutenConfig.REMOVE_NATIVE_WRITE_FILES_SORT_AND_PROJECT)) { + if (!GlutenConfig.get.getConf(GlutenConfig.REMOVE_NATIVE_WRITE_FILES_SORT_AND_PROJECT)) { return plan } diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/UnionTransformerRule.scala b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/UnionTransformerRule.scala index f0eea08018dd..e90ae133bcf0 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/UnionTransformerRule.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/UnionTransformerRule.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.extension.columnar -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{ColumnarUnionExec, UnionExecTransformer} import org.apache.spark.sql.catalyst.plans.physical.UnknownPartitioning @@ -32,7 +32,7 @@ import org.apache.spark.sql.execution.SparkPlan */ case class UnionTransformerRule() extends Rule[SparkPlan] { override def apply(plan: SparkPlan): SparkPlan = { - if (!GlutenConfig.getConf.enableNativeUnion) { + if (!GlutenConfig.get.enableNativeUnion) { return plan } plan.transformUp { diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/enumerated/planner/cost/RoughCoster2.scala b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/enumerated/planner/cost/RoughCoster2.scala index 832c524d7b55..e46274a79f69 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/enumerated/planner/cost/RoughCoster2.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/enumerated/planner/cost/RoughCoster2.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.extension.columnar.enumerated.planner.cost -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.extension.columnar.transition.{ColumnarToRowLike, RowToColumnarLike} import org.apache.gluten.utils.PlanUtil @@ -45,11 +45,11 @@ object RoughCoster2 extends LongCoster { case ColumnarToRowLike(_) => 1L case RowToColumnarLike(_) => // If sizeBytes is less than the threshold, the cost of RowToColumnarLike is ignored. - if (sizeFactor == 0) 1L else GlutenConfig.getConf.rasRough2R2cCost + if (sizeFactor == 0) 1L else GlutenConfig.get.rasRough2R2cCost case p if PlanUtil.isGlutenColumnarOp(p) => 1L - case p if PlanUtil.isVanillaColumnarOp(p) => GlutenConfig.getConf.rasRough2VanillaCost + case p if PlanUtil.isVanillaColumnarOp(p) => GlutenConfig.get.rasRough2VanillaCost // Other row ops. Usually a vanilla row op. - case _ => GlutenConfig.getConf.rasRough2VanillaCost + case _ => GlutenConfig.get.rasRough2VanillaCost } opCost * Math.max(1, sizeFactor) } @@ -61,7 +61,7 @@ object RoughCoster2 extends LongCoster { case _: LeafExecNode => 0L case p => p.children.map(getStatSizeBytes).sum } - sizeBytes / GlutenConfig.getConf.rasRough2SizeBytesThreshold + sizeBytes / GlutenConfig.get.rasRough2SizeBytesThreshold } private def getStatSizeBytes(plan: SparkPlan): Long = { diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/heuristic/ExpandFallbackPolicy.scala b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/heuristic/ExpandFallbackPolicy.scala index 3418d3dddc99..6c9a0ee8977a 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/heuristic/ExpandFallbackPolicy.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/heuristic/ExpandFallbackPolicy.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.extension.columnar.heuristic -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.GlutenPlan import org.apache.gluten.extension.columnar.{FallbackTag, FallbackTags} import org.apache.gluten.extension.columnar.FallbackTags.add @@ -74,7 +74,7 @@ case class ExpandFallbackPolicy(isAdaptiveContext: Boolean, originalPlan: SparkP import ExpandFallbackPolicy._ private def countTransitionCost(plan: SparkPlan): Int = { - val ignoreRowToColumnar = GlutenConfig.getConf.fallbackIgnoreRowToColumnar + val ignoreRowToColumnar = GlutenConfig.get.fallbackIgnoreRowToColumnar var transitionCost = 0 def countFallbackInternal(plan: SparkPlan): Unit = { plan match { @@ -183,7 +183,7 @@ case class ExpandFallbackPolicy(isAdaptiveContext: Boolean, originalPlan: SparkP private def fallback(plan: SparkPlan): FallbackInfo = { val fallbackThreshold = if (isAdaptiveContext) { - GlutenConfig.getConf.wholeStageFallbackThreshold + GlutenConfig.get.wholeStageFallbackThreshold } else if (plan.find(_.isInstanceOf[AdaptiveSparkPlanExec]).isDefined) { // if we are here, that means we are now at `QueryExecution.preparations` and // AQE is actually not applied. We do nothing for this case, and later in @@ -191,7 +191,7 @@ case class ExpandFallbackPolicy(isAdaptiveContext: Boolean, originalPlan: SparkP return FallbackInfo.DO_NOT_FALLBACK() } else { // AQE is not applied, so we use the whole query threshold to check if should fallback - GlutenConfig.getConf.queryFallbackThreshold + GlutenConfig.get.queryFallbackThreshold } if (fallbackThreshold < 0) { return FallbackInfo.DO_NOT_FALLBACK() @@ -255,7 +255,7 @@ case class ExpandFallbackPolicy(isAdaptiveContext: Boolean, originalPlan: SparkP val vanillaSparkPlan = fallbackToRowBasedPlan(plan, outputsColumnar) val vanillaSparkTransitionCost = countTransitionCostForVanillaSparkPlan(vanillaSparkPlan) if ( - GlutenConfig.getConf.fallbackPreferColumnar && + GlutenConfig.get.fallbackPreferColumnar && fallbackInfo.netTransitionCost <= vanillaSparkTransitionCost ) { plan diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/offload/OffloadSingleNodeRules.scala b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/offload/OffloadSingleNodeRules.scala index 7f258d10b2fc..8d0abcaa686a 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/offload/OffloadSingleNodeRules.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/offload/OffloadSingleNodeRules.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.extension.columnar.offload -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution._ import org.apache.gluten.extension.columnar.FallbackTags import org.apache.gluten.logging.LogLevelUtil @@ -43,7 +43,7 @@ case class OffloadExchange() extends OffloadSingleNode with LogLevelUtil { case p if FallbackTags.nonEmpty(p) => p case s: ShuffleExchangeExec - if (s.child.supportsColumnar || GlutenConfig.getConf.enablePreferColumnar) && + if (s.child.supportsColumnar || GlutenConfig.get.enablePreferColumnar) && BackendsApiManager.getSettings.supportColumnarShuffleExec() => logDebug(s"Columnar Processing for ${s.getClass} is currently supported.") BackendsApiManager.getSparkPlanExecApiInstance.genColumnarShuffleExchange(s) @@ -140,7 +140,7 @@ object OffloadJoin { } // Both left and right are buildable. Find out the better one. - if (!GlutenConfig.getConf.shuffledHashJoinOptimizeBuildSide) { + if (!GlutenConfig.get.shuffledHashJoinOptimizeBuildSide) { // User disabled build side re-optimization. Return original build side from vanilla Spark. return shj.buildSide } @@ -314,7 +314,7 @@ object OffloadOthers { // Velox backend uses ColumnarArrowEvalPythonExec. if ( !BackendsApiManager.getSettings.supportColumnarArrowUdf() || - !GlutenConfig.getConf.enableColumnarArrowUDF + !GlutenConfig.get.enableColumnarArrowUDF ) { EvalPythonExecTransformer(plan.udfs, plan.resultAttrs, child) } else { diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteJoin.scala b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteJoin.scala index 2d844a616c5b..42bbbba39d02 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteJoin.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/rewrite/RewriteJoin.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.extension.columnar.rewrite -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.extension.columnar.offload.OffloadJoin import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide, JoinSelectionHelper} @@ -56,7 +56,7 @@ object RewriteJoin extends RewriteSingleNode with JoinSelectionHelper { } override def rewrite(plan: SparkPlan): SparkPlan = plan match { - case smj: SortMergeJoinExec if GlutenConfig.getConf.forceShuffledHashJoin => + case smj: SortMergeJoinExec if GlutenConfig.get.forceShuffledHashJoin => getSmjBuildSide(smj) match { case Some(buildSide) => ShuffledHashJoinExec( diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala index 93c444fbd01b..509d7c02ae19 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/validator/Validators.scala @@ -16,8 +16,8 @@ */ package org.apache.gluten.extension.columnar.validator -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.{BackendsApiManager, BackendSettingsApi} +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution._ import org.apache.gluten.expression.ExpressionUtils import org.apache.gluten.extension.columnar.FallbackTags @@ -37,7 +37,7 @@ import org.apache.spark.sql.hive.HiveTableScanExecTransformer object Validators { implicit class ValidatorBuilderImplicits(builder: Validator.Builder) { - private val conf = GlutenConfig.getConf + private val conf = GlutenConfig.get private val settings = BackendsApiManager.getSettings /** Fails validation if a plan node was already tagged with TRANSFORM_UNSUPPORTED. */ diff --git a/gluten-substrait/src/main/scala/org/apache/spark/shuffle/GlutenShuffleUtils.scala b/gluten-substrait/src/main/scala/org/apache/spark/shuffle/GlutenShuffleUtils.scala index e3e374256ff4..8278c0dfa703 100644 --- a/gluten-substrait/src/main/scala/org/apache/spark/shuffle/GlutenShuffleUtils.scala +++ b/gluten-substrait/src/main/scala/org/apache/spark/shuffle/GlutenShuffleUtils.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.shuffle -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.gluten.vectorized.NativePartitioning @@ -51,7 +51,7 @@ object GlutenShuffleUtils { s"${validValues.mkString(", ")}, but was $codec") } } - val glutenConfig = GlutenConfig.getConf + val glutenConfig = GlutenConfig.get glutenConfig.columnarShuffleCodec match { case Some(codec) => val glutenCodecKey = GlutenConfig.COLUMNAR_SHUFFLE_CODEC.key diff --git a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseTransformStages.scala b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseTransformStages.scala index 2bdc1cf4b371..82b75a053f32 100644 --- a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseTransformStages.scala +++ b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarCollapseTransformStages.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.execution -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution._ import org.apache.gluten.extension.columnar.transition.{Convention, ConventionReq} import org.apache.gluten.metrics.MetricsUpdater diff --git a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExec.scala b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExec.scala index 007a93bd06a1..70cc76475408 100644 --- a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExec.scala +++ b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/ColumnarShuffleExchangeExec.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.execution -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.ValidatablePlan import org.apache.gluten.extension.ValidationResult import org.apache.gluten.extension.columnar.transition.Convention diff --git a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/GlutenFallbackReporter.scala b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/GlutenFallbackReporter.scala index c28395941b0c..ce69c2281088 100644 --- a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/GlutenFallbackReporter.scala +++ b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/GlutenFallbackReporter.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.execution -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.events.GlutenPlanFallbackEvent import org.apache.gluten.execution.GlutenPlan import org.apache.gluten.extension.columnar.FallbackTags diff --git a/gluten-substrait/src/main/scala/org/apache/spark/sql/hive/HiveTableScanNestedColumnPruning.scala b/gluten-substrait/src/main/scala/org/apache/spark/sql/hive/HiveTableScanNestedColumnPruning.scala index 70b08a2ed8bb..b00dc997969b 100644 --- a/gluten-substrait/src/main/scala/org/apache/spark/sql/hive/HiveTableScanNestedColumnPruning.scala +++ b/gluten-substrait/src/main/scala/org/apache/spark/sql/hive/HiveTableScanNestedColumnPruning.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.hive -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.ProjectExecTransformer import org.apache.spark.internal.Logging @@ -32,7 +32,7 @@ object HiveTableScanNestedColumnPruning extends Logging { import org.apache.spark.sql.catalyst.expressions.SchemaPruning._ def supportNestedColumnPruning(project: ProjectExecTransformer): Boolean = { - if (GlutenConfig.getConf.enableColumnarHiveTableScanNestedColumnPruning) { + if (GlutenConfig.get.enableColumnarHiveTableScanNestedColumnPruning) { project.child match { case HiveTableScanExecTransformer(_, relation, _, _) => relation.tableMeta.storage.inputFormat match { diff --git a/gluten-substrait/src/main/scala/org/apache/spark/util/ExecutorManager.scala b/gluten-substrait/src/main/scala/org/apache/spark/util/ExecutorManager.scala index 049bcf13c032..4e55aff0d37e 100644 --- a/gluten-substrait/src/main/scala/org/apache/spark/util/ExecutorManager.scala +++ b/gluten-substrait/src/main/scala/org/apache/spark/util/ExecutorManager.scala @@ -16,9 +16,9 @@ */ package org.apache.spark.util -import org.apache.spark.{SparkContext, SparkEnv} +import org.apache.gluten.config.GlutenNumaBindingInfo -import _root_.org.apache.gluten._ +import org.apache.spark.{SparkContext, SparkEnv} import java.lang.management.ManagementFactory diff --git a/gluten-substrait/src/test/scala/org/apache/gluten/execution/WholeStageTransformerSuite.scala b/gluten-substrait/src/test/scala/org/apache/gluten/execution/WholeStageTransformerSuite.scala index 08081fadb5f9..dd16892017e4 100644 --- a/gluten-substrait/src/test/scala/org/apache/gluten/execution/WholeStageTransformerSuite.scala +++ b/gluten-substrait/src/test/scala/org/apache/gluten/execution/WholeStageTransformerSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.test.FallbackUtil import org.apache.gluten.utils.Arm diff --git a/gluten-substrait/src/test/scala/org/apache/spark/softaffinity/SoftAffinitySuite.scala b/gluten-substrait/src/test/scala/org/apache/spark/softaffinity/SoftAffinitySuite.scala index 4145fd5bc6b2..caca7ca9e36d 100644 --- a/gluten-substrait/src/test/scala/org/apache/spark/softaffinity/SoftAffinitySuite.scala +++ b/gluten-substrait/src/test/scala/org/apache/spark/softaffinity/SoftAffinitySuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.softaffinity -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.GlutenPartition import org.apache.gluten.softaffinity.SoftAffinityManager import org.apache.gluten.sql.shims.SparkShimLoader diff --git a/gluten-substrait/src/test/scala/org/apache/spark/softaffinity/SoftAffinityWithRDDInfoSuite.scala b/gluten-substrait/src/test/scala/org/apache/spark/softaffinity/SoftAffinityWithRDDInfoSuite.scala index c91df0b6d8ac..2100518e741c 100644 --- a/gluten-substrait/src/test/scala/org/apache/spark/softaffinity/SoftAffinityWithRDDInfoSuite.scala +++ b/gluten-substrait/src/test/scala/org/apache/spark/softaffinity/SoftAffinityWithRDDInfoSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.softaffinity -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.GlutenPartition import org.apache.gluten.softaffinity.{AffinityManager, SoftAffinityManager} import org.apache.gluten.sql.shims.SparkShimLoader @@ -49,7 +49,7 @@ class SoftAffinityWithRDDInfoSuite extends QueryTest with SharedSparkSession wit .set(GlutenConfig.GLUTEN_SOFT_AFFINITY_DUPLICATE_READING_DETECT_ENABLED, "true") .set(GlutenConfig.GLUTEN_SOFT_AFFINITY_REPLICATIONS_NUM, "2") .set(GlutenConfig.GLUTEN_SOFT_AFFINITY_MIN_TARGET_HOSTS, "2") - .set(GlutenConfig.SOFT_AFFINITY_LOG_LEVEL, "INFO") + .set(GlutenConfig.SOFT_AFFINITY_LOG_LEVEL.key, "INFO") test("Soft Affinity Scheduler with duplicate reading detection") { if (SparkShimLoader.getSparkShims.supportDuplicateReadingTracking) { diff --git a/gluten-ut/common/src/test/scala/org/apache/spark/sql/DummyFilterColmnarHelper.scala b/gluten-ut/common/src/test/scala/org/apache/spark/sql/DummyFilterColmnarHelper.scala index 7208d8097cd6..6605a942cc09 100644 --- a/gluten-ut/common/src/test/scala/org/apache/spark/sql/DummyFilterColmnarHelper.scala +++ b/gluten-ut/common/src/test/scala/org/apache/spark/sql/DummyFilterColmnarHelper.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} import org.apache.spark.rdd.RDD diff --git a/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenSQLTestsBaseTrait.scala b/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenSQLTestsBaseTrait.scala index 4c06b02a1fb4..a16b459f8e46 100644 --- a/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenSQLTestsBaseTrait.scala +++ b/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenSQLTestsBaseTrait.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} import org.apache.spark.SparkConf diff --git a/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsTrait.scala b/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsTrait.scala index 5df9d007193e..9a975812f0ba 100644 --- a/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsTrait.scala +++ b/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsTrait.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.ProjectExecTransformer import org.apache.gluten.test.TestStats import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala index 1e34d2c8199d..e2fa10f943d8 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{BatchScanExecTransformer, FileSourceScanExecTransformer, FilterExecTransformerBase} import org.apache.gluten.utils.BackendTestUtils diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala index fec513349d5f..6d1744423fbc 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.exception.GlutenException import org.apache.gluten.utils.{BackendTestSettings, BackendTestUtils, SystemParameters} diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenSparkSessionExtensionSuite.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenSparkSessionExtensionSuite.scala index 5e3fed736697..ae9b3901afb9 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenSparkSessionExtensionSuite.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenSparkSessionExtensionSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig class GlutenSparkSessionExtensionSuite extends SparkSessionExtensionSuite @@ -31,8 +31,10 @@ class GlutenSparkSessionExtensionSuite try { session.range(2).write.format("parquet").mode("overwrite").saveAsTable("a") def testWithFallbackSettings(scanFallback: Boolean, aggFallback: Boolean): Unit = { - session.sessionState.conf.setConf(GlutenConfig.COLUMNAR_FILESCAN_ENABLED, scanFallback) - session.sessionState.conf.setConf(GlutenConfig.COLUMNAR_HASHAGG_ENABLED, aggFallback) + session.sessionState.conf + .setConfString(GlutenConfig.COLUMNAR_FILESCAN_ENABLED.key, scanFallback.toString) + session.sessionState.conf + .setConfString(GlutenConfig.COLUMNAR_HASHAGG_ENABLED.key, aggFallback.toString) val df = session.sql("SELECT max(id) FROM a") val newDf = DummyFilterColmnarHelper.dfWithDummyFilterColumnar( session, diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala index 754c29241864..97a779289d1f 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.execution -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{BasicScanExecTransformer, GlutenPlan} import org.apache.gluten.extension.GlutenSessionExtensions import org.apache.gluten.extension.columnar.ColumnarRuleApplier.ColumnarRuleCall @@ -172,7 +172,7 @@ private object FallbackStrategiesSuite { List(c => ExpandFallbackPolicy(c.ac.isAdaptiveContext(), c.ac.originalPlan())), List( c => RemoveTopmostColumnarToRow(c.session, c.ac.isAdaptiveContext()), - _ => ColumnarCollapseTransformStages(GlutenConfig.getConf) + _ => ColumnarCollapseTransformStages(GlutenConfig.get) ), List(_ => RemoveFallbackTagRule()) ) diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala index 3db750dc20a4..de4dce405874 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.execution.benchmarks -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{FileSourceScanExecTransformer, WholeStageTransformer} import org.apache.gluten.extension.columnar.transition.Transitions import org.apache.gluten.jni.JniLibLoader diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenWriterColumnarRulesSuite.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenWriterColumnarRulesSuite.scala index a5d3f0e3f183..10abca1c6dd3 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenWriterColumnarRulesSuite.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenWriterColumnarRulesSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.execution.datasources -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.sql.{GlutenSQLTestsBaseTrait, SaveMode} import org.apache.spark.sql.execution.QueryExecution diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala index f418ec06645c..f516f71d69c8 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.execution.joins -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{BroadcastHashJoinExecTransformerBase, BroadcastNestedLoopJoinExecTransformer, ColumnarToRowExecBase, WholeStageTransformer} import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/extension/GlutenCollapseProjectExecTransformerSuite.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/extension/GlutenCollapseProjectExecTransformerSuite.scala index 5d5f4d2701a7..87a4ce14d4a2 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/extension/GlutenCollapseProjectExecTransformerSuite.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/extension/GlutenCollapseProjectExecTransformerSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.extension -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.ProjectExecTransformer import org.apache.spark.sql.GlutenSQLTestsTrait diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala index d08b2837a599..a8eb2361ab3e 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.extension -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.SparkConf import org.apache.spark.sql.GlutenSQLTestsTrait diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala index e7d573ca5e7d..56ec103349e7 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.hive.execution -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.utils.SystemParameters import org.apache.spark.{DebugFilesystem, SparkConf} diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala index e525eb1a9c01..ebfedfed6235 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.statistics -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.GlutenPlan import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenBloomFilterAggregateQuerySuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenBloomFilterAggregateQuerySuite.scala index ddd4cf1d4c31..e70f2153293f 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenBloomFilterAggregateQuerySuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenBloomFilterAggregateQuerySuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.HashAggregateExecBaseTransformer import org.apache.spark.SparkConf @@ -37,7 +37,7 @@ class GlutenBloomFilterAggregateQuerySuite GlutenConfig.COLUMNAR_VELOX_BLOOM_FILTER_MAX_NUM_BITS.key -> "4194304" ) { val numEstimatedItems = 5000000L - val numBits = GlutenConfig.getConf.veloxBloomFilterMaxNumBits + val numBits = GlutenConfig.get.veloxBloomFilterMaxNumBits val sqlString = s""" |SELECT every(might_contain( | (SELECT bloom_filter_agg(col, @@ -72,7 +72,7 @@ class GlutenBloomFilterAggregateQuerySuite testGluten("Test bloom_filter_agg filter fallback") { val table = "bloom_filter_test" val numEstimatedItems = 5000000L - val numBits = GlutenConfig.getConf.veloxBloomFilterMaxNumBits + val numBits = GlutenConfig.get.veloxBloomFilterMaxNumBits val sqlString = s""" |SELECT col positive_membership_test |FROM $table @@ -118,7 +118,7 @@ class GlutenBloomFilterAggregateQuerySuite testGluten("Test bloom_filter_agg agg fallback") { val table = "bloom_filter_test" val numEstimatedItems = 5000000L - val numBits = GlutenConfig.getConf.veloxBloomFilterMaxNumBits + val numBits = GlutenConfig.get.veloxBloomFilterMaxNumBits val sqlString = s""" |SELECT col positive_membership_test |FROM $table diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala index e146c1310db7..f8b61bed484e 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.SparkConf import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala index a08d3eff3bf2..947d9368c879 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{BatchScanExecTransformer, FileSourceScanExecTransformer, FilterExecTransformerBase} import org.apache.gluten.utils.BackendTestUtils diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala index 39fe8cf57a18..893acc650155 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.exception.GlutenException import org.apache.gluten.utils.{BackendTestSettings, BackendTestUtils, SystemParameters} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenSparkSessionExtensionSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenSparkSessionExtensionSuite.scala index 5e3fed736697..ae9b3901afb9 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenSparkSessionExtensionSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenSparkSessionExtensionSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig class GlutenSparkSessionExtensionSuite extends SparkSessionExtensionSuite @@ -31,8 +31,10 @@ class GlutenSparkSessionExtensionSuite try { session.range(2).write.format("parquet").mode("overwrite").saveAsTable("a") def testWithFallbackSettings(scanFallback: Boolean, aggFallback: Boolean): Unit = { - session.sessionState.conf.setConf(GlutenConfig.COLUMNAR_FILESCAN_ENABLED, scanFallback) - session.sessionState.conf.setConf(GlutenConfig.COLUMNAR_HASHAGG_ENABLED, aggFallback) + session.sessionState.conf + .setConfString(GlutenConfig.COLUMNAR_FILESCAN_ENABLED.key, scanFallback.toString) + session.sessionState.conf + .setConfString(GlutenConfig.COLUMNAR_HASHAGG_ENABLED.key, aggFallback.toString) val df = session.sql("SELECT max(id) FROM a") val newDf = DummyFilterColmnarHelper.dfWithDummyFilterColumnar( session, diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala index 88e0ecf65a9a..8b68c36bb3fe 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.execution -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{BasicScanExecTransformer, GlutenPlan} import org.apache.gluten.extension.GlutenSessionExtensions import org.apache.gluten.extension.columnar.{FallbackTags, RemoveFallbackTagRule} @@ -182,7 +182,7 @@ private object FallbackStrategiesSuite { List(c => ExpandFallbackPolicy(c.ac.isAdaptiveContext(), c.ac.originalPlan())), List( c => RemoveTopmostColumnarToRow(c.session, c.ac.isAdaptiveContext()), - _ => ColumnarCollapseTransformStages(GlutenConfig.getConf) + _ => ColumnarCollapseTransformStages(GlutenConfig.get) ), List(_ => RemoveFallbackTagRule()) ) diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala index 706c30d67db8..85aa24a56392 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.execution.benchmarks -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{FileSourceScanExecTransformer, WholeStageTransformer} import org.apache.gluten.extension.columnar.transition.Transitions import org.apache.gluten.jni.JniLibLoader diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenWriterColumnarRulesSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenWriterColumnarRulesSuite.scala index a5d3f0e3f183..10abca1c6dd3 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenWriterColumnarRulesSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenWriterColumnarRulesSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.execution.datasources -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.sql.{GlutenSQLTestsBaseTrait, SaveMode} import org.apache.spark.sql.execution.QueryExecution diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala index 13317444c1d3..f68000fd0bea 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.execution.joins -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} import org.apache.spark.sql.{GlutenTestsCommonTrait, SparkSession} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/GlutenCollapseProjectExecTransformerSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/GlutenCollapseProjectExecTransformerSuite.scala index 5d5f4d2701a7..87a4ce14d4a2 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/GlutenCollapseProjectExecTransformerSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/GlutenCollapseProjectExecTransformerSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.extension -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.ProjectExecTransformer import org.apache.spark.sql.GlutenSQLTestsTrait diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala index d08b2837a599..a8eb2361ab3e 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.extension -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.SparkConf import org.apache.spark.sql.GlutenSQLTestsTrait diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala index 4a73a8dc72de..15fb9aced217 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala @@ -16,7 +16,8 @@ */ package org.apache.spark.sql.gluten -import org.apache.gluten.{GlutenBuildInfo, GlutenConfig} +import org.apache.gluten.GlutenBuildInfo +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.events.GlutenPlanFallbackEvent import org.apache.gluten.execution.FileSourceScanExecTransformer import org.apache.gluten.utils.BackendTestUtils diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala index e7d573ca5e7d..56ec103349e7 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.hive.execution -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.utils.SystemParameters import org.apache.spark.{DebugFilesystem, SparkConf} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala index 34e97273d603..eabe7a479292 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.statistics -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.GlutenPlan import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenBloomFilterAggregateQuerySuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenBloomFilterAggregateQuerySuite.scala index 0c75db830830..2efdfd19ee8a 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenBloomFilterAggregateQuerySuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenBloomFilterAggregateQuerySuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.HashAggregateExecBaseTransformer import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper @@ -36,7 +36,7 @@ class GlutenBloomFilterAggregateQuerySuite GlutenConfig.COLUMNAR_VELOX_BLOOM_FILTER_MAX_NUM_BITS.key -> "4194304" ) { val numEstimatedItems = 5000000L - val numBits = GlutenConfig.getConf.veloxBloomFilterMaxNumBits + val numBits = GlutenConfig.get.veloxBloomFilterMaxNumBits val sqlString = s""" |SELECT every(might_contain( | (SELECT bloom_filter_agg(col, @@ -71,7 +71,7 @@ class GlutenBloomFilterAggregateQuerySuite testGluten("Test bloom_filter_agg filter fallback") { val table = "bloom_filter_test" val numEstimatedItems = 5000000L - val numBits = GlutenConfig.getConf.veloxBloomFilterMaxNumBits + val numBits = GlutenConfig.get.veloxBloomFilterMaxNumBits val sqlString = s""" |SELECT col positive_membership_test |FROM $table @@ -117,7 +117,7 @@ class GlutenBloomFilterAggregateQuerySuite testGluten("Test bloom_filter_agg agg fallback") { val table = "bloom_filter_test" val numEstimatedItems = 5000000L - val numBits = GlutenConfig.getConf.veloxBloomFilterMaxNumBits + val numBits = GlutenConfig.get.veloxBloomFilterMaxNumBits val sqlString = s""" |SELECT col positive_membership_test |FROM $table diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala index e146c1310db7..f8b61bed484e 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.SparkConf import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala index 559aaada4a8c..c27b2064486a 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{BatchScanExecTransformer, FileSourceScanExecTransformer, FilterExecTransformerBase} import org.apache.spark.SparkConf diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala index 65578eb52bca..740f9c1595cb 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.exception.GlutenException import org.apache.gluten.utils.{BackendTestSettings, BackendTestUtils, SystemParameters} diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenSparkSessionExtensionSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenSparkSessionExtensionSuite.scala index 5e3fed736697..ae9b3901afb9 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenSparkSessionExtensionSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenSparkSessionExtensionSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig class GlutenSparkSessionExtensionSuite extends SparkSessionExtensionSuite @@ -31,8 +31,10 @@ class GlutenSparkSessionExtensionSuite try { session.range(2).write.format("parquet").mode("overwrite").saveAsTable("a") def testWithFallbackSettings(scanFallback: Boolean, aggFallback: Boolean): Unit = { - session.sessionState.conf.setConf(GlutenConfig.COLUMNAR_FILESCAN_ENABLED, scanFallback) - session.sessionState.conf.setConf(GlutenConfig.COLUMNAR_HASHAGG_ENABLED, aggFallback) + session.sessionState.conf + .setConfString(GlutenConfig.COLUMNAR_FILESCAN_ENABLED.key, scanFallback.toString) + session.sessionState.conf + .setConfString(GlutenConfig.COLUMNAR_HASHAGG_ENABLED.key, aggFallback.toString) val df = session.sql("SELECT max(id) FROM a") val newDf = DummyFilterColmnarHelper.dfWithDummyFilterColumnar( session, diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala index 88e0ecf65a9a..8b68c36bb3fe 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.execution -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{BasicScanExecTransformer, GlutenPlan} import org.apache.gluten.extension.GlutenSessionExtensions import org.apache.gluten.extension.columnar.{FallbackTags, RemoveFallbackTagRule} @@ -182,7 +182,7 @@ private object FallbackStrategiesSuite { List(c => ExpandFallbackPolicy(c.ac.isAdaptiveContext(), c.ac.originalPlan())), List( c => RemoveTopmostColumnarToRow(c.session, c.ac.isAdaptiveContext()), - _ => ColumnarCollapseTransformStages(GlutenConfig.getConf) + _ => ColumnarCollapseTransformStages(GlutenConfig.get) ), List(_ => RemoveFallbackTagRule()) ) diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala index cdfb63aa2074..8ddd85a0f232 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.execution.benchmarks -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{FileSourceScanExecTransformer, WholeStageTransformer} import org.apache.gluten.extension.columnar.transition.Transitions import org.apache.gluten.jni.JniLibLoader diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala index 207f316dd8c2..20374382debb 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.execution.joins -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} import org.apache.spark.sql.{GlutenTestsCommonTrait, SparkSession} diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/extension/GlutenCollapseProjectExecTransformerSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/extension/GlutenCollapseProjectExecTransformerSuite.scala index 5d5f4d2701a7..87a4ce14d4a2 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/extension/GlutenCollapseProjectExecTransformerSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/extension/GlutenCollapseProjectExecTransformerSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.extension -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.ProjectExecTransformer import org.apache.spark.sql.GlutenSQLTestsTrait diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala index d08b2837a599..a8eb2361ab3e 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.extension -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.SparkConf import org.apache.spark.sql.GlutenSQLTestsTrait diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala index 7855f289a707..3fe46bcdec00 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala @@ -16,7 +16,8 @@ */ package org.apache.spark.sql.gluten -import org.apache.gluten.{GlutenBuildInfo, GlutenConfig} +import org.apache.gluten.GlutenBuildInfo +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.events.GlutenPlanFallbackEvent import org.apache.gluten.execution.FileSourceScanExecTransformer import org.apache.gluten.utils.BackendTestUtils diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala index e7d573ca5e7d..56ec103349e7 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.hive.execution -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.utils.SystemParameters import org.apache.spark.{DebugFilesystem, SparkConf} diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala index f7cc114859cd..6a7fe611ee82 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.statistics -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.GlutenPlan import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenBloomFilterAggregateQuerySuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenBloomFilterAggregateQuerySuite.scala index ddd4cf1d4c31..e70f2153293f 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenBloomFilterAggregateQuerySuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenBloomFilterAggregateQuerySuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.HashAggregateExecBaseTransformer import org.apache.spark.SparkConf @@ -37,7 +37,7 @@ class GlutenBloomFilterAggregateQuerySuite GlutenConfig.COLUMNAR_VELOX_BLOOM_FILTER_MAX_NUM_BITS.key -> "4194304" ) { val numEstimatedItems = 5000000L - val numBits = GlutenConfig.getConf.veloxBloomFilterMaxNumBits + val numBits = GlutenConfig.get.veloxBloomFilterMaxNumBits val sqlString = s""" |SELECT every(might_contain( | (SELECT bloom_filter_agg(col, @@ -72,7 +72,7 @@ class GlutenBloomFilterAggregateQuerySuite testGluten("Test bloom_filter_agg filter fallback") { val table = "bloom_filter_test" val numEstimatedItems = 5000000L - val numBits = GlutenConfig.getConf.veloxBloomFilterMaxNumBits + val numBits = GlutenConfig.get.veloxBloomFilterMaxNumBits val sqlString = s""" |SELECT col positive_membership_test |FROM $table @@ -118,7 +118,7 @@ class GlutenBloomFilterAggregateQuerySuite testGluten("Test bloom_filter_agg agg fallback") { val table = "bloom_filter_test" val numEstimatedItems = 5000000L - val numBits = GlutenConfig.getConf.veloxBloomFilterMaxNumBits + val numBits = GlutenConfig.get.veloxBloomFilterMaxNumBits val sqlString = s""" |SELECT col positive_membership_test |FROM $table diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala index abd5fddf3bd1..0afabae6e5fd 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.SparkConf import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDataFrameSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDataFrameSuite.scala index 3b2db7117f4a..f47110c13821 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDataFrameSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDataFrameSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{ProjectExecTransformer, WholeStageTransformer} import org.apache.spark.SparkException diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala index 94a6e3bdc293..4678f881a28c 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{BatchScanExecTransformer, FileSourceScanExecTransformer, FilterExecTransformerBase} import org.apache.spark.SparkConf diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala index 276bc95be586..fd731169eae8 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.exception.GlutenException import org.apache.gluten.utils.{BackendTestSettings, BackendTestUtils, SystemParameters} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenSparkSessionExtensionSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenSparkSessionExtensionSuite.scala index 5e3fed736697..ae9b3901afb9 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenSparkSessionExtensionSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenSparkSessionExtensionSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig class GlutenSparkSessionExtensionSuite extends SparkSessionExtensionSuite @@ -31,8 +31,10 @@ class GlutenSparkSessionExtensionSuite try { session.range(2).write.format("parquet").mode("overwrite").saveAsTable("a") def testWithFallbackSettings(scanFallback: Boolean, aggFallback: Boolean): Unit = { - session.sessionState.conf.setConf(GlutenConfig.COLUMNAR_FILESCAN_ENABLED, scanFallback) - session.sessionState.conf.setConf(GlutenConfig.COLUMNAR_HASHAGG_ENABLED, aggFallback) + session.sessionState.conf + .setConfString(GlutenConfig.COLUMNAR_FILESCAN_ENABLED.key, scanFallback.toString) + session.sessionState.conf + .setConfString(GlutenConfig.COLUMNAR_HASHAGG_ENABLED.key, aggFallback.toString) val df = session.sql("SELECT max(id) FROM a") val newDf = DummyFilterColmnarHelper.dfWithDummyFilterColumnar( session, diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala index 8908047a3321..898967d4f300 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/FallbackStrategiesSuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.execution -import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{BasicScanExecTransformer, GlutenPlan} import org.apache.gluten.extension.GlutenSessionExtensions import org.apache.gluten.extension.columnar.{FallbackTags, RemoveFallbackTagRule} @@ -183,7 +183,7 @@ private object FallbackStrategiesSuite { List(c => ExpandFallbackPolicy(c.ac.isAdaptiveContext(), c.ac.originalPlan())), List( c => RemoveTopmostColumnarToRow(c.session, c.ac.isAdaptiveContext()), - _ => ColumnarCollapseTransformStages(GlutenConfig.getConf) + _ => ColumnarCollapseTransformStages(GlutenConfig.get) ), List(_ => RemoveFallbackTagRule()) ) diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala index 706c30d67db8..85aa24a56392 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.execution.benchmarks -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{FileSourceScanExecTransformer, WholeStageTransformer} import org.apache.gluten.extension.columnar.transition.Transitions import org.apache.gluten.jni.JniLibLoader diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenReadSchemaSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenReadSchemaSuite.scala index 7cca2bc3cd7f..6ea7dd910194 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenReadSchemaSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/GlutenReadSchemaSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.execution.datasources -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.SparkConf import org.apache.spark.sql.GlutenSQLTestsBaseTrait diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/csv/GlutenCSVSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/csv/GlutenCSVSuite.scala index 8b75dad33c38..6cfa9f2028e8 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/csv/GlutenCSVSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/csv/GlutenCSVSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.execution.datasources.csv -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.exception.GlutenException import org.apache.spark.{SparkConf, SparkException} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala index 13317444c1d3..f68000fd0bea 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.execution.joins -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} import org.apache.spark.sql.{GlutenTestsCommonTrait, SparkSession} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/joins/GlutenInnerJoinSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/joins/GlutenInnerJoinSuite.scala index 8d9e82df195a..94d5629107df 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/joins/GlutenInnerJoinSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/joins/GlutenInnerJoinSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.execution.joins -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.SparkConf import org.apache.spark.sql.GlutenSQLTestsBaseTrait diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/joins/GlutenOuterJoinSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/joins/GlutenOuterJoinSuite.scala index f48939c3d012..04a555cfce30 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/joins/GlutenOuterJoinSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/joins/GlutenOuterJoinSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.execution.joins -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.SparkConf import org.apache.spark.sql.GlutenSQLTestsBaseTrait diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/extension/GlutenCollapseProjectExecTransformerSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/extension/GlutenCollapseProjectExecTransformerSuite.scala index 5d5f4d2701a7..87a4ce14d4a2 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/extension/GlutenCollapseProjectExecTransformerSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/extension/GlutenCollapseProjectExecTransformerSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.extension -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.ProjectExecTransformer import org.apache.spark.sql.GlutenSQLTestsTrait diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala index d08b2837a599..a8eb2361ab3e 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/extension/GlutenCustomerExtensionSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.extension -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.SparkConf import org.apache.spark.sql.GlutenSQLTestsTrait diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala index 7855f289a707..3fe46bcdec00 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala @@ -16,7 +16,8 @@ */ package org.apache.spark.sql.gluten -import org.apache.gluten.{GlutenBuildInfo, GlutenConfig} +import org.apache.gluten.GlutenBuildInfo +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.events.GlutenPlanFallbackEvent import org.apache.gluten.execution.FileSourceScanExecTransformer import org.apache.gluten.utils.BackendTestUtils diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala index e7d573ca5e7d..56ec103349e7 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.hive.execution -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.utils.SystemParameters import org.apache.spark.{DebugFilesystem, SparkConf} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala index f7cc114859cd..6a7fe611ee82 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.statistics -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.GlutenPlan import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} diff --git a/gluten-ut/test/src/test/scala/org/apache/gluten/execution/MergeTwoPhasesHashBaseAggregateSuite.scala b/gluten-ut/test/src/test/scala/org/apache/gluten/execution/MergeTwoPhasesHashBaseAggregateSuite.scala index 2aaf1be21bf3..7a6c6cd8fc45 100644 --- a/gluten-ut/test/src/test/scala/org/apache/gluten/execution/MergeTwoPhasesHashBaseAggregateSuite.scala +++ b/gluten-ut/test/src/test/scala/org/apache/gluten/execution/MergeTwoPhasesHashBaseAggregateSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.execution -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} import org.apache.spark.SparkConf diff --git a/gluten-ut/test/src/test/scala/org/apache/gluten/expressions/GlutenExpressionMappingSuite.scala b/gluten-ut/test/src/test/scala/org/apache/gluten/expressions/GlutenExpressionMappingSuite.scala index 693b46283cde..4daef607d793 100644 --- a/gluten-ut/test/src/test/scala/org/apache/gluten/expressions/GlutenExpressionMappingSuite.scala +++ b/gluten-ut/test/src/test/scala/org/apache/gluten/expressions/GlutenExpressionMappingSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.expressions -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.ProjectExecTransformer import org.apache.gluten.expression.ExpressionMappings import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} diff --git a/gluten-ut/test/src/test/scala/org/apache/gluten/sql/SQLQuerySuite.scala b/gluten-ut/test/src/test/scala/org/apache/gluten/sql/SQLQuerySuite.scala index 39b9ee33b9c9..8058aab259ca 100644 --- a/gluten-ut/test/src/test/scala/org/apache/gluten/sql/SQLQuerySuite.scala +++ b/gluten-ut/test/src/test/scala/org/apache/gluten/sql/SQLQuerySuite.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.sql -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.WholeStageTransformerSuite import org.apache.gluten.utils.BackendTestUtils import org.apache.gluten.utils.SystemParameters diff --git a/gluten-ut/test/src/test/scala/org/apache/spark/sql/GlutenExpressionDataTypesValidation.scala b/gluten-ut/test/src/test/scala/org/apache/spark/sql/GlutenExpressionDataTypesValidation.scala index 2088c90c019a..e9d6233e9144 100644 --- a/gluten-ut/test/src/test/scala/org/apache/spark/sql/GlutenExpressionDataTypesValidation.scala +++ b/gluten-ut/test/src/test/scala/org/apache/spark/sql/GlutenExpressionDataTypesValidation.scala @@ -16,14 +16,14 @@ */ package org.apache.spark.sql -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{ProjectExecTransformer, TransformSupport, WholeStageTransformerSuite} import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} import org.apache.spark.SparkConf import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions.{BinaryArrayExpressionWithImplicitCast, _} +import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.execution.LeafExecNode import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.types._ diff --git a/shims/common/src/main/scala/org/apache/gluten/config/ConfigBuilder.scala b/shims/common/src/main/scala/org/apache/gluten/config/ConfigBuilder.scala new file mode 100644 index 000000000000..475fe69410a1 --- /dev/null +++ b/shims/common/src/main/scala/org/apache/gluten/config/ConfigBuilder.scala @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.config + +import org.apache.spark.network.util.ByteUnit +import org.apache.spark.network.util.JavaUtils + +import java.util.concurrent.TimeUnit +import java.util.regex.Pattern; + +object BackendType extends Enumeration { + type BackendType = Value + val COMMON, VELOX, CLICKHOUSE = Value +} + +private[gluten] case class ConfigBuilder(key: String) { + import ConfigHelpers._ + + private[config] var _doc = "" + private[config] var _version = "" + private[config] var _backend = BackendType.COMMON + private[config] var _public = true + private[config] var _alternatives = List.empty[String] + private[config] var _onCreate: Option[ConfigEntry[_] => Unit] = None + + def doc(s: String): ConfigBuilder = { + _doc = s + this + } + + def version(s: String): ConfigBuilder = { + _version = s + this + } + + def backend(backend: BackendType.BackendType): ConfigBuilder = { + _backend = backend + this + } + + def internal(): ConfigBuilder = { + _public = false + this + } + + def onCreate(callback: ConfigEntry[_] => Unit): ConfigBuilder = { + _onCreate = Option(callback) + this + } + + def withAlternative(key: String): ConfigBuilder = { + _alternatives = _alternatives :+ key + this + } + + def intConf: TypedConfigBuilder[Int] = { + new TypedConfigBuilder(this, toNumber(_, _.toInt, key, "int")) + } + + def longConf: TypedConfigBuilder[Long] = { + new TypedConfigBuilder(this, toNumber(_, _.toLong, key, "long")) + } + + def doubleConf: TypedConfigBuilder[Double] = { + new TypedConfigBuilder(this, toNumber(_, _.toDouble, key, "double")) + } + + def booleanConf: TypedConfigBuilder[Boolean] = { + new TypedConfigBuilder(this, toBoolean(_, key)) + } + + def stringConf: TypedConfigBuilder[String] = { + new TypedConfigBuilder(this, identity) + } + + def timeConf(unit: TimeUnit): TypedConfigBuilder[Long] = { + new TypedConfigBuilder(this, timeFromString(_, unit), timeToString(_, unit)) + } + + def bytesConf(unit: ByteUnit): TypedConfigBuilder[Long] = { + new TypedConfigBuilder(this, byteFromString(_, unit), byteToString(_, unit)) + } + + def fallbackConf[T](fallback: ConfigEntry[T]): ConfigEntry[T] = { + val entry = + new ConfigEntryFallback[T](key, _doc, _version, _backend, _public, _alternatives, fallback) + _onCreate.foreach(_(entry)) + entry + } +} + +private object ConfigHelpers { + def toNumber[T](s: String, converter: String => T, key: String, configType: String): T = { + try { + converter(s.trim) + } catch { + case _: NumberFormatException => + throw new IllegalArgumentException(s"$key should be $configType, but was $s") + } + } + + def toBoolean(s: String, key: String): Boolean = { + try { + s.trim.toBoolean + } catch { + case _: IllegalArgumentException => + throw new IllegalArgumentException(s"$key should be boolean, but was $s") + } + } + + private val TIME_STRING_PATTERN = Pattern.compile("(-?[0-9]+)([a-z]+)?") + + def timeFromString(str: String, unit: TimeUnit): Long = JavaUtils.timeStringAs(str, unit) + + def timeToString(v: Long, unit: TimeUnit): String = s"${TimeUnit.MILLISECONDS.convert(v, unit)}ms" + + def byteFromString(str: String, unit: ByteUnit): Long = { + val (input, multiplier) = + if (str.length() > 0 && str.charAt(0) == '-') { + (str.substring(1), -1) + } else { + (str, 1) + } + multiplier * JavaUtils.byteStringAs(input, unit) + } + + def byteToString(v: Long, unit: ByteUnit): String = s"${unit.convertTo(v, ByteUnit.BYTE)}b" +} + +private[gluten] class TypedConfigBuilder[T]( + val parent: ConfigBuilder, + val converter: String => T, + val stringConverter: T => String) { + + def this(parent: ConfigBuilder, converter: String => T) = { + this(parent, converter, { v: T => v.toString }) + } + + def transform(fn: T => T): TypedConfigBuilder[T] = { + new TypedConfigBuilder(parent, s => fn(converter(s)), stringConverter) + } + + def checkValue(validator: T => Boolean, errorMsg: String): TypedConfigBuilder[T] = { + transform { + v => + if (!validator(v)) { + throw new IllegalArgumentException(s"'$v' in ${parent.key} is invalid. $errorMsg") + } + v + } + } + + def checkValues(validValues: Set[T]): TypedConfigBuilder[T] = { + transform { + v => + if (!validValues.contains(v)) { + throw new IllegalArgumentException( + s"The value of ${parent.key} should be one of ${validValues.mkString(", ")}, " + + s"but was $v") + } + v + } + } + + def createOptional: OptionalConfigEntry[T] = { + val entry = new OptionalConfigEntry[T]( + parent.key, + parent._doc, + parent._version, + parent._backend, + parent._public, + parent._alternatives, + converter, + stringConverter) + parent._onCreate.foreach(_(entry)) + entry + } + + def createWithDefault(default: T): ConfigEntry[T] = { + assert(default != null, "Use createOptional.") + default match { + case str: String => createWithDefaultString(str) + case _ => + val transformedDefault = converter(stringConverter(default)) + val entry = new ConfigEntryWithDefault[T]( + parent.key, + parent._doc, + parent._version, + parent._backend, + parent._public, + parent._alternatives, + converter, + stringConverter, + transformedDefault) + parent._onCreate.foreach(_(entry)) + entry + } + } + + def createWithDefaultString(default: String): ConfigEntry[T] = { + val entry = new ConfigEntryWithDefaultString[T]( + parent.key, + parent._doc, + parent._version, + parent._backend, + parent._public, + parent._alternatives, + converter, + stringConverter, + default + ) + parent._onCreate.foreach(_(entry)) + entry + } +} diff --git a/shims/common/src/main/scala/org/apache/gluten/config/ConfigEntry.scala b/shims/common/src/main/scala/org/apache/gluten/config/ConfigEntry.scala new file mode 100644 index 000000000000..ea8ab99f4622 --- /dev/null +++ b/shims/common/src/main/scala/org/apache/gluten/config/ConfigEntry.scala @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.config + +import org.apache.gluten.config.BackendType.BackendType + +trait ConfigEntry[T] { + def key: String + def doc: String + def version: String + def backend: BackendType.BackendType + def isPublic: Boolean + def alternatives: List[String] + def valueConverter: String => T + def stringConverter: T => String + def readFrom(conf: ConfigProvider): T + def defaultValue: Option[T] + def defaultValueString: String + + final protected def readString(provider: ConfigProvider): Option[String] = { + alternatives.foldLeft(provider.get(key))((res, nextKey) => res.orElse(provider.get(nextKey))) + } + + override def toString: String = { + s"ConfigEntry(key=$key, defaultValue=$defaultValueString, doc=$doc, " + + s"public=$isPublic, version=$version)" + } + + ConfigEntry.registerEntry(this) +} + +private[gluten] class OptionalConfigEntry[T]( + _key: String, + _doc: String, + _version: String, + _backend: BackendType, + _isPublic: Boolean, + _alternatives: List[String], + _valueConverter: String => T, + _stringConverter: T => String) + extends ConfigEntry[Option[T]] { + override def key: String = _key + + override def doc: String = _doc + + override def version: String = _version + + override def backend: BackendType = _backend + + override def isPublic: Boolean = _isPublic + + override def alternatives: List[String] = _alternatives + + override def valueConverter: String => Option[T] = s => Option(_valueConverter(s)) + + override def stringConverter: Option[T] => String = v => v.map(_stringConverter).orNull + + override def readFrom(conf: ConfigProvider): Option[T] = readString(conf).map(_valueConverter) + + override def defaultValue: Option[Option[T]] = None + + override def defaultValueString: String = ConfigEntry.UNDEFINED +} + +private[gluten] class ConfigEntryWithDefault[T]( + _key: String, + _doc: String, + _version: String, + _backend: BackendType, + _isPublic: Boolean, + _alternatives: List[String], + _valueConverter: String => T, + _stringConverter: T => String, + _defaultVal: T) + extends ConfigEntry[T] { + override def key: String = _key + + override def doc: String = _doc + + override def version: String = _version + + override def backend: BackendType = _backend + + override def isPublic: Boolean = _isPublic + + override def alternatives: List[String] = _alternatives + + override def valueConverter: String => T = _valueConverter + + override def stringConverter: T => String = _stringConverter + + override def readFrom(conf: ConfigProvider): T = { + readString(conf).map(valueConverter).getOrElse(_defaultVal) + } + + override def defaultValue: Option[T] = Option(_defaultVal) + + override def defaultValueString: String = stringConverter(_defaultVal) +} + +private class ConfigEntryWithDefaultString[T]( + _key: String, + _doc: String, + _version: String, + _backend: BackendType, + _isPublic: Boolean, + _alternatives: List[String], + _valueConverter: String => T, + _stringConverter: T => String, + _defaultVal: String) + extends ConfigEntry[T] { + override def key: String = _key + + override def doc: String = _doc + + override def version: String = _version + + override def backend: BackendType = _backend + + override def isPublic: Boolean = _isPublic + + override def alternatives: List[String] = _alternatives + + override def valueConverter: String => T = _valueConverter + + override def stringConverter: T => String = _stringConverter + + override def readFrom(conf: ConfigProvider): T = { + val value = readString(conf).getOrElse(_defaultVal) + valueConverter(value) + } + + override def defaultValue: Option[T] = Some(valueConverter(_defaultVal)) + + override def defaultValueString: String = _defaultVal +} + +private[gluten] class ConfigEntryFallback[T]( + _key: String, + _doc: String, + _version: String, + _backend: BackendType, + _isPublic: Boolean, + _alternatives: List[String], + fallback: ConfigEntry[T]) + extends ConfigEntry[T] { + override def key: String = _key + + override def doc: String = _doc + + override def version: String = _version + + override def backend: BackendType = _backend + + override def isPublic: Boolean = _isPublic + + override def alternatives: List[String] = _alternatives + + override def valueConverter: String => T = fallback.valueConverter + + override def stringConverter: T => String = fallback.stringConverter + + override def readFrom(conf: ConfigProvider): T = { + readString(conf).map(valueConverter).getOrElse(fallback.readFrom(conf)) + } + + override def defaultValue: Option[T] = fallback.defaultValue + + override def defaultValueString: String = fallback.defaultValueString +} + +object ConfigEntry { + + val UNDEFINED = "" + + private val knownConfigs = + new java.util.concurrent.ConcurrentHashMap[String, ConfigEntry[_]]() + + private def registerEntry(entry: ConfigEntry[_]): Unit = { + val existing = knownConfigs.putIfAbsent(entry.key, entry) + require(existing == null, s"Config entry ${entry.key} already registered!") + } + + def containsEntry(entry: ConfigEntry[_]): Boolean = { + Option(knownConfigs.get(entry.key)).isDefined + } + + def findEntry(key: String): ConfigEntry[_] = knownConfigs.get(key) +} diff --git a/shims/common/src/main/scala/org/apache/gluten/config/ConfigProvider.scala b/shims/common/src/main/scala/org/apache/gluten/config/ConfigProvider.scala new file mode 100644 index 000000000000..8f5e3014286b --- /dev/null +++ b/shims/common/src/main/scala/org/apache/gluten/config/ConfigProvider.scala @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.config + +import scala.collection.immutable + +class ConfigProvider(conf: immutable.Map[String, String]) { + def get(key: String): Option[String] = { + conf.get(key) + } +} diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/config/GlutenConfig.scala similarity index 85% rename from shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala rename to shims/common/src/main/scala/org/apache/gluten/config/GlutenConfig.scala index 296153346ef8..565df5c2ad2d 100644 --- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/config/GlutenConfig.scala @@ -14,11 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten +package org.apache.gluten.config import org.apache.spark.internal.Logging import org.apache.spark.network.util.{ByteUnit, JavaUtils} import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.internal.SQLConf.ANSI_ENABLED import com.google.common.collect.ImmutableList import org.apache.hadoop.security.UserGroupInformation @@ -37,97 +38,90 @@ case class GlutenNumaBindingInfo( class GlutenConfig(conf: SQLConf) extends Logging { import GlutenConfig._ - def enableAnsiMode: Boolean = conf.ansiEnabled + def enableAnsiMode: Boolean = conf.getConf(ANSI_ENABLED) - def enableGluten: Boolean = conf.getConf(GLUTEN_ENABLED) + def enableGluten: Boolean = getConf(GLUTEN_ENABLED) // FIXME the option currently controls both JVM and native validation against a Substrait plan. - def enableNativeValidation: Boolean = conf.getConf(NATIVE_VALIDATION_ENABLED) + def enableNativeValidation: Boolean = getConf(NATIVE_VALIDATION_ENABLED) - def enableColumnarBatchScan: Boolean = conf.getConf(COLUMNAR_BATCHSCAN_ENABLED) + def enableColumnarBatchScan: Boolean = getConf(COLUMNAR_BATCHSCAN_ENABLED) - def enableColumnarFileScan: Boolean = conf.getConf(COLUMNAR_FILESCAN_ENABLED) + def enableColumnarFileScan: Boolean = getConf(COLUMNAR_FILESCAN_ENABLED) - def enableColumnarHiveTableScan: Boolean = conf.getConf(COLUMNAR_HIVETABLESCAN_ENABLED) + def enableColumnarHiveTableScan: Boolean = getConf(COLUMNAR_HIVETABLESCAN_ENABLED) def enableColumnarHiveTableScanNestedColumnPruning: Boolean = - conf.getConf(COLUMNAR_HIVETABLESCAN_NESTED_COLUMN_PRUNING_ENABLED) + getConf(COLUMNAR_HIVETABLESCAN_NESTED_COLUMN_PRUNING_ENABLED) - def enableVanillaVectorizedReaders: Boolean = conf.getConf(VANILLA_VECTORIZED_READERS_ENABLED) + def enableVanillaVectorizedReaders: Boolean = getConf(VANILLA_VECTORIZED_READERS_ENABLED) - def enableColumnarHashAgg: Boolean = conf.getConf(COLUMNAR_HASHAGG_ENABLED) + def enableColumnarHashAgg: Boolean = getConf(COLUMNAR_HASHAGG_ENABLED) - def forceToUseHashAgg: Boolean = conf.getConf(COLUMNAR_FORCE_HASHAGG_ENABLED) + def forceToUseHashAgg: Boolean = getConf(COLUMNAR_FORCE_HASHAGG_ENABLED) - def mergeTwoPhasesAggEnabled: Boolean = conf.getConf(MERGE_TWO_PHASES_ENABLED) + def mergeTwoPhasesAggEnabled: Boolean = getConf(MERGE_TWO_PHASES_ENABLED) - def enableColumnarProject: Boolean = conf.getConf(COLUMNAR_PROJECT_ENABLED) + def enableColumnarProject: Boolean = getConf(COLUMNAR_PROJECT_ENABLED) - def enableColumnarFilter: Boolean = conf.getConf(COLUMNAR_FILTER_ENABLED) + def enableColumnarFilter: Boolean = getConf(COLUMNAR_FILTER_ENABLED) - def enableColumnarSort: Boolean = conf.getConf(COLUMNAR_SORT_ENABLED) + def enableColumnarSort: Boolean = getConf(COLUMNAR_SORT_ENABLED) - def enableColumnarWindow: Boolean = conf.getConf(COLUMNAR_WINDOW_ENABLED) + def enableColumnarWindow: Boolean = getConf(COLUMNAR_WINDOW_ENABLED) - def enableColumnarWindowGroupLimit: Boolean = conf.getConf(COLUMNAR_WINDOW_GROUP_LIMIT_ENABLED) + def enableColumnarWindowGroupLimit: Boolean = getConf(COLUMNAR_WINDOW_GROUP_LIMIT_ENABLED) - def veloxColumnarWindowType: String = conf.getConfString(COLUMNAR_VELOX_WINDOW_TYPE.key) + def veloxColumnarWindowType: String = getConf(COLUMNAR_VELOX_WINDOW_TYPE) - def enableColumnarShuffledHashJoin: Boolean = conf.getConf(COLUMNAR_SHUFFLED_HASH_JOIN_ENABLED) + def enableColumnarShuffledHashJoin: Boolean = getConf(COLUMNAR_SHUFFLED_HASH_JOIN_ENABLED) def shuffledHashJoinOptimizeBuildSide: Boolean = - conf.getConf(COLUMNAR_SHUFFLED_HASH_JOIN_OPTIMIZE_BUILD_SIDE) + getConf(COLUMNAR_SHUFFLED_HASH_JOIN_OPTIMIZE_BUILD_SIDE) - def enableNativeColumnarToRow: Boolean = conf.getConf(COLUMNAR_COLUMNAR_TO_ROW_ENABLED) + def forceShuffledHashJoin: Boolean = getConf(COLUMNAR_FORCE_SHUFFLED_HASH_JOIN_ENABLED) - def forceShuffledHashJoin: Boolean = conf.getConf(COLUMNAR_FORCE_SHUFFLED_HASH_JOIN_ENABLED) + def enableColumnarSortMergeJoin: Boolean = getConf(COLUMNAR_SORTMERGEJOIN_ENABLED) - def enableColumnarSortMergeJoin: Boolean = conf.getConf(COLUMNAR_SORTMERGEJOIN_ENABLED) + def enableColumnarUnion: Boolean = getConf(COLUMNAR_UNION_ENABLED) - def enableColumnarUnion: Boolean = conf.getConf(COLUMNAR_UNION_ENABLED) + def enableNativeUnion: Boolean = getConf(NATIVE_UNION_ENABLED) - def enableNativeUnion: Boolean = conf.getConf(NATIVE_UNION_ENABLED) + def enableColumnarExpand: Boolean = getConf(COLUMNAR_EXPAND_ENABLED) - def enableColumnarExpand: Boolean = conf.getConf(COLUMNAR_EXPAND_ENABLED) + def enableColumnarBroadcastExchange: Boolean = getConf(COLUMNAR_BROADCAST_EXCHANGE_ENABLED) - def enableColumnarBroadcastExchange: Boolean = conf.getConf(COLUMNAR_BROADCAST_EXCHANGE_ENABLED) + def enableColumnarBroadcastJoin: Boolean = getConf(COLUMNAR_BROADCAST_JOIN_ENABLED) - def enableColumnarBroadcastJoin: Boolean = conf.getConf(COLUMNAR_BROADCAST_JOIN_ENABLED) + def enableColumnarSample: Boolean = getConf(COLUMNAR_SAMPLE_ENABLED) - def enableColumnarSample: Boolean = conf.getConf(COLUMNAR_SAMPLE_ENABLED) + def enableColumnarArrowUDF: Boolean = getConf(COLUMNAR_ARROW_UDF_ENABLED) - def enableColumnarArrowUDF: Boolean = conf.getConf(COLUMNAR_ARROW_UDF_ENABLED) + def enableColumnarCoalesce: Boolean = getConf(COLUMNAR_COALESCE_ENABLED) - def enableColumnarCoalesce: Boolean = conf.getConf(COLUMNAR_COALESCE_ENABLED) + def enableRewriteDateTimestampComparison: Boolean = getConf( + ENABLE_REWRITE_DATE_TIMESTAMP_COMPARISON) - def columnarTableCacheEnabled: Boolean = conf.getConf(COLUMNAR_TABLE_CACHE_ENABLED) + def enableCHRewriteDateConversion: Boolean = getConf(ENABLE_CH_REWRITE_DATE_CONVERSION) - def enableRewriteDateTimestampComparison: Boolean = - conf.getConf(ENABLE_REWRITE_DATE_TIMESTAMP_COMPARISON) + def enableCommonSubexpressionEliminate: Boolean = getConf(ENABLE_COMMON_SUBEXPRESSION_ELIMINATE) - def enableCHRewriteDateConversion: Boolean = - conf.getConf(ENABLE_CH_REWRITE_DATE_CONVERSION) - - def enableCommonSubexpressionEliminate: Boolean = - conf.getConf(ENABLE_COMMON_SUBEXPRESSION_ELIMINATE) - - def enableCountDistinctWithoutExpand: Boolean = - conf.getConf(ENABLE_COUNT_DISTINCT_WITHOUT_EXPAND) + def enableCountDistinctWithoutExpand: Boolean = getConf(ENABLE_COUNT_DISTINCT_WITHOUT_EXPAND) def enableExtendedColumnPruning: Boolean = - conf.getConf(ENABLE_EXTENDED_COLUMN_PRUNING) + getConf(ENABLE_EXTENDED_COLUMN_PRUNING) def veloxOrcScanEnabled: Boolean = - conf.getConf(VELOX_ORC_SCAN_ENABLED) + getConf(VELOX_ORC_SCAN_ENABLED) def forceOrcCharTypeScanFallbackEnabled: Boolean = - conf.getConf(VELOX_FORCE_ORC_CHAR_TYPE_SCAN_FALLBACK) + getConf(VELOX_FORCE_ORC_CHAR_TYPE_SCAN_FALLBACK) def forceParquetTimestampTypeScanFallbackEnabled: Boolean = - conf.getConf(VELOX_FORCE_PARQUET_TIMESTAMP_TYPE_SCAN_FALLBACK) + getConf(VELOX_FORCE_PARQUET_TIMESTAMP_TYPE_SCAN_FALLBACK) def scanFileSchemeValidationEnabled: Boolean = - conf.getConf(VELOX_SCAN_FILE_SCHEME_VALIDATION_ENABLED) + getConf(VELOX_SCAN_FILE_SCHEME_VALIDATION_ENABLED) // whether to use ColumnarShuffleManager def isUseColumnarShuffleManager: Boolean = @@ -151,44 +145,43 @@ class GlutenConfig(conf: SQLConf) extends Logging { .getConfString("spark.celeborn.client.spark.shuffle.writer", GLUTEN_HASH_SHUFFLE_WRITER) .toLowerCase(Locale.ROOT) - def enableColumnarShuffle: Boolean = conf.getConf(COLUMNAR_SHUFFLE_ENABLED) + def enableColumnarShuffle: Boolean = getConf(COLUMNAR_SHUFFLE_ENABLED) - def enablePreferColumnar: Boolean = conf.getConf(COLUMNAR_PREFER_ENABLED) + def enablePreferColumnar: Boolean = getConf(COLUMNAR_PREFER_ENABLED) - def enableOneRowRelationColumnar: Boolean = conf.getConf(COLUMNAR_ONE_ROW_RELATION_ENABLED) + def enableOneRowRelationColumnar: Boolean = getConf(COLUMNAR_ONE_ROW_RELATION_ENABLED) def physicalJoinOptimizationThrottle: Integer = - conf.getConf(COLUMNAR_PHYSICAL_JOIN_OPTIMIZATION_THROTTLE) + getConf(COLUMNAR_PHYSICAL_JOIN_OPTIMIZATION_THROTTLE) def enablePhysicalJoinOptimize: Boolean = - conf.getConf(COLUMNAR_PHYSICAL_JOIN_OPTIMIZATION_ENABLED) + getConf(COLUMNAR_PHYSICAL_JOIN_OPTIMIZATION_ENABLED) def logicalJoinOptimizationThrottle: Integer = - conf.getConf(COLUMNAR_LOGICAL_JOIN_OPTIMIZATION_THROTTLE) + getConf(COLUMNAR_LOGICAL_JOIN_OPTIMIZATION_THROTTLE) - def enableScanOnly: Boolean = conf.getConf(COLUMNAR_SCAN_ONLY_ENABLED) + def enableScanOnly: Boolean = getConf(COLUMNAR_SCAN_ONLY_ENABLED) - def tmpFile: Option[String] = conf.getConf(COLUMNAR_TEMP_DIR) + def tmpFile: Option[String] = getConf(COLUMNAR_TEMP_DIR) - @deprecated def broadcastCacheTimeout: Int = conf.getConf(COLUMNAR_BROADCAST_CACHE_TIMEOUT) + @deprecated def broadcastCacheTimeout: Int = getConf(COLUMNAR_BROADCAST_CACHE_TIMEOUT) def columnarShuffleSortPartitionsThreshold: Int = - conf.getConf(COLUMNAR_SHUFFLE_SORT_PARTITIONS_THRESHOLD) + getConf(COLUMNAR_SHUFFLE_SORT_PARTITIONS_THRESHOLD) def columnarShuffleSortColumnsThreshold: Int = - conf.getConf(COLUMNAR_SHUFFLE_SORT_COLUMNS_THRESHOLD) + getConf(COLUMNAR_SHUFFLE_SORT_COLUMNS_THRESHOLD) - def columnarShuffleReallocThreshold: Double = conf.getConf(COLUMNAR_SHUFFLE_REALLOC_THRESHOLD) + def columnarShuffleReallocThreshold: Double = getConf(COLUMNAR_SHUFFLE_REALLOC_THRESHOLD) - def columnarShuffleMergeThreshold: Double = conf.getConf(SHUFFLE_WRITER_MERGE_THRESHOLD) + def columnarShuffleMergeThreshold: Double = getConf(SHUFFLE_WRITER_MERGE_THRESHOLD) - def columnarShuffleCodec: Option[String] = conf.getConf(COLUMNAR_SHUFFLE_CODEC) + def columnarShuffleCodec: Option[String] = getConf(COLUMNAR_SHUFFLE_CODEC) def columnarShuffleCompressionMode: String = - conf.getConf(COLUMNAR_SHUFFLE_COMPRESSION_MODE) + getConf(COLUMNAR_SHUFFLE_COMPRESSION_MODE) - def columnarShuffleCodecBackend: Option[String] = conf - .getConf(COLUMNAR_SHUFFLE_CODEC_BACKEND) + def columnarShuffleCodecBackend: Option[String] = getConf(COLUMNAR_SHUFFLE_CODEC_BACKEND) .filter(Set(GLUTEN_QAT_BACKEND_NAME, GLUTEN_IAA_BACKEND_NAME).contains(_)) def columnarShuffleEnableQat: Boolean = @@ -198,54 +191,53 @@ class GlutenConfig(conf: SQLConf) extends Logging { columnarShuffleCodecBackend.contains(GlutenConfig.GLUTEN_IAA_BACKEND_NAME) def columnarShuffleCompressionThreshold: Int = - conf.getConf(COLUMNAR_SHUFFLE_COMPRESSION_THRESHOLD) + getConf(COLUMNAR_SHUFFLE_COMPRESSION_THRESHOLD) def columnarShuffleReaderBufferSize: Long = - conf.getConf(COLUMNAR_SHUFFLE_READER_BUFFER_SIZE) + getConf(COLUMNAR_SHUFFLE_READER_BUFFER_SIZE) - def maxBatchSize: Int = conf.getConf(COLUMNAR_MAX_BATCH_SIZE) + def maxBatchSize: Int = getConf(COLUMNAR_MAX_BATCH_SIZE) def columnarToRowMemThreshold: Long = - conf.getConf(GLUTEN_COLUMNAR_TO_ROW_MEM_THRESHOLD) + getConf(GLUTEN_COLUMNAR_TO_ROW_MEM_THRESHOLD) - def shuffleWriterBufferSize: Int = conf - .getConf(SHUFFLE_WRITER_BUFFER_SIZE) + def shuffleWriterBufferSize: Int = getConf(SHUFFLE_WRITER_BUFFER_SIZE) .getOrElse(maxBatchSize) - def enableColumnarLimit: Boolean = conf.getConf(COLUMNAR_LIMIT_ENABLED) + def enableColumnarLimit: Boolean = getConf(COLUMNAR_LIMIT_ENABLED) - def enableColumnarGenerate: Boolean = conf.getConf(COLUMNAR_GENERATE_ENABLED) + def enableColumnarGenerate: Boolean = getConf(COLUMNAR_GENERATE_ENABLED) def enableTakeOrderedAndProject: Boolean = - conf.getConf(COLUMNAR_TAKE_ORDERED_AND_PROJECT_ENABLED) + getConf(COLUMNAR_TAKE_ORDERED_AND_PROJECT_ENABLED) - def enableNativeBloomFilter: Boolean = conf.getConf(COLUMNAR_NATIVE_BLOOMFILTER_ENABLED) + def enableNativeBloomFilter: Boolean = getConf(COLUMNAR_NATIVE_BLOOMFILTER_ENABLED) def enableNativeHyperLogLogAggregateFunction: Boolean = - conf.getConf(COLUMNAR_NATIVE_HYPERLOGLOG_AGGREGATE_ENABLED) + getConf(COLUMNAR_NATIVE_HYPERLOGLOG_AGGREGATE_ENABLED) def columnarParquetWriteBlockSize: Long = - conf.getConf(COLUMNAR_PARQUET_WRITE_BLOCK_SIZE) + getConf(COLUMNAR_PARQUET_WRITE_BLOCK_SIZE) def columnarParquetWriteBlockRows: Long = - conf.getConf(COLUMNAR_PARQUET_WRITE_BLOCK_ROWS) + getConf(COLUMNAR_PARQUET_WRITE_BLOCK_ROWS) - def wholeStageFallbackThreshold: Int = conf.getConf(COLUMNAR_WHOLESTAGE_FALLBACK_THRESHOLD) + def wholeStageFallbackThreshold: Int = getConf(COLUMNAR_WHOLESTAGE_FALLBACK_THRESHOLD) - def queryFallbackThreshold: Int = conf.getConf(COLUMNAR_QUERY_FALLBACK_THRESHOLD) + def queryFallbackThreshold: Int = getConf(COLUMNAR_QUERY_FALLBACK_THRESHOLD) - def fallbackIgnoreRowToColumnar: Boolean = conf.getConf(COLUMNAR_FALLBACK_IGNORE_ROW_TO_COLUMNAR) + def fallbackIgnoreRowToColumnar: Boolean = getConf(COLUMNAR_FALLBACK_IGNORE_ROW_TO_COLUMNAR) - def fallbackExpressionsThreshold: Int = conf.getConf(COLUMNAR_FALLBACK_EXPRESSIONS_THRESHOLD) + def fallbackExpressionsThreshold: Int = getConf(COLUMNAR_FALLBACK_EXPRESSIONS_THRESHOLD) - def fallbackPreferColumnar: Boolean = conf.getConf(COLUMNAR_FALLBACK_PREFER_COLUMNAR) + def fallbackPreferColumnar: Boolean = getConf(COLUMNAR_FALLBACK_PREFER_COLUMNAR) def numaBindingInfo: GlutenNumaBindingInfo = { - val enableNumaBinding: Boolean = conf.getConf(COLUMNAR_NUMA_BINDING_ENABLED) + val enableNumaBinding: Boolean = getConf(COLUMNAR_NUMA_BINDING_ENABLED) if (!enableNumaBinding) { GlutenNumaBindingInfo(enableNumaBinding = false) } else { - val tmp = conf.getConf(COLUMNAR_NUMA_BINDING_CORE_RANGE) + val tmp = getConf(COLUMNAR_NUMA_BINDING_CORE_RANGE) if (tmp.isEmpty) { GlutenNumaBindingInfo(enableNumaBinding = false) } else { @@ -257,78 +249,78 @@ class GlutenConfig(conf: SQLConf) extends Logging { } } - def memoryIsolation: Boolean = conf.getConf(COLUMNAR_MEMORY_ISOLATION) + def memoryIsolation: Boolean = getConf(COLUMNAR_MEMORY_ISOLATION) - def memoryBacktraceAllocation: Boolean = conf.getConf(COLUMNAR_MEMORY_BACKTRACE_ALLOCATION) + def memoryBacktraceAllocation: Boolean = getConf(COLUMNAR_MEMORY_BACKTRACE_ALLOCATION) def numTaskSlotsPerExecutor: Int = { - val numSlots = conf.getConf(NUM_TASK_SLOTS_PER_EXECUTOR) + val numSlots = getConf(NUM_TASK_SLOTS_PER_EXECUTOR) assert(numSlots > 0, s"Number of task slot not found. This should not happen.") numSlots } - def offHeapMemorySize: Long = conf.getConf(COLUMNAR_OFFHEAP_SIZE_IN_BYTES) + def offHeapMemorySize: Long = getConf(COLUMNAR_OFFHEAP_SIZE_IN_BYTES) - def taskOffHeapMemorySize: Long = conf.getConf(COLUMNAR_TASK_OFFHEAP_SIZE_IN_BYTES) + def taskOffHeapMemorySize: Long = getConf(COLUMNAR_TASK_OFFHEAP_SIZE_IN_BYTES) - def memoryOverAcquiredRatio: Double = conf.getConf(COLUMNAR_MEMORY_OVER_ACQUIRED_RATIO) + def memoryOverAcquiredRatio: Double = getConf(COLUMNAR_MEMORY_OVER_ACQUIRED_RATIO) - def memoryReservationBlockSize: Long = conf.getConf(COLUMNAR_MEMORY_RESERVATION_BLOCK_SIZE) + def memoryReservationBlockSize: Long = getConf(COLUMNAR_MEMORY_RESERVATION_BLOCK_SIZE) def conservativeTaskOffHeapMemorySize: Long = - conf.getConf(COLUMNAR_CONSERVATIVE_TASK_OFFHEAP_SIZE_IN_BYTES) + getConf(COLUMNAR_CONSERVATIVE_TASK_OFFHEAP_SIZE_IN_BYTES) // Options used by RAS. - def enableRas: Boolean = conf.getConf(RAS_ENABLED) + def enableRas: Boolean = getConf(RAS_ENABLED) - def rasCostModel: String = conf.getConf(RAS_COST_MODEL) + def rasCostModel: String = getConf(RAS_COST_MODEL) - def rasRough2SizeBytesThreshold: Long = conf.getConf(RAS_ROUGH2_SIZEBYTES_THRESHOLD) + def rasRough2SizeBytesThreshold: Long = getConf(RAS_ROUGH2_SIZEBYTES_THRESHOLD) - def rasRough2R2cCost: Long = conf.getConf(RAS_ROUGH2_R2C_COST) + def rasRough2R2cCost: Long = getConf(RAS_ROUGH2_R2C_COST) - def rasRough2VanillaCost: Long = conf.getConf(RAS_ROUGH2_VANILLA_COST) + def rasRough2VanillaCost: Long = getConf(RAS_ROUGH2_VANILLA_COST) - def enableVeloxCache: Boolean = conf.getConf(COLUMNAR_VELOX_CACHE_ENABLED) + def enableVeloxCache: Boolean = getConf(COLUMNAR_VELOX_CACHE_ENABLED) - def veloxMemCacheSize: Long = conf.getConf(COLUMNAR_VELOX_MEM_CACHE_SIZE) + def veloxMemCacheSize: Long = getConf(COLUMNAR_VELOX_MEM_CACHE_SIZE) - def veloxSsdCachePath: String = conf.getConf(COLUMNAR_VELOX_SSD_CACHE_PATH) + def veloxSsdCachePath: String = getConf(COLUMNAR_VELOX_SSD_CACHE_PATH) - def veloxSsdCacheSize: Long = conf.getConf(COLUMNAR_VELOX_SSD_CACHE_SIZE) + def veloxSsdCacheSize: Long = getConf(COLUMNAR_VELOX_SSD_CACHE_SIZE) - def veloxSsdCacheShards: Integer = conf.getConf(COLUMNAR_VELOX_SSD_CACHE_SHARDS) + def veloxSsdCacheShards: Integer = getConf(COLUMNAR_VELOX_SSD_CACHE_SHARDS) - def veloxSsdCacheIOThreads: Integer = conf.getConf(COLUMNAR_VELOX_SSD_CACHE_IO_THREADS) + def veloxSsdCacheIOThreads: Integer = getConf(COLUMNAR_VELOX_SSD_CACHE_IO_THREADS) - def veloxSsdODirectEnabled: Boolean = conf.getConf(COLUMNAR_VELOX_SSD_ODIRECT_ENABLED) + def veloxSsdODirectEnabled: Boolean = getConf(COLUMNAR_VELOX_SSD_ODIRECT_ENABLED) def veloxConnectorIOThreads: Int = { - conf.getConf(COLUMNAR_VELOX_CONNECTOR_IO_THREADS).getOrElse(numTaskSlotsPerExecutor) + getConf(COLUMNAR_VELOX_CONNECTOR_IO_THREADS).getOrElse(numTaskSlotsPerExecutor) } - def veloxSplitPreloadPerDriver: Integer = conf.getConf(COLUMNAR_VELOX_SPLIT_PRELOAD_PER_DRIVER) + def veloxSplitPreloadPerDriver: Integer = getConf(COLUMNAR_VELOX_SPLIT_PRELOAD_PER_DRIVER) - def veloxSpillStrategy: String = conf.getConf(COLUMNAR_VELOX_SPILL_STRATEGY) + def veloxSpillStrategy: String = getConf(COLUMNAR_VELOX_SPILL_STRATEGY) - def veloxMaxSpillLevel: Int = conf.getConf(COLUMNAR_VELOX_MAX_SPILL_LEVEL) + def veloxMaxSpillLevel: Int = getConf(COLUMNAR_VELOX_MAX_SPILL_LEVEL) - def veloxMaxSpillFileSize: Long = conf.getConf(COLUMNAR_VELOX_MAX_SPILL_FILE_SIZE) + def veloxMaxSpillFileSize: Long = getConf(COLUMNAR_VELOX_MAX_SPILL_FILE_SIZE) - def veloxSpillFileSystem: String = conf.getConf(COLUMNAR_VELOX_SPILL_FILE_SYSTEM) + def veloxSpillFileSystem: String = getConf(COLUMNAR_VELOX_SPILL_FILE_SYSTEM) - def veloxMaxSpillRunRows: Long = conf.getConf(COLUMNAR_VELOX_MAX_SPILL_RUN_ROWS) + def veloxMaxSpillRunRows: Long = getConf(COLUMNAR_VELOX_MAX_SPILL_RUN_ROWS) - def veloxMaxSpillBytes: Long = conf.getConf(COLUMNAR_VELOX_MAX_SPILL_BYTES) + def veloxMaxSpillBytes: Long = getConf(COLUMNAR_VELOX_MAX_SPILL_BYTES) def veloxBloomFilterExpectedNumItems: Long = - conf.getConf(COLUMNAR_VELOX_BLOOM_FILTER_EXPECTED_NUM_ITEMS) + getConf(COLUMNAR_VELOX_BLOOM_FILTER_EXPECTED_NUM_ITEMS) - def veloxBloomFilterNumBits: Long = conf.getConf(COLUMNAR_VELOX_BLOOM_FILTER_NUM_BITS) + def veloxBloomFilterNumBits: Long = getConf(COLUMNAR_VELOX_BLOOM_FILTER_NUM_BITS) - def veloxBloomFilterMaxNumBits: Long = conf.getConf(COLUMNAR_VELOX_BLOOM_FILTER_MAX_NUM_BITS) + def veloxBloomFilterMaxNumBits: Long = getConf(COLUMNAR_VELOX_BLOOM_FILTER_MAX_NUM_BITS) - def castFromVarcharAddTrimNode: Boolean = conf.getConf(CAST_FROM_VARCHAR_ADD_TRIM_NODE) + def castFromVarcharAddTrimNode: Boolean = getConf(CAST_FROM_VARCHAR_ADD_TRIM_NODE) case class ResizeRange(min: Int, max: Int) { assert(max >= min) @@ -346,62 +338,61 @@ class GlutenConfig(conf: SQLConf) extends Logging { } def veloxResizeBatchesShuffleInput: Boolean = - conf.getConf(COLUMNAR_VELOX_RESIZE_BATCHES_SHUFFLE_INPUT) + getConf(COLUMNAR_VELOX_RESIZE_BATCHES_SHUFFLE_INPUT) def veloxResizeBatchesShuffleInputRange: ResizeRange = { - val standardSize = conf.getConf(COLUMNAR_MAX_BATCH_SIZE) + val standardSize = getConf(COLUMNAR_MAX_BATCH_SIZE) val defaultMinSize: Int = (0.25 * standardSize).toInt.max(1) - val minSize = conf - .getConf(COLUMNAR_VELOX_RESIZE_BATCHES_SHUFFLE_INPUT_MIN_SIZE) + val minSize = getConf(COLUMNAR_VELOX_RESIZE_BATCHES_SHUFFLE_INPUT_MIN_SIZE) .getOrElse(defaultMinSize) ResizeRange(minSize, Int.MaxValue) } def chColumnarShuffleSpillThreshold: Long = { - val threshold = conf.getConf(COLUMNAR_CH_SHUFFLE_SPILL_THRESHOLD) + val threshold = getConf(COLUMNAR_CH_SHUFFLE_SPILL_THRESHOLD) if (threshold == 0) { - (conf.getConf(COLUMNAR_TASK_OFFHEAP_SIZE_IN_BYTES) * 0.9).toLong + (getConf(COLUMNAR_TASK_OFFHEAP_SIZE_IN_BYTES) * 0.9).toLong } else { threshold } } - def chColumnarMaxSortBufferSize: Long = conf.getConf(COLUMNAR_CH_MAX_SORT_BUFFER_SIZE) + def chColumnarMaxSortBufferSize: Long = getConf(COLUMNAR_CH_MAX_SORT_BUFFER_SIZE) def chColumnarForceMemorySortShuffle: Boolean = - conf.getConf(COLUMNAR_CH_FORCE_MEMORY_SORT_SHUFFLE) + getConf(COLUMNAR_CH_FORCE_MEMORY_SORT_SHUFFLE) def cartesianProductTransformerEnabled: Boolean = - conf.getConf(CARTESIAN_PRODUCT_TRANSFORMER_ENABLED) + getConf(CARTESIAN_PRODUCT_TRANSFORMER_ENABLED) def broadcastNestedLoopJoinTransformerTransformerEnabled: Boolean = - conf.getConf(BROADCAST_NESTED_LOOP_JOIN_TRANSFORMER_ENABLED) + getConf(BROADCAST_NESTED_LOOP_JOIN_TRANSFORMER_ENABLED) - def transformPlanLogLevel: String = conf.getConf(TRANSFORM_PLAN_LOG_LEVEL) + def transformPlanLogLevel: String = getConf(TRANSFORM_PLAN_LOG_LEVEL) - def substraitPlanLogLevel: String = conf.getConf(SUBSTRAIT_PLAN_LOG_LEVEL) + def substraitPlanLogLevel: String = getConf(SUBSTRAIT_PLAN_LOG_LEVEL) - def validationLogLevel: String = conf.getConf(VALIDATION_LOG_LEVEL) + def validationLogLevel: String = getConf(VALIDATION_LOG_LEVEL) - def softAffinityLogLevel: String = conf.getConf(SOFT_AFFINITY_LOG_LEVEL) + def softAffinityLogLevel: String = getConf(SOFT_AFFINITY_LOG_LEVEL) // A comma-separated list of classes for the extended columnar pre rules - def extendedColumnarTransformRules: String = conf.getConf(EXTENDED_COLUMNAR_TRANSFORM_RULES) + def extendedColumnarTransformRules: String = getConf(EXTENDED_COLUMNAR_TRANSFORM_RULES) // A comma-separated list of classes for the extended columnar post rules - def extendedColumnarPostRules: String = conf.getConf(EXTENDED_COLUMNAR_POST_RULES) + def extendedColumnarPostRules: String = getConf(EXTENDED_COLUMNAR_POST_RULES) - def extendedExpressionTransformer: String = conf.getConf(EXTENDED_EXPRESSION_TRAN_CONF) + def extendedExpressionTransformer: String = getConf(EXTENDED_EXPRESSION_TRAN_CONF) def expressionBlacklist: Set[String] = { - val blacklist = conf.getConf(EXPRESSION_BLACK_LIST) + val blacklist = getConf(EXPRESSION_BLACK_LIST) val blacklistSet: Set[String] = if (blacklist.isDefined) { blacklist.get.toLowerCase(Locale.ROOT).trim.split(",").toSet } else { Set.empty } - if (conf.getConf(FALLBACK_REGEXP_EXPRESSIONS)) { + if (getConf(FALLBACK_REGEXP_EXPRESSIONS)) { val regexpList = "rlike,regexp_replace,regexp_extract,regexp_extract_all,split" regexpList.trim.split(",").toSet ++ blacklistSet } else { @@ -410,86 +401,96 @@ class GlutenConfig(conf: SQLConf) extends Logging { } def printStackOnValidationFailure: Boolean = - conf.getConf(VALIDATION_PRINT_FAILURE_STACK_) + getConf(VALIDATION_PRINT_FAILURE_STACK_) - def enableFallbackReport: Boolean = conf.getConf(FALLBACK_REPORTER_ENABLED) + def enableFallbackReport: Boolean = getConf(FALLBACK_REPORTER_ENABLED) def enableVeloxUserExceptionStacktrace: Boolean = - conf.getConf(COLUMNAR_VELOX_ENABLE_USER_EXCEPTION_STACKTRACE) + getConf(COLUMNAR_VELOX_ENABLE_USER_EXCEPTION_STACKTRACE) def memoryUseHugePages: Boolean = - conf.getConf(COLUMNAR_VELOX_MEMORY_USE_HUGE_PAGES) - - def debug: Boolean = conf.getConf(DEBUG_ENABLED) - def debugKeepJniWorkspace: Boolean = conf.getConf(DEBUG_KEEP_JNI_WORKSPACE) - def collectUtStats: Boolean = conf.getConf(UT_STATISTIC) - def benchmarkStageId: Int = conf.getConf(BENCHMARK_TASK_STAGEID) - def benchmarkPartitionId: String = conf.getConf(BENCHMARK_TASK_PARTITIONID) - def benchmarkTaskId: String = conf.getConf(BENCHMARK_TASK_TASK_ID) - def benchmarkSaveDir: String = conf.getConf(BENCHMARK_SAVE_DIR) - def textInputMaxBlockSize: Long = conf.getConf(TEXT_INPUT_ROW_MAX_BLOCK_SIZE) - def textIputEmptyAsDefault: Boolean = conf.getConf(TEXT_INPUT_EMPTY_AS_DEFAULT) + getConf(COLUMNAR_VELOX_MEMORY_USE_HUGE_PAGES) + + def debug: Boolean = getConf(DEBUG_ENABLED) + def debugKeepJniWorkspace: Boolean = getConf(DEBUG_KEEP_JNI_WORKSPACE) + def collectUtStats: Boolean = getConf(UT_STATISTIC) + def benchmarkStageId: Int = getConf(BENCHMARK_TASK_STAGEID) + def benchmarkPartitionId: String = getConf(BENCHMARK_TASK_PARTITIONID) + def benchmarkTaskId: String = getConf(BENCHMARK_TASK_TASK_ID) + def benchmarkSaveDir: String = getConf(BENCHMARK_SAVE_DIR) + def textInputMaxBlockSize: Long = getConf(TEXT_INPUT_ROW_MAX_BLOCK_SIZE) + def textIputEmptyAsDefault: Boolean = getConf(TEXT_INPUT_EMPTY_AS_DEFAULT) def enableParquetRowGroupMaxMinIndex: Boolean = - conf.getConf(ENABLE_PARQUET_ROW_GROUP_MAX_MIN_INDEX) + getConf(ENABLE_PARQUET_ROW_GROUP_MAX_MIN_INDEX) def enableVeloxFlushablePartialAggregation: Boolean = - conf.getConf(VELOX_FLUSHABLE_PARTIAL_AGGREGATION_ENABLED) + getConf(VELOX_FLUSHABLE_PARTIAL_AGGREGATION_ENABLED) def maxFlushableAggregationMemoryRatio: Double = - conf.getConf(MAX_PARTIAL_AGGREGATION_MEMORY_RATIO) + getConf(MAX_PARTIAL_AGGREGATION_MEMORY_RATIO) def maxExtendedFlushableAggregationMemoryRatio: Double = - conf.getConf(MAX_PARTIAL_AGGREGATION_MEMORY_RATIO) + getConf(MAX_PARTIAL_AGGREGATION_MEMORY_RATIO) def abandonFlushableAggregationMinPct: Int = - conf.getConf(ABANDON_PARTIAL_AGGREGATION_MIN_PCT) - def abandonFlushableAggregationMinRows: Int = - conf.getConf(ABANDON_PARTIAL_AGGREGATION_MIN_ROWS) + getConf(ABANDON_PARTIAL_AGGREGATION_MIN_PCT) + def abandonFlushableAggregationMinRows: Int = getConf(ABANDON_PARTIAL_AGGREGATION_MIN_ROWS) // Please use `BackendsApiManager.getSettings.enableNativeWriteFiles()` instead - def enableNativeWriter: Option[Boolean] = conf.getConf(NATIVE_WRITER_ENABLED) + def enableNativeWriter: Option[Boolean] = getConf(NATIVE_WRITER_ENABLED) - def enableNativeArrowReader: Boolean = conf.getConf(NATIVE_ARROW_READER_ENABLED) + def enableNativeArrowReader: Boolean = getConf(NATIVE_ARROW_READER_ENABLED) def directorySizeGuess: Long = - conf.getConf(DIRECTORY_SIZE_GUESS) + getConf(DIRECTORY_SIZE_GUESS) def filePreloadThreshold: Long = - conf.getConf(FILE_PRELOAD_THRESHOLD) + getConf(FILE_PRELOAD_THRESHOLD) def prefetchRowGroups: Int = - conf.getConf(PREFETCH_ROW_GROUPS) + getConf(PREFETCH_ROW_GROUPS) def loadQuantum: Long = - conf.getConf(LOAD_QUANTUM) + getConf(LOAD_QUANTUM) def maxCoalescedDistance: String = - conf.getConf(MAX_COALESCED_DISTANCE_BYTES) + getConf(MAX_COALESCED_DISTANCE_BYTES) def maxCoalescedBytes: Long = - conf.getConf(MAX_COALESCED_BYTES) + getConf(MAX_COALESCED_BYTES) def cachePrefetchMinPct: Int = - conf.getConf(CACHE_PREFETCH_MINPCT) + getConf(CACHE_PREFETCH_MINPCT) - def enableColumnarProjectCollapse: Boolean = conf.getConf(ENABLE_COLUMNAR_PROJECT_COLLAPSE) + def enableColumnarProjectCollapse: Boolean = getConf(ENABLE_COLUMNAR_PROJECT_COLLAPSE) - def enableColumnarPartialProject: Boolean = conf.getConf(ENABLE_COLUMNAR_PARTIAL_PROJECT) + def enableColumnarPartialProject: Boolean = getConf(ENABLE_COLUMNAR_PARTIAL_PROJECT) - def awsSdkLogLevel: String = conf.getConf(AWS_SDK_LOG_LEVEL) + def awsSdkLogLevel: String = getConf(AWS_SDK_LOG_LEVEL) - def awsS3RetryMode: String = conf.getConf(AWS_S3_RETRY_MODE) + def awsS3RetryMode: String = getConf(AWS_S3_RETRY_MODE) - def awsConnectionTimeout: String = conf.getConf(AWS_S3_CONNECT_TIMEOUT) + def awsConnectionTimeout: String = getConf(AWS_S3_CONNECT_TIMEOUT) - def enableCastAvgAggregateFunction: Boolean = conf.getConf(COLUMNAR_NATIVE_CAST_AGGREGATE_ENABLED) + def enableCastAvgAggregateFunction: Boolean = getConf(COLUMNAR_NATIVE_CAST_AGGREGATE_ENABLED) def dynamicOffHeapSizingEnabled: Boolean = - conf.getConf(DYNAMIC_OFFHEAP_SIZING_ENABLED) + getConf(DYNAMIC_OFFHEAP_SIZING_ENABLED) + + def enableHiveFileFormatWriter: Boolean = getConf(NATIVE_HIVEFILEFORMAT_WRITER_ENABLED) - def enableHiveFileFormatWriter: Boolean = conf.getConf(NATIVE_HIVEFILEFORMAT_WRITER_ENABLED) + def enableCelebornFallback: Boolean = getConf(CELEBORN_FALLBACK_ENABLED) - def enableCelebornFallback: Boolean = conf.getConf(CELEBORN_FALLBACK_ENABLED) + def enableHdfsViewfs: Boolean = getConf(HDFS_VIEWFS_ENABLED) - def enableHdfsViewfs: Boolean = conf.getConf(HDFS_VIEWFS_ENABLED) + private val configProvider = new ConfigProvider(conf.getAllConfs) + + def getConf[T](entry: ConfigEntry[T]): T = { + require(ConfigEntry.containsEntry(entry), s"$entry is not registered") + entry.readFrom(configProvider) + } } object GlutenConfig { import SQLConf._ - val GLUTEN_ENABLED_BY_DEFAULT = true - val GLUTEN_ENABLED_KEY = "spark.gluten.enabled" + def buildConf(key: String): ConfigBuilder = ConfigBuilder(key) + + def buildStaticConf(key: String): ConfigBuilder = { + ConfigBuilder(key).onCreate(_ => SQLConf.registerStaticConfigKey(key)) + } + val GLUTEN_LIB_NAME = "spark.gluten.sql.columnar.libname" val GLUTEN_LIB_PATH = "spark.gluten.sql.columnar.libpath" val GLUTEN_EXECUTOR_LIB_PATH = "spark.gluten.sql.columnar.executor.libpath" @@ -549,8 +550,6 @@ object GlutenConfig { val SPARK_GCS_AUTH_TYPE: String = HADOOP_PREFIX + GCS_PREFIX + AUTH_TYPE val SPARK_GCS_AUTH_SERVICE_ACCOUNT_JSON_KEYFILE: String = HADOOP_PREFIX + GCS_PREFIX + AUTH_SERVICE_ACCOUNT_JSON_KEYFILE - val SPARK_GCS_MAX_RETRY_COUNT: String = HADOOP_PREFIX + GCS_PREFIX + HTTP_MAX_RETRY_COUNT - val SPARK_GCS_MAX_RETRY_TIME: String = HADOOP_PREFIX + GCS_PREFIX + HTTP_MAX_RETRY_TIME // QAT config val GLUTEN_QAT_BACKEND_NAME = "qat" @@ -564,7 +563,6 @@ object GlutenConfig { // Private Spark configs. val SPARK_ONHEAP_SIZE_KEY = "spark.executor.memory" val SPARK_OVERHEAD_SIZE_KEY = "spark.executor.memoryOverhead" - val SPARK_OVERHEAD_FACTOR_KEY = "spark.executor.memoryOverheadFactor" val SPARK_OFFHEAP_SIZE_KEY = "spark.memory.offHeap.size" val SPARK_OFFHEAP_ENABLED = "spark.memory.offHeap.enabled" val SPARK_REDACTION_REGEX = "spark.redaction.regex" @@ -670,7 +668,7 @@ object GlutenConfig { var ins: GlutenConfig = _ - def getConf: GlutenConfig = { + def get: GlutenConfig = { new GlutenConfig(SQLConf.get) } @@ -857,12 +855,12 @@ object GlutenConfig { } val GLUTEN_ENABLED = - buildConf(GLUTEN_ENABLED_KEY) + buildConf("spark.gluten.enabled") .internal() .doc("Whether to enable gluten. Default value is true. Just an experimental property." + " Recommend to enable/disable Gluten through the setting for spark.plugins.") .booleanConf - .createWithDefault(GLUTEN_ENABLED_BY_DEFAULT) + .createWithDefault(true) // FIXME the option currently controls both JVM and native validation against a Substrait plan. val NATIVE_VALIDATION_ENABLED = diff --git a/shims/common/src/main/scala/org/apache/spark/sql/internal/GlutenConfigUtil.scala b/shims/common/src/main/scala/org/apache/spark/sql/internal/GlutenConfigUtil.scala index 1a45572acdac..a709af161a2e 100644 --- a/shims/common/src/main/scala/org/apache/spark/sql/internal/GlutenConfigUtil.scala +++ b/shims/common/src/main/scala/org/apache/spark/sql/internal/GlutenConfigUtil.scala @@ -16,15 +16,13 @@ */ package org.apache.spark.sql.internal -import org.apache.spark.internal.config.ConfigReader - -import scala.collection.JavaConverters._ +import org.apache.gluten.config._ object GlutenConfigUtil { - private def getConfString(reader: ConfigReader, key: String, value: String): String = { - Option(SQLConf.getConfigEntry(key)) + private def getConfString(configProvider: ConfigProvider, key: String, value: String): String = { + Option(ConfigEntry.findEntry(key)) .map { - _.readFrom(reader) match { + _.readFrom(configProvider) match { case o: Option[_] => o.map(_.toString).getOrElse(value) case null => value case v => v.toString @@ -34,11 +32,11 @@ object GlutenConfigUtil { } def parseConfig(conf: Map[String, String]): Map[String, String] = { - val reader = new ConfigReader(conf.filter(_._1.startsWith("spark.gluten.")).asJava) + val provider = new ConfigProvider(conf.filter(_._1.startsWith("spark.gluten."))) conf.map { case (k, v) => if (k.startsWith("spark.gluten.")) { - (k, getConfString(reader, k, v)) + (k, getConfString(provider, k, v)) } else { (k, v) } diff --git a/shims/spark32/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala b/shims/spark32/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala index a6f066462a1a..ab2c1b28197c 100644 --- a/shims/spark32/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala +++ b/shims/spark32/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala @@ -16,14 +16,14 @@ */ package org.apache.spark.sql.execution.adaptive -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.sql.execution.SparkPlan /** This [[CostEvaluator]] is to force use the new physical plan when cost is equal. */ case class GlutenCostEvaluator() extends CostEvaluator { override def evaluateCost(plan: SparkPlan): Cost = { - if (GlutenConfig.getConf.enableGluten) { + if (GlutenConfig.get.enableGluten) { new GlutenCost(SimpleCostEvaluator, plan) } else { SimpleCostEvaluator.evaluateCost(plan) diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala index 8fcfa735f463..2feb66031751 100644 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala +++ b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.execution.adaptive -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.sql.catalyst.SQLConfHelper import org.apache.spark.sql.execution.SparkPlan @@ -26,7 +26,7 @@ import org.apache.spark.sql.internal.SQLConf case class GlutenCostEvaluator() extends CostEvaluator with SQLConfHelper { override def evaluateCost(plan: SparkPlan): Cost = { val forceOptimizeSkewedJoin = conf.getConf(SQLConf.ADAPTIVE_FORCE_OPTIMIZE_SKEWED_JOIN) - if (GlutenConfig.getConf.enableGluten) { + if (GlutenConfig.get.enableGluten) { new GlutenCost(SimpleCostEvaluator(forceOptimizeSkewedJoin), plan) } else { SimpleCostEvaluator(forceOptimizeSkewedJoin).evaluateCost(plan) diff --git a/shims/spark34/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala b/shims/spark34/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala index 8fcfa735f463..2feb66031751 100644 --- a/shims/spark34/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala +++ b/shims/spark34/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.execution.adaptive -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.sql.catalyst.SQLConfHelper import org.apache.spark.sql.execution.SparkPlan @@ -26,7 +26,7 @@ import org.apache.spark.sql.internal.SQLConf case class GlutenCostEvaluator() extends CostEvaluator with SQLConfHelper { override def evaluateCost(plan: SparkPlan): Cost = { val forceOptimizeSkewedJoin = conf.getConf(SQLConf.ADAPTIVE_FORCE_OPTIMIZE_SKEWED_JOIN) - if (GlutenConfig.getConf.enableGluten) { + if (GlutenConfig.get.enableGluten) { new GlutenCost(SimpleCostEvaluator(forceOptimizeSkewedJoin), plan) } else { SimpleCostEvaluator(forceOptimizeSkewedJoin).evaluateCost(plan) diff --git a/shims/spark35/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala b/shims/spark35/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala index 8fcfa735f463..2feb66031751 100644 --- a/shims/spark35/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala +++ b/shims/spark35/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.sql.execution.adaptive -import org.apache.gluten.GlutenConfig +import org.apache.gluten.config.GlutenConfig import org.apache.spark.sql.catalyst.SQLConfHelper import org.apache.spark.sql.execution.SparkPlan @@ -26,7 +26,7 @@ import org.apache.spark.sql.internal.SQLConf case class GlutenCostEvaluator() extends CostEvaluator with SQLConfHelper { override def evaluateCost(plan: SparkPlan): Cost = { val forceOptimizeSkewedJoin = conf.getConf(SQLConf.ADAPTIVE_FORCE_OPTIMIZE_SKEWED_JOIN) - if (GlutenConfig.getConf.enableGluten) { + if (GlutenConfig.get.enableGluten) { new GlutenCost(SimpleCostEvaluator(forceOptimizeSkewedJoin), plan) } else { SimpleCostEvaluator(forceOptimizeSkewedJoin).evaluateCost(plan)