From 7aa2441467529603f2353a167ca1c2f97c5e2a59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Wa=C5=9B?= Date: Thu, 7 Sep 2023 13:07:56 +0200 Subject: [PATCH 1/5] Rename singlenode-hdp3 test env to better reflect its usage --- ...enodeHdp3.java => EnvSinglenodeHiveAcid.java} | 16 ++++------------ .../env/environment/EnvSinglenodeSparkHive.java | 4 ++-- .../launcher/suite/suites/Suite8NonGeneric.java | 4 ++-- .../apply-hive-config.sh} | 2 +- .../hive-site-overrides.xml | 0 5 files changed, 9 insertions(+), 17 deletions(-) rename testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/{EnvSinglenodeHdp3.java => EnvSinglenodeHiveAcid.java} (75%) rename testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/{singlenode-hdp3/apply-hdp3-config.sh => singlenode-hive-acid/apply-hive-config.sh} (64%) rename testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/{singlenode-hdp3 => singlenode-hive-acid}/hive-site-overrides.xml (100%) diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeHdp3.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeHiveAcid.java similarity index 75% rename from testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeHdp3.java rename to testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeHiveAcid.java index 74cb4a16f153..75690e03de28 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeHdp3.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeHiveAcid.java @@ -17,7 +17,6 @@ import com.google.inject.Inject; import io.trino.tests.product.launcher.docker.DockerFiles; import io.trino.tests.product.launcher.env.Environment; -import io.trino.tests.product.launcher.env.EnvironmentConfig; import io.trino.tests.product.launcher.env.EnvironmentProvider; import io.trino.tests.product.launcher.env.common.Hadoop; import io.trino.tests.product.launcher.env.common.Standard; @@ -30,33 +29,26 @@ import static java.util.Objects.requireNonNull; import static org.testcontainers.utility.MountableFile.forHostPath; -// HDP 3.1 images (code) + HDP 3.1-like configuration. -// See https://github.com/trinodb/trino/issues/1841 for more information. @TestsEnvironment -public class EnvSinglenodeHdp3 +public class EnvSinglenodeHiveAcid extends EnvironmentProvider { private final DockerFiles dockerFiles; - private final String hadoopImagesVersion; @Inject - protected EnvSinglenodeHdp3(DockerFiles dockerFiles, Standard standard, Hadoop hadoop, EnvironmentConfig environmentConfig) + protected EnvSinglenodeHiveAcid(DockerFiles dockerFiles, Standard standard, Hadoop hadoop) { super(ImmutableList.of(standard, hadoop)); this.dockerFiles = requireNonNull(dockerFiles, "dockerFiles is null"); - this.hadoopImagesVersion = environmentConfig.getHadoopImagesVersion(); } @Override public void extendEnvironment(Environment.Builder builder) { - String dockerImageName = "ghcr.io/trinodb/testing/hdp3.1-hive:" + hadoopImagesVersion; - builder.configureContainer(HADOOP, dockerContainer -> { - dockerContainer.setDockerImageName(dockerImageName); dockerContainer.withCopyFileToContainer( - forHostPath(dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-hdp3/apply-hdp3-config.sh")), - CONTAINER_HADOOP_INIT_D + "apply-hdp3-config.sh"); + forHostPath(dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-hive-acid/apply-hive-config.sh")), + CONTAINER_HADOOP_INIT_D + "apply-hive-config.sh"); }); builder.configureContainer(TESTS, dockerContainer -> { diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkHive.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkHive.java index 8fb0a46f9f05..41658ae53703 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkHive.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkHive.java @@ -61,8 +61,8 @@ public void extendEnvironment(Environment.Builder builder) builder.configureContainer(HADOOP, dockerContainer -> { dockerContainer.setDockerImageName(dockerImageName); dockerContainer.withCopyFileToContainer( - forHostPath(dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-hdp3/apply-hdp3-config.sh")), - CONTAINER_HADOOP_INIT_D + "apply-hdp3-config.sh"); + forHostPath(dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-hive-acid/apply-hive-config.sh")), + CONTAINER_HADOOP_INIT_D + "apply-hive-config.sh"); }); builder.configureContainer(TESTS, dockerContainer -> { diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite8NonGeneric.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite8NonGeneric.java index f4e4e0d00361..be6a4c7860e3 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite8NonGeneric.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/Suite8NonGeneric.java @@ -16,7 +16,7 @@ import com.google.common.collect.ImmutableList; import io.trino.tests.product.launcher.env.EnvironmentConfig; import io.trino.tests.product.launcher.env.EnvironmentDefaults; -import io.trino.tests.product.launcher.env.environment.EnvSinglenodeHdp3; +import io.trino.tests.product.launcher.env.environment.EnvSinglenodeHiveAcid; import io.trino.tests.product.launcher.suite.Suite; import io.trino.tests.product.launcher.suite.SuiteTestRun; @@ -34,7 +34,7 @@ public List getTestRuns(EnvironmentConfig config) verify(config.getHadoopBaseImage().equals(EnvironmentDefaults.HADOOP_BASE_IMAGE), "The suite should be run with default HADOOP_BASE_IMAGE. Leave HADOOP_BASE_IMAGE unset."); return ImmutableList.of( - testOnEnvironment(EnvSinglenodeHdp3.class) + testOnEnvironment(EnvSinglenodeHiveAcid.class) .withGroups("configured_features", "hdp3_only", "hive_transactional") .build()); } diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hdp3/apply-hdp3-config.sh b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-acid/apply-hive-config.sh similarity index 64% rename from testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hdp3/apply-hdp3-config.sh rename to testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-acid/apply-hive-config.sh index f0c60845b0a2..b29bcc092b1f 100755 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hdp3/apply-hdp3-config.sh +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-acid/apply-hive-config.sh @@ -2,4 +2,4 @@ set -exuo pipefail echo "Applying HDP3 hive-site configuration overrides" -apply-site-xml-override /etc/hive/conf/hive-site.xml "/docker/presto-product-tests/conf/environment/singlenode-hdp3/hive-site-overrides.xml" +apply-site-xml-override /etc/hive/conf/hive-site.xml "/docker/presto-product-tests/conf/environment/singlenode-hive-acid/hive-site-overrides.xml" diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hdp3/hive-site-overrides.xml b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-acid/hive-site-overrides.xml similarity index 100% rename from testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hdp3/hive-site-overrides.xml rename to testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-acid/hive-site-overrides.xml From 3d364f4efe308e0be75f10c3912f383de1a9ea10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Wa=C5=9B?= Date: Wed, 13 Sep 2023 09:48:45 +0200 Subject: [PATCH 2/5] Disable renewing Kerberos tickets when running against KDC on CentOS7 --- .../conf/environment/two-kerberos-hives/presto-krb5.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-kerberos-hives/presto-krb5.conf b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-kerberos-hives/presto-krb5.conf index 4b9e021504a8..0dc73408a2c9 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-kerberos-hives/presto-krb5.conf +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-kerberos-hives/presto-krb5.conf @@ -8,7 +8,8 @@ dns_lookup_realm = false dns_lookup_kdc = false ticket_lifetime = 24h - renew_lifetime = 7d + # this setting is causing a Message stream modified (41) error when talking to KDC running on CentOS 7: https://stackoverflow.com/a/60978520 + # renew_lifetime = 7d forwardable = true [realms] From 1d5aab935e668639f20c440de1c7a8ea56f30bd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Wa=C5=9B?= Date: Tue, 12 Sep 2023 10:44:33 +0200 Subject: [PATCH 3/5] Move testing from HDP 2.6 to HDP 3.1 --- .../java/io/trino/filesystem/hdfs/Hadoop.java | 2 +- .../plugin/hive/TestHive2OnDataLake.java | 25 ------- .../TestHiveAnalyzeCorruptStatistics.java | 6 +- .../plugin/hive/containers/HiveHadoop.java | 3 +- .../hive/containers/HiveMinioDataLake.java | 2 +- pom.xml | 2 +- .../launcher/env/EnvironmentDefaults.java | 2 +- .../product/launcher/env/common/Hadoop.java | 6 +- .../env/common/HadoopKerberosKms.java | 5 +- .../HadoopKerberosKmsWithImpersonation.java | 2 +- .../env/configs/ConfigApacheHive3.java | 2 +- .../launcher/env/configs/ConfigHdp3.java | 6 -- .../EnvSinglenodeDeltaLakeOss.java | 11 +-- .../environment/EnvSinglenodeHiveAcid.java | 8 --- .../environment/EnvSinglenodeSparkHive.java | 13 +--- .../EnvSinglenodeSparkIceberg.java | 13 +--- .../EnvSinglenodeSparkIcebergJdbcCatalog.java | 13 +--- .../env/environment/EnvTwoKerberosHives.java | 5 +- .../env/environment/EnvTwoMixedHives.java | 9 ++- .../two-kerberos-hives/update-location.sh | 9 +++ .../hadoop-master-2/core-site.xml | 72 +++++++++++++++++++ .../hadoop-master-2/hdfs-site.xml | 23 ++++++ .../hadoop-master-2/update-location.sh | 9 +++ .../tempto-configuration-for-hive3.yaml | 11 --- .../main/resources/tempto-configuration.yaml | 4 +- 25 files changed, 149 insertions(+), 114 deletions(-) delete mode 100644 plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHive2OnDataLake.java create mode 100644 testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-kerberos-hives/update-location.sh create mode 100644 testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-mixed-hives/hadoop-master-2/hdfs-site.xml create mode 100644 testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-mixed-hives/hadoop-master-2/update-location.sh delete mode 100644 testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/tempto/tempto-configuration-for-hive3.yaml diff --git a/lib/trino-hdfs/src/test/java/io/trino/filesystem/hdfs/Hadoop.java b/lib/trino-hdfs/src/test/java/io/trino/filesystem/hdfs/Hadoop.java index c31c6fea2dd4..8b7b3ec66d0c 100644 --- a/lib/trino-hdfs/src/test/java/io/trino/filesystem/hdfs/Hadoop.java +++ b/lib/trino-hdfs/src/test/java/io/trino/filesystem/hdfs/Hadoop.java @@ -32,7 +32,7 @@ public class Hadoop { private static final Logger log = Logger.get(Hadoop.class); - private static final String IMAGE = "ghcr.io/trinodb/testing/hdp2.6-hive:" + getDockerImagesVersion(); + private static final String IMAGE = "ghcr.io/trinodb/testing/hdp3.1-hive:" + getDockerImagesVersion(); private static final int HDFS_PORT = 9000; diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHive2OnDataLake.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHive2OnDataLake.java deleted file mode 100644 index b55338274da4..000000000000 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHive2OnDataLake.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.hive; - -import io.trino.plugin.hive.containers.HiveHadoop; - -public class TestHive2OnDataLake - extends BaseTestHiveOnDataLake -{ - public TestHive2OnDataLake() - { - super(HiveHadoop.DEFAULT_IMAGE); - } -} diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveAnalyzeCorruptStatistics.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveAnalyzeCorruptStatistics.java index ff1d8992eff3..ea34c08e924a 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveAnalyzeCorruptStatistics.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveAnalyzeCorruptStatistics.java @@ -57,7 +57,7 @@ public void testAnalyzeCorruptColumnStatisticsOnEmptyTable() // ANALYZE and drop_stats are unsupported for tables having broken column statistics assertThatThrownBy(() -> query("ANALYZE " + tableName)) - .hasMessage("%s: Socket is closed by peer.", hiveMinioDataLake.getHiveHadoop().getHiveMetastoreEndpoint()) + .hasMessage("Unexpected 2 statistics for 1 columns") .hasStackTraceContaining("ThriftHiveMetastore.setTableColumnStatistics"); assertThatThrownBy(() -> query("CALL system.drop_stats('tpch', '" + tableName + "')")) @@ -73,7 +73,7 @@ private void prepareBrokenColumnStatisticsTable(String tableName) // Insert duplicated row to simulate broken column statistics status https://github.com/trinodb/trino/issues/13787 assertEquals(onMetastore("SELECT COUNT(1) FROM TAB_COL_STATS WHERE db_name = 'tpch' AND table_name = '" + tableName + "'"), "1"); onMetastore("INSERT INTO TAB_COL_STATS " + - "SELECT cs_id + 1, db_name, table_name, column_name, column_type, tbl_id, long_low_value, long_high_value, double_high_value, double_low_value, big_decimal_low_value, big_decimal_high_value, num_nulls, num_distincts, avg_col_len, max_col_len, num_trues, num_falses, last_analyzed " + + "SELECT cs_id + 1, cat_name, db_name, table_name, column_name, column_type, tbl_id, long_low_value, long_high_value, double_high_value, double_low_value, big_decimal_low_value, big_decimal_high_value, num_nulls, num_distincts, bit_vector, avg_col_len, max_col_len, num_trues, num_falses, last_analyzed " + "FROM TAB_COL_STATS WHERE db_name = 'tpch' AND table_name = '" + tableName + "'"); assertEquals(onMetastore("SELECT COUNT(1) FROM TAB_COL_STATS WHERE db_name = 'tpch' AND table_name = '" + tableName + "'"), "2"); } @@ -104,7 +104,7 @@ private void prepareBrokenPartitionStatisticsTable(String tableName) // Insert duplicated row to simulate broken partition statistics status https://github.com/trinodb/trino/issues/13787 assertEquals(onMetastore("SELECT COUNT(1) FROM PART_COL_STATS WHERE db_name = 'tpch' AND table_name = '" + tableName + "'"), "1"); onMetastore("INSERT INTO PART_COL_STATS " + - "SELECT cs_id + 1, db_name, table_name, partition_name, column_name, column_type, part_id, long_low_value, long_high_value, double_high_value, double_low_value, big_decimal_low_value, big_decimal_high_value, num_nulls, num_distincts, avg_col_len, max_col_len, num_trues, num_falses, last_analyzed " + + "SELECT cs_id + 1, cat_name, db_name, table_name, partition_name, column_name, column_type, part_id, long_low_value, long_high_value, double_high_value, double_low_value, big_decimal_low_value, big_decimal_high_value, num_nulls, num_distincts, bit_vector, avg_col_len, max_col_len, num_trues, num_falses, last_analyzed " + "FROM PART_COL_STATS WHERE db_name = 'tpch' AND table_name = '" + tableName + "'"); assertEquals(onMetastore("SELECT COUNT(1) FROM PART_COL_STATS WHERE db_name = 'tpch' AND table_name = '" + tableName + "'"), "2"); } diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/HiveHadoop.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/HiveHadoop.java index 977c88f5e0b6..7959030b8661 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/HiveHadoop.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/HiveHadoop.java @@ -33,7 +33,6 @@ public class HiveHadoop { private static final Logger log = Logger.get(HiveHadoop.class); - public static final String DEFAULT_IMAGE = "ghcr.io/trinodb/testing/hdp2.6-hive:" + TestingProperties.getDockerImagesVersion(); public static final String HIVE3_IMAGE = "ghcr.io/trinodb/testing/hdp3.1-hive:" + TestingProperties.getDockerImagesVersion(); public static final String HOST_NAME = "hadoop-master"; @@ -104,7 +103,7 @@ public static class Builder { private Builder() { - this.image = DEFAULT_IMAGE; + this.image = HIVE3_IMAGE; this.hostName = HOST_NAME; this.exposePorts = ImmutableSet.of(HIVE_METASTORE_PORT); } diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/HiveMinioDataLake.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/HiveMinioDataLake.java index 15e7f0178450..2e8bba014380 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/HiveMinioDataLake.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/containers/HiveMinioDataLake.java @@ -51,7 +51,7 @@ public class HiveMinioDataLake public HiveMinioDataLake(String bucketName) { - this(bucketName, HiveHadoop.DEFAULT_IMAGE); + this(bucketName, HiveHadoop.HIVE3_IMAGE); } public HiveMinioDataLake(String bucketName, String hiveHadoopImage) diff --git a/pom.xml b/pom.xml index 95b98e911248..271c98b3c66a 100644 --- a/pom.xml +++ b/pom.xml @@ -189,7 +189,7 @@ 1.37 5.10.0 - 81 + 86 + + hadoop.proxyuser.oozie.hosts + * + + + hadoop.proxyuser.oozie.groups + * + + + + + hadoop.proxyuser.httpfs.hosts + * + + + hadoop.proxyuser.httpfs.groups + * + + + + + hadoop.proxyuser.llama.hosts + * + + + hadoop.proxyuser.llama.groups + * + + + + + hadoop.proxyuser.hue.hosts + * + + + hadoop.proxyuser.hue.groups + * + + + + + hadoop.proxyuser.mapred.hosts + * + + + hadoop.proxyuser.mapred.groups + * + + + + + hadoop.proxyuser.hive.hosts + * + + + + hadoop.proxyuser.hive.groups + * + + + + + hadoop.proxyuser.hdfs.groups + * + + + + hadoop.proxyuser.hdfs.hosts + * + diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-mixed-hives/hadoop-master-2/hdfs-site.xml b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-mixed-hives/hadoop-master-2/hdfs-site.xml new file mode 100644 index 000000000000..c8f55aff9808 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-mixed-hives/hadoop-master-2/hdfs-site.xml @@ -0,0 +1,23 @@ + + + + dfs.namenode.name.dir + /var/lib/hadoop-hdfs/cache/name/ + + + + dfs.datanode.data.dir + /var/lib/hadoop-hdfs/cache/data/ + + + + fs.viewfs.mounttable.hadoop-viewfs.link./default + hdfs://hadoop-master-2:9000/user/hive/warehouse + + + + + dfs.safemode.threshold.pct + 0 + + diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-mixed-hives/hadoop-master-2/update-location.sh b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-mixed-hives/hadoop-master-2/update-location.sh new file mode 100644 index 000000000000..d0802cb5c08e --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-mixed-hives/hadoop-master-2/update-location.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +/usr/bin/mysqld_safe & +while ! mysqladmin ping -proot --silent; do sleep 1; done + +hive --service metatool -updateLocation hdfs://hadoop-master-2:9000/user/hive/warehouse hdfs://hadoop-master:9000/user/hive/warehouse + +killall mysqld +while pgrep mysqld; do sleep 1; done diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/tempto/tempto-configuration-for-hive3.yaml b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/tempto/tempto-configuration-for-hive3.yaml deleted file mode 100644 index 9142bbddb72f..000000000000 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/tempto/tempto-configuration-for-hive3.yaml +++ /dev/null @@ -1,11 +0,0 @@ -hdfs: - webhdfs: - # 9870 is the name node's default port in Hadoop 3 - uri: http://${databases.hive.host}:9870 - -databases: - hive: - prepare_statement: - - USE ${databases.hive.schema} - # Hive 3 gathers stats by default. For test purposes we need to disable this behavior. - - SET hive.stats.column.autogather=false diff --git a/testing/trino-product-tests/src/main/resources/tempto-configuration.yaml b/testing/trino-product-tests/src/main/resources/tempto-configuration.yaml index 839c23c7b112..45989ef16dfa 100644 --- a/testing/trino-product-tests/src/main/resources/tempto-configuration.yaml +++ b/testing/trino-product-tests/src/main/resources/tempto-configuration.yaml @@ -1,7 +1,7 @@ hdfs: username: hive webhdfs: - uri: http://${databases.hive.host}:50070 + uri: http://${databases.hive.host}:9870 databases: hive: @@ -14,6 +14,8 @@ databases: schema: default prepare_statement: - USE ${databases.hive.schema} + # Hive 3 gathers stats by default. For test purposes we need to disable this behavior. + - SET hive.stats.column.autogather=false table_manager_type: hive warehouse_directory_path: /user/hive/warehouse inject_stats_for_immutable_tables: true From de36bd6d78db8b3ea608e8f2b9dbc3f98ffe80b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Wa=C5=9B?= Date: Thu, 7 Sep 2023 10:28:29 +0200 Subject: [PATCH 4/5] Remove ConfigHdp3 since it's same as the default config --- .github/workflows/ci.yml | 92 ++++--------------- .../product/launcher/env/common/Hadoop.java | 2 + .../launcher/env/configs/ConfigHdp3.java | 62 ------------- .../env/environment/EnvMultinodeGcs.java | 1 - .../EnvSinglenodeHiveHudiRedirections.java | 3 - .../env/environment/EnvSinglenodeHudi.java | 2 - ...EnvSinglenodeSparkHiveNoStatsFallback.java | 1 - .../common/hadoop/hive.properties | 2 + .../hadoop/hive_timestamp_nanos.properties | 2 + .../hive_with_external_writes.properties | 2 + .../hive_with_run_view_as_invoker.properties | 2 + .../common/standard/presto-init-hdp3.sh | 10 -- .../multinode-azure/hive.properties | 2 + .../hive-coordinator.properties | 2 + .../multinode-cached/hive-worker.properties | 2 + .../environment/multinode-gcs/hive.properties | 2 + .../multinode-minio-data-lake/hive.properties | 2 + .../multinode-tls-kerberos/hive.properties | 2 + .../hive-hadoop2.properties | 2 + .../singlenode-compatibility/hive.properties | 2 + .../hive.properties | 2 + .../singlenode-delta-lake-oss/hive.properties | 2 + .../hive.properties | 2 + .../hive.properties | 2 + .../hive.properties | 2 + .../hive.properties | 2 + .../singlenode-hudi/hive.properties | 2 + .../hive.properties | 2 + .../hive.properties | 2 + .../hive.properties | 2 + .../hive.properties | 2 + .../hive.properties | 2 + .../hive.properties | 2 + .../hive.properties | 2 + .../hive.properties | 2 + .../hive.properties | 2 + .../hive.properties | 2 + .../hive.properties | 2 + .../hive.properties | 2 + .../hive.properties | 1 + .../singlenode-spark-hive/hive.properties | 1 + .../two-kerberos-hives/hive1.properties | 2 + .../two-kerberos-hives/hive2.properties | 2 + .../two-mixed-hives/hive1.properties | 2 + .../two-mixed-hives/hive2.properties | 2 + testing/trino-product-tests/README.md | 6 +- 46 files changed, 92 insertions(+), 159 deletions(-) delete mode 100644 testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/configs/ConfigHdp3.java delete mode 100755 testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/standard/presto-init-hdp3.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a88b369b843..33f1539ee658 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -902,14 +902,15 @@ jobs: cat < .github/test-pt-matrix.yaml config: - default - - hdp3 - # TODO: config-apache-hive3 suite: - suite-1 - suite-2 - suite-3 # suite-4 does not exist - suite-5 + - suite-6-non-generic + - suite-7-non-generic + - suite-8-non-generic - suite-azure - suite-delta-lake-databricks91 - suite-delta-lake-databricks104 @@ -920,18 +921,22 @@ jobs: - suite-clients - suite-functions - suite-tpch + - suite-tpcds - suite-storage-formats-detailed + - suite-parquet + - suite-oauth2 + - suite-ldap + - suite-compatibility + - suite-all-connectors-smoke + - suite-delta-lake-oss + - suite-kafka + - suite-cassandra + - suite-clickhouse + - suite-mysql + - suite-iceberg + - suite-hudi + - suite-ignite exclude: - - config: default - ignore exclusion if: >- - ${{ github.event_name != 'pull_request' - || github.event.pull_request.head.repo.full_name == github.repository - || contains(github.event.pull_request.labels.*.name, 'tests:all') - || contains(github.event.pull_request.labels.*.name, 'tests:hive') - }} - - - suite: suite-azure - config: default - suite: suite-azure ignore exclusion if: >- ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || @@ -939,34 +944,22 @@ jobs: secrets.AZURE_ABFS_ACCOUNT != '' || secrets.AZURE_ABFS_ACCESSKEY != '' }} - - suite: suite-gcs - config: default - suite: suite-gcs ignore exclusion if: >- ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.GCP_CREDENTIALS_KEY != '' }} - - suite: suite-delta-lake-databricks91 - config: hdp3 - suite: suite-delta-lake-databricks91 ignore exclusion if: >- ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.DATABRICKS_TOKEN != '' }} - - suite: suite-delta-lake-databricks104 - config: hdp3 - suite: suite-delta-lake-databricks104 ignore exclusion if: >- ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.DATABRICKS_TOKEN != '' }} - - suite: suite-delta-lake-databricks113 - config: hdp3 - suite: suite-delta-lake-databricks113 ignore exclusion if: >- ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.DATABRICKS_TOKEN != '' }} - - suite: suite-delta-lake-databricks122 - config: hdp3 - suite: suite-delta-lake-databricks122 ignore exclusion if: >- ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.DATABRICKS_TOKEN != '' }} - - suite: suite-delta-lake-databricks133 - config: hdp3 - suite: suite-delta-lake-databricks133 ignore exclusion if: >- ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.DATABRICKS_TOKEN != '' }} @@ -983,60 +976,9 @@ jobs: # value of the property, and the exclusion will apply normally. - "false" include: - # this suite is not meant to be run with different configs - - config: default - suite: suite-6-non-generic - # this suite is not meant to be run with different configs - - config: default - suite: suite-7-non-generic - # this suite is not meant to be run with different configs - - config: default - suite: suite-8-non-generic - # this suite is not meant to be run with different configs - - config: default - suite: suite-tpcds - # this suite is not meant to be run with different configs - - config: default - suite: suite-parquet - # this suite is not meant to be run with different configs - - config: default - suite: suite-oauth2 - # this suite is not meant to be run with different configs - - config: default - suite: suite-ldap - # this suite is not meant to be run with different configs - - config: default - suite: suite-compatibility # this suite is designed specifically for apache-hive3. TODO remove the suite once we can run all regular tests on apache-hive3. - config: apache-hive3 suite: suite-hms-only - # this suite is not meant to be run with different configs - - config: default - suite: suite-all-connectors-smoke - # this suite is not meant to be run with different configs - - config: default - suite: suite-delta-lake-oss - # this suite is not meant to be run with different configs - - config: default - suite: suite-kafka - # this suite is not meant to be run with different configs - - config: default - suite: suite-cassandra - # this suite is not meant to be run with different configs - - config: default - suite: suite-clickhouse - # this suite is not meant to be run with different configs - - config: default - suite: suite-mysql - # this suite is not meant to be run with different configs - - config: default - suite: suite-iceberg - # this suite is not meant to be run with different configs - - config: default - suite: suite-hudi - # this suite is not meant to be run with different configs - - config: default - suite: suite-ignite EOF - name: Build PT matrix (all) if: | diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/Hadoop.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/Hadoop.java index e08d630a4899..f404ddc15de3 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/Hadoop.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/Hadoop.java @@ -81,6 +81,8 @@ public static DockerContainer createHadoopContainer(DockerFiles dockerFiles, Por .withCopyFileToContainer(forHostPath(dockerFiles.getDockerFilesHostPath("health-checks/hadoop-health-check.sh")), CONTAINER_HEALTH_D + "hadoop-health-check.sh") .withCopyFileToContainer(forHostPath(dockerFiles.getDockerFilesHostPath("common/hadoop/hadoop-run.sh")), "/usr/local/hadoop-run.sh") .withCopyFileToContainer(forHostPath(dockerFiles.getDockerFilesHostPath("common/hadoop/apply-config-overrides.sh")), CONTAINER_HADOOP_INIT_D + "00-apply-config-overrides.sh") + // When hive performs implicit coercion to/from timestamp for ORC files, it depends on timezone of the HiveServer + .withEnv("TZ", "UTC") .withCommand("/usr/local/hadoop-run.sh") .withExposedLogPaths("/var/log/hadoop-yarn", "/var/log/hadoop-hdfs", "/var/log/hive", "/var/log/container-health.log") .withStartupCheckStrategy(new IsRunningStartupCheckStrategy()) diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/configs/ConfigHdp3.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/configs/ConfigHdp3.java deleted file mode 100644 index c4c32aa4c7b5..000000000000 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/configs/ConfigHdp3.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.tests.product.launcher.env.configs; - -import com.google.inject.Inject; -import io.trino.tests.product.launcher.docker.DockerFiles; -import io.trino.tests.product.launcher.env.Environment; - -import static io.trino.tests.product.launcher.env.EnvironmentContainers.HADOOP; -import static io.trino.tests.product.launcher.env.EnvironmentContainers.TRINO; -import static java.util.Objects.requireNonNull; -import static org.testcontainers.utility.MountableFile.forHostPath; - -public class ConfigHdp3 - extends ConfigDefault -{ - private final DockerFiles dockerFiles; - - @Inject - public ConfigHdp3(DockerFiles dockerFiles) - { - this.dockerFiles = requireNonNull(dockerFiles, "dockerFiles is null"); - } - - /** - * export HADOOP_BASE_IMAGE="ghcr.io/trinodb/testing/hdp3.1-hive" - * export TEMPTO_ENVIRONMENT_CONFIG_FILE="/docker/presto-product-tests/conf/tempto/tempto-configuration-for-hive3.yaml" - * export DISTRO_SKIP_GROUP=iceberg - */ - @Override - public String getHadoopBaseImage() - { - return "ghcr.io/trinodb/testing/hdp3.1-hive"; - } - - @Override - public void extendEnvironment(Environment.Builder builder) - { - builder.configureContainers(container -> { - if (container.getLogicalName().startsWith(TRINO)) { - container.withCopyFileToContainer(forHostPath( - // HDP3's handling of timestamps is incompatible with previous versions of Hive (see https://issues.apache.org/jira/browse/HIVE-21002); - // in order for Trino to deal with the differences, we must set catalog properties for Parquet and RCFile - dockerFiles.getDockerFilesHostPath("common/standard/presto-init-hdp3.sh")), - "/docker/presto-init.d/presto-init-hdp3.sh"); - } - }); - // When hive performs implicit coercion to/from timestamp for ORC files, it depends on timezone of the HiveServer - builder.configureContainer(HADOOP, container -> container.withEnv("TZ", "UTC")); - } -} diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeGcs.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeGcs.java index ca85d9238fe1..9fbb25e61866 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeGcs.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeGcs.java @@ -87,7 +87,6 @@ public void extendEnvironment(Environment.Builder builder) String containerGcpCredentialsFile = CONTAINER_TRINO_ETC + "gcp-credentials.json"; builder.configureContainer(HADOOP, container -> { - container.setDockerImageName("ghcr.io/trinodb/testing/hdp3.1-hive:" + hadoopImageVersion); container.withCopyFileToContainer( forHostPath(getCoreSiteOverrideXml(containerGcpCredentialsFile)), "/docker/presto-product-tests/conf/environment/multinode-gcs/core-site-overrides.xml"); diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeHiveHudiRedirections.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeHiveHudiRedirections.java index 21977f9f2715..52dae080594e 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeHiveHudiRedirections.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeHiveHudiRedirections.java @@ -75,9 +75,6 @@ public EnvSinglenodeHiveHudiRedirections( @Override public void extendEnvironment(Environment.Builder builder) { - // Using hdp3.1 so we are using Hive metastore with version close to versions of hive-*.jars Spark uses - builder.configureContainer(HADOOP, container -> container.setDockerImageName("ghcr.io/trinodb/testing/hdp3.1-hive:" + hadoopImagesVersion)); - builder.addConnector("hive", forHostPath(configDir.getPath("hive.properties"))); builder.addConnector("hudi", forHostPath(configDir.getPath("hudi.properties"))); diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeHudi.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeHudi.java index 0da2e8e9af58..134aaa54934e 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeHudi.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeHudi.java @@ -80,8 +80,6 @@ public EnvSinglenodeHudi( @Override public void extendEnvironment(Environment.Builder builder) { - // Using hdp3.1 so we are using Hive metastore with version close to versions of hive-*.jars Spark uses - builder.configureContainer(HADOOP, container -> container.setDockerImageName("ghcr.io/trinodb/testing/hdp3.1-hive:" + hadoopImagesVersion)); builder.addConnector( "hive", forHostPath(configDir.getPath("hive.properties")), diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkHiveNoStatsFallback.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkHiveNoStatsFallback.java index 29bd1ef632c4..6f0555c367c8 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkHiveNoStatsFallback.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkHiveNoStatsFallback.java @@ -55,7 +55,6 @@ public EnvSinglenodeSparkHiveNoStatsFallback(Standard standard, Hadoop hadoop, D @Override public void extendEnvironment(Environment.Builder builder) { - builder.configureContainer(HADOOP, container -> container.setDockerImageName("ghcr.io/trinodb/testing/hdp3.1-hive:" + hadoopImagesVersion)); builder.addConnector("hive", forHostPath(configDir.getPath("hive.properties"))); builder.addContainer(createSpark()).containerDependsOn("spark", HADOOP); } diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/hadoop/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/hadoop/hive.properties index d0ffc37b843a..1a5ada2f085a 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/hadoop/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/hadoop/hive.properties @@ -14,3 +14,5 @@ hive.fs.cache.max-size=10 hive.max-partitions-per-scan=100 hive.max-partitions-for-eager-load=100 hive.hive-views.enabled=true +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/hadoop/hive_timestamp_nanos.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/hadoop/hive_timestamp_nanos.properties index 9ca0bbb8b3df..9c5164373090 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/hadoop/hive_timestamp_nanos.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/hadoop/hive_timestamp_nanos.properties @@ -5,3 +5,5 @@ hive.allow-drop-table=true hive.metastore-cache-ttl=0s hive.hive-views.enabled=true hive.timestamp-precision=NANOSECONDS +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/hadoop/hive_with_external_writes.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/hadoop/hive_with_external_writes.properties index 83662e70e4a6..9b25bca84b2a 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/hadoop/hive_with_external_writes.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/hadoop/hive_with_external_writes.properties @@ -15,3 +15,5 @@ hive.max-partitions-per-scan=100 hive.max-partitions-for-eager-load=100 hive.hive-views.enabled=true hive.non-managed-table-writes-enabled=true +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/hadoop/hive_with_run_view_as_invoker.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/hadoop/hive_with_run_view_as_invoker.properties index 440d35a7b07e..e094cbcf1c2a 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/hadoop/hive_with_run_view_as_invoker.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/hadoop/hive_with_run_view_as_invoker.properties @@ -6,3 +6,5 @@ hive.fs.cache.max-size=10 hive.hive-views.enabled=true hive.hive-views.run-as-invoker=true hive.security=sql-standard +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/standard/presto-init-hdp3.sh b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/standard/presto-init-hdp3.sh deleted file mode 100755 index 932b9fe2a572..000000000000 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/standard/presto-init-hdp3.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -set -xeuo pipefail - -for hive_properties in $(find '/docker/presto-product-tests/conf/presto/etc/catalog' -name '*hive*.properties'); do - echo "Updating $hive_properties for HDP3" - # Add file format time zone properties - echo "hive.parquet.time-zone=UTC" >> "${hive_properties}" - echo "hive.rcfile.time-zone=UTC" >> "${hive_properties}" -done diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-azure/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-azure/hive.properties index 8a9f623b07ec..139e457a87db 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-azure/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-azure/hive.properties @@ -11,3 +11,5 @@ hive.allow-comment-table=true hive.allow-drop-table=true hive.allow-rename-table=true hive.translate-hive-views=true +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-cached/hive-coordinator.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-cached/hive-coordinator.properties index d4d3e56ba08d..8a5d028c9e34 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-cached/hive-coordinator.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-cached/hive-coordinator.properties @@ -3,3 +3,5 @@ hive.config.resources=/docker/presto-product-tests/conf/presto/etc/hive-default- hive.metastore.uri=thrift://hadoop-master:9083 hive.allow-drop-table=true hive.cache.enabled=true +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-cached/hive-worker.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-cached/hive-worker.properties index d5dc79022610..4bee1ef360a7 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-cached/hive-worker.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-cached/hive-worker.properties @@ -4,3 +4,5 @@ hive.metastore.uri=thrift://hadoop-master:9083 hive.allow-drop-table=true hive.cache.enabled=true hive.cache.location=/tmp/cache +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-gcs/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-gcs/hive.properties index 32ca8d9482d8..921d1b8a73d2 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-gcs/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-gcs/hive.properties @@ -9,3 +9,5 @@ hive.allow-rename-column=true hive.allow-comment-table=true hive.allow-drop-table=true hive.allow-rename-table=true +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-minio-data-lake/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-minio-data-lake/hive.properties index 9c4b9cf36f06..1852ad915eda 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-minio-data-lake/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-minio-data-lake/hive.properties @@ -6,3 +6,5 @@ hive.s3.endpoint=http://minio:9080/ hive.s3.path-style-access=true hive.s3.ssl.enabled=false hive.allow-drop-table=true +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-tls-kerberos/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-tls-kerberos/hive.properties index 52c85f4632cb..007bc8691c12 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-tls-kerberos/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-tls-kerberos/hive.properties @@ -9,6 +9,8 @@ hive.metastore-cache-ttl=0s hive.allow-add-column=true hive.allow-drop-column=true hive.allow-rename-column=true +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC hive.metastore.authentication.type=KERBEROS hive.metastore.service.principal=hive/hadoop-master@LABS.TERADATA.COM diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-compatibility/hive-hadoop2.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-compatibility/hive-hadoop2.properties index eca055ccdeab..59e6560302fb 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-compatibility/hive-hadoop2.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-compatibility/hive-hadoop2.properties @@ -1,2 +1,4 @@ connector.name=hive-hadoop2 hive.metastore.uri=thrift://hadoop-master:9083 +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-compatibility/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-compatibility/hive.properties index d9fe1819cbc9..934cffb83c6e 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-compatibility/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-compatibility/hive.properties @@ -1,2 +1,4 @@ connector.name=hive hive.metastore.uri=thrift://hadoop-master:9083 +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-delta-lake-databricks/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-delta-lake-databricks/hive.properties index 1a207291a5ed..82d96d95b948 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-delta-lake-databricks/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-delta-lake-databricks/hive.properties @@ -11,3 +11,5 @@ hive.allow-comment-table=true hive.allow-comment-column=true hive.allow-rename-table=true hive.delta-lake-catalog-name=delta +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-delta-lake-oss/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-delta-lake-oss/hive.properties index b7c12af22503..fe3951947a23 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-delta-lake-oss/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-delta-lake-oss/hive.properties @@ -13,3 +13,5 @@ hive.allow-comment-table=true hive.allow-comment-column=true hive.allow-rename-table=true hive.delta-lake-catalog-name=delta +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hdfs-impersonation/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hdfs-impersonation/hive.properties index f3794109bcb5..cd177028fb42 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hdfs-impersonation/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hdfs-impersonation/hive.properties @@ -15,3 +15,5 @@ hive.fs.cache.max-size=10 hive.max-partitions-per-scan=100 hive.max-partitions-for-eager-load=100 hive.non-managed-table-writes-enabled=true +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-hudi-redirections/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-hudi-redirections/hive.properties index 3e565f9bde02..ab04e3cbd014 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-hudi-redirections/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-hudi-redirections/hive.properties @@ -9,3 +9,5 @@ hive.allow-drop-table=true hive.allow-rename-table=true hive.hudi-catalog-name=hudi hive.hive-views.enabled=true +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-iceberg-redirections/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-iceberg-redirections/hive.properties index 8c8bd114f10c..00343c8845c1 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-iceberg-redirections/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-iceberg-redirections/hive.properties @@ -9,3 +9,5 @@ hive.allow-drop-table=true hive.allow-rename-table=true hive.iceberg-catalog-name=iceberg hive.hive-views.enabled=true +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-impersonation/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-impersonation/hive.properties index dd977ce5226a..42d444e5a717 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-impersonation/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-impersonation/hive.properties @@ -17,3 +17,5 @@ hive.fs.cache.max-size=10 hive.max-partitions-per-scan=100 hive.max-partitions-for-eager-load=100 hive.non-managed-table-writes-enabled=true +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hudi/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hudi/hive.properties index 0920bd77bc76..7d020ed0235b 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hudi/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hudi/hive.properties @@ -2,3 +2,5 @@ connector.name=hive hive.metastore.uri=thrift://hadoop-master:9083 hive.config.resources=/docker/presto-product-tests/conf/presto/etc/hive-default-fs-site.xml hive.allow-drop-table=true +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation-cross-realm/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation-cross-realm/hive.properties index 100f9af6b5aa..05cb0a2144a8 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation-cross-realm/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation-cross-realm/hive.properties @@ -15,6 +15,8 @@ hive.metastore-cache-ttl=0s hive.allow-add-column=true hive.allow-drop-column=true hive.allow-rename-column=true +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC hive.metastore.authentication.type=KERBEROS hive.metastore.service.principal=hive/hadoop-master@LABS.TERADATA.COM diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation-with-data-protection/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation-with-data-protection/hive.properties index 3b30e924111c..b580e2c05fd4 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation-with-data-protection/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation-with-data-protection/hive.properties @@ -2,6 +2,8 @@ connector.name=hive hive.metastore.uri=thrift://hadoop-master:9083 hive.config.resources=/docker/presto-product-tests/conf/presto/etc/hive-default-fs-site.xml,/docker/presto-product-tests/conf/presto/etc/hive-data-protection-site.xml hive.metastore-cache-ttl=0s +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC hive.metastore.authentication.type=KERBEROS hive.metastore.service.principal=hive/hadoop-master@LABS.TERADATA.COM diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation-with-wire-encryption/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation-with-wire-encryption/hive.properties index f3c23d43660b..1a9a43cef24b 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation-with-wire-encryption/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation-with-wire-encryption/hive.properties @@ -2,6 +2,8 @@ connector.name=hive hive.metastore.uri=thrift://hadoop-master:9083 hive.config.resources=/docker/presto-product-tests/conf/presto/etc/hive-default-fs-site.xml hive.metastore-cache-ttl=0s +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC hive.metastore.authentication.type=KERBEROS hive.metastore.service.principal=hive/hadoop-master@LABS.TERADATA.COM diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation/hive.properties index e05fe7a37af1..b55e8077bc12 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation/hive.properties @@ -2,6 +2,8 @@ connector.name=hive hive.metastore.uri=thrift://hadoop-master:9083 hive.config.resources=/docker/presto-product-tests/conf/presto/etc/hive-default-fs-site.xml hive.metastore-cache-ttl=0s +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC hive.metastore.authentication.type=KERBEROS hive.metastore.service.principal=hive/hadoop-master@LABS.TERADATA.COM diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-no-impersonation/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-no-impersonation/hive.properties index 9dc23a5de5ac..c139292f2925 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-no-impersonation/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-no-impersonation/hive.properties @@ -8,6 +8,8 @@ hive.metastore-cache-ttl=0s hive.allow-add-column=true hive.allow-drop-column=true hive.allow-rename-column=true +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC hive.metastore.authentication.type=KERBEROS hive.metastore.service.principal=hive/hadoop-master@LABS.TERADATA.COM diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-impersonation-with-credential-cache/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-impersonation-with-credential-cache/hive.properties index e90065ef200f..17e2033e5e7d 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-impersonation-with-credential-cache/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-impersonation-with-credential-cache/hive.properties @@ -2,6 +2,8 @@ connector.name=hive hive.metastore.uri=thrift://hadoop-master:9083 hive.config.resources=/docker/presto-product-tests/conf/presto/etc/hive-default-fs-site.xml hive.metastore-cache-ttl=0s +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC hive.metastore.authentication.type=KERBEROS hive.metastore.thrift.impersonation.enabled=true diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-impersonation/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-impersonation/hive.properties index b367e08a0885..e81984f40414 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-impersonation/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-impersonation/hive.properties @@ -2,6 +2,8 @@ connector.name=hive hive.metastore.uri=thrift://hadoop-master:9083 hive.config.resources=/docker/presto-product-tests/conf/presto/etc/hive-default-fs-site.xml hive.metastore-cache-ttl=0s +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC hive.metastore.authentication.type=KERBEROS hive.metastore.thrift.impersonation.enabled=true diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-no-impersonation-with-credential-cache/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-no-impersonation-with-credential-cache/hive.properties index 97b3a12bbc94..16f65d30066e 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-no-impersonation-with-credential-cache/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-no-impersonation-with-credential-cache/hive.properties @@ -8,6 +8,8 @@ hive.metastore-cache-ttl=0s hive.allow-add-column=true hive.allow-drop-column=true hive.allow-rename-column=true +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC hive.metastore.authentication.type=KERBEROS hive.metastore.service.principal=hive/hadoop-master@LABS.TERADATA.COM diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-impersonation-with-credential-cache/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-impersonation-with-credential-cache/hive.properties index e2bd095360e9..c21e95701370 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-impersonation-with-credential-cache/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-impersonation-with-credential-cache/hive.properties @@ -5,6 +5,8 @@ hive.allow-rename-table=true hive.allow-add-column=true hive.allow-drop-column=true hive.allow-rename-column=true +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC hive.metastore.authentication.type=KERBEROS hive.metastore.service.principal=hive/_HOST@LABS.TERADATA.COM diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-impersonation/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-impersonation/hive.properties index 100538ce71a4..964548264701 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-impersonation/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-impersonation/hive.properties @@ -5,6 +5,8 @@ hive.allow-rename-table=true hive.allow-add-column=true hive.allow-drop-column=true hive.allow-rename-column=true +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC hive.metastore.authentication.type=KERBEROS hive.metastore.service.principal=hive/_HOST@LABS.TERADATA.COM diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-no-impersonation-with-credential-cache/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-no-impersonation-with-credential-cache/hive.properties index 62db20623271..5958c78979be 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-no-impersonation-with-credential-cache/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-no-impersonation-with-credential-cache/hive.properties @@ -5,6 +5,8 @@ hive.allow-rename-table=true hive.allow-add-column=true hive.allow-drop-column=true hive.allow-rename-column=true +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC hive.metastore.authentication.type=KERBEROS hive.metastore.service.principal=hive/_HOST@LABS.TERADATA.COM diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-no-impersonation/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-no-impersonation/hive.properties index 93ae094ed300..08c6e5af715c 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-no-impersonation/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-no-impersonation/hive.properties @@ -5,6 +5,8 @@ hive.allow-rename-table=true hive.allow-add-column=true hive.allow-drop-column=true hive.allow-rename-column=true +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC hive.metastore.authentication.type=KERBEROS hive.metastore.service.principal=hive/_HOST@LABS.TERADATA.COM diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-spark-hive-no-stats-fallback/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-spark-hive-no-stats-fallback/hive.properties index 66e861f1fc52..155ce80fc4cf 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-spark-hive-no-stats-fallback/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-spark-hive-no-stats-fallback/hive.properties @@ -6,3 +6,4 @@ hive.allow-drop-table=true # Note: it's currently unclear why this one is needed, while also hive.orc.time-zone=UTC is not needed. hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-spark-hive/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-spark-hive/hive.properties index 25bfc1eabf97..43ab687153f5 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-spark-hive/hive.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-spark-hive/hive.properties @@ -5,3 +5,4 @@ hive.allow-drop-table=true # Note: it's currently unclear why this one is needed, while also hive.orc.time-zone=UTC is not needed. hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-kerberos-hives/hive1.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-kerberos-hives/hive1.properties index 28b4d97243bb..a4f1d8374acc 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-kerberos-hives/hive1.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-kerberos-hives/hive1.properties @@ -2,6 +2,8 @@ connector.name=hive hive.metastore.uri=thrift://hadoop-master:9083 hive.config.resources=/docker/presto-product-tests/conf/presto/etc/hive-default-fs-site.xml hive.metastore-cache-ttl=0s +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC hive.metastore.authentication.type=KERBEROS hive.metastore.thrift.impersonation.enabled=true diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-kerberos-hives/hive2.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-kerberos-hives/hive2.properties index 955111047d84..bd2616ce9ba2 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-kerberos-hives/hive2.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-kerberos-hives/hive2.properties @@ -3,6 +3,8 @@ hive.metastore.uri=thrift://hadoop-master-2:9083 hive.config.resources=/docker/presto-product-tests/conf/environment/two-kerberos-hives/hive2-default-fs-site.xml,\ /docker/presto-product-tests/conf/environment/two-kerberos-hives/auth-to-local.xml hive.metastore-cache-ttl=0s +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC hive.metastore.authentication.type=KERBEROS hive.metastore.service.principal=hive/hadoop-master-2@OTHERREALM.COM diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-mixed-hives/hive1.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-mixed-hives/hive1.properties index 28b4d97243bb..a4f1d8374acc 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-mixed-hives/hive1.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-mixed-hives/hive1.properties @@ -2,6 +2,8 @@ connector.name=hive hive.metastore.uri=thrift://hadoop-master:9083 hive.config.resources=/docker/presto-product-tests/conf/presto/etc/hive-default-fs-site.xml hive.metastore-cache-ttl=0s +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC hive.metastore.authentication.type=KERBEROS hive.metastore.thrift.impersonation.enabled=true diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-mixed-hives/hive2.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-mixed-hives/hive2.properties index 172f0cad19d0..b01bac39ace1 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-mixed-hives/hive2.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-mixed-hives/hive2.properties @@ -11,3 +11,5 @@ hive.metastore-cache-ttl=0s hive.fs.cache.max-size=10 hive.max-partitions-per-scan=100 hive.max-partitions-for-eager-load=100 +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC diff --git a/testing/trino-product-tests/README.md b/testing/trino-product-tests/README.md index f8c30a763ffd..03a219b44fe2 100644 --- a/testing/trino-product-tests/README.md +++ b/testing/trino-product-tests/README.md @@ -104,11 +104,7 @@ testing/bin/ptl env list #### Environment config -Most of the Hadoop-based environments can be run in multiple configurations that use different Hadoop distribution: - -- **config-default** - executes tests against vanilla Hadoop distribution -- **config-hdp3** - executes tests against HDP3 distribution of Hadoop - +Most of the Hadoop-based environments can be run in multiple configurations. You can obtain list of available environment configurations using command: ``` From 8753d532170dac53268c9b6dec7f425e98bf4820 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Wa=C5=9B?= Date: Wed, 13 Sep 2023 13:55:09 +0200 Subject: [PATCH 5/5] Format long insert queries in Hive tests --- .../TestHiveAnalyzeCorruptStatistics.java | 61 +++++++++++++++++-- 1 file changed, 55 insertions(+), 6 deletions(-) diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveAnalyzeCorruptStatistics.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveAnalyzeCorruptStatistics.java index ea34c08e924a..ea0315b280ca 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveAnalyzeCorruptStatistics.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveAnalyzeCorruptStatistics.java @@ -72,9 +72,33 @@ private void prepareBrokenColumnStatisticsTable(String tableName) // Insert duplicated row to simulate broken column statistics status https://github.com/trinodb/trino/issues/13787 assertEquals(onMetastore("SELECT COUNT(1) FROM TAB_COL_STATS WHERE db_name = 'tpch' AND table_name = '" + tableName + "'"), "1"); - onMetastore("INSERT INTO TAB_COL_STATS " + - "SELECT cs_id + 1, cat_name, db_name, table_name, column_name, column_type, tbl_id, long_low_value, long_high_value, double_high_value, double_low_value, big_decimal_low_value, big_decimal_high_value, num_nulls, num_distincts, bit_vector, avg_col_len, max_col_len, num_trues, num_falses, last_analyzed " + - "FROM TAB_COL_STATS WHERE db_name = 'tpch' AND table_name = '" + tableName + "'"); + onMetastore(""" + INSERT INTO TAB_COL_STATS + SELECT + cs_id + 1, + cat_name, + db_name, + table_name, + column_name, + column_type, + tbl_id, + long_low_value, + long_high_value, + double_high_value, + double_low_value, + big_decimal_low_value, + big_decimal_high_value, + num_nulls, + num_distincts, + bit_vector, + avg_col_len, + max_col_len, + num_trues, + num_falses, + last_analyzed + FROM TAB_COL_STATS + WHERE db_name = 'tpch' AND table_name = '%s' + """.formatted(tableName)); assertEquals(onMetastore("SELECT COUNT(1) FROM TAB_COL_STATS WHERE db_name = 'tpch' AND table_name = '" + tableName + "'"), "2"); } @@ -103,9 +127,34 @@ private void prepareBrokenPartitionStatisticsTable(String tableName) // Insert duplicated row to simulate broken partition statistics status https://github.com/trinodb/trino/issues/13787 assertEquals(onMetastore("SELECT COUNT(1) FROM PART_COL_STATS WHERE db_name = 'tpch' AND table_name = '" + tableName + "'"), "1"); - onMetastore("INSERT INTO PART_COL_STATS " + - "SELECT cs_id + 1, cat_name, db_name, table_name, partition_name, column_name, column_type, part_id, long_low_value, long_high_value, double_high_value, double_low_value, big_decimal_low_value, big_decimal_high_value, num_nulls, num_distincts, bit_vector, avg_col_len, max_col_len, num_trues, num_falses, last_analyzed " + - "FROM PART_COL_STATS WHERE db_name = 'tpch' AND table_name = '" + tableName + "'"); + onMetastore(""" + INSERT INTO PART_COL_STATS + SELECT + cs_id + 1, + cat_name, + db_name, + table_name, + partition_name, + column_name, + column_type, + part_id, + long_low_value, + long_high_value, + double_high_value, + double_low_value, + big_decimal_low_value, + big_decimal_high_value, + num_nulls, + num_distincts, + bit_vector, + avg_col_len, + max_col_len, + num_trues, + num_falses, + last_analyzed + FROM PART_COL_STATS + WHERE db_name = 'tpch' AND table_name = '%s' + """.formatted(tableName)); assertEquals(onMetastore("SELECT COUNT(1) FROM PART_COL_STATS WHERE db_name = 'tpch' AND table_name = '" + tableName + "'"), "2"); }