From d92445381e98e5c8e7138c5f3e52b37e1f60a272 Mon Sep 17 00:00:00 2001 From: jiangshequan Date: Fri, 13 Dec 2019 18:22:20 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E9=85=8D=E7=BD=AEaccurateCar?= =?UTF-8?q?dinality=E7=AE=97=E6=B3=95=E5=9C=A8handler=E4=B8=AD=E4=BD=BF?= =?UTF-8?q?=E7=94=A8=E4=B8=AD=E6=98=AF=E5=90=A6=E5=BC=80=E5=90=AFoneId?= =?UTF-8?q?=E6=9C=8D=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../apache/hadoop/hive/druid/DruidStorageHandlerUtils.java | 7 +++++++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/bigo-druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java b/bigo-druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java index ccc9a1a09cec..7db83829ab5b 100644 --- a/bigo-druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java +++ b/bigo-druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java @@ -1038,6 +1038,8 @@ public static FieldTypeEnum getFieldType(String fieldName) { } int k = Integer.parseInt(HiveConf.getVar(jc, HiveConf.ConfVars.HIVE_DRUID_QUANTILES_PARAM_K)); + String nameSpace = HiveConf.getVar(jc, HiveConf.ConfVars.HIVE_DRUID_ACCRUATE_CARDINALITY_NAMESPACE); + String openOneId = HiveConf.getVar(jc, HiveConf.ConfVars.HIVE_DRUID_ACCRUATE_CARDINALITY_OPEN_ONEID); LOG.info("hive.druid.quantiles.k {}", k); String druidHllTgtType = getTableProperty(tableProperties, jc, @@ -1096,6 +1098,11 @@ public static FieldTypeEnum getFieldType(String fieldName) { aggregatorFactoryBuilder.add(new OldSketchBuildAggregatorFactory(dColumnName, dColumnName, size)); continue; + } else if (fieldTypeEnum == FieldTypeEnum.ACC) { + LOG.info("column " + dColumnName + " treat as acc metric"); + aggregatorFactoryBuilder.add(new AccurateCardinalityAggregatorFactory(dColumnName, + dColumnName, nameSpace, openOneId)); + continue; } final PrimitiveObjectInspector.PrimitiveCategory diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index bea3635bcc68..ed30dc9dd2fd 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1960,6 +1960,12 @@ public static enum ConfVars { new PatternSet("-1", "1048576","524288","262144", "131072", "65536", "32768", "16384", "8192", "4096", "2048", "1024", "512", "256", "128", "64", "32"), "https://datasketches.github.io/docs/Theta/ThetaSize.html" ), + HIVE_DRUID_ACCRUATE_CARDINALITY_NAMESPACE("hive.druid.accurate.cardinality.namespace", "hdid", + "the namespace for oneId service in Druid, Only using and working when hive.druid.accurate.cardinality.open.oneid set to true" + ), + HIVE_DRUID_ACCRUATE_CARDINALITY_OPEN_ONEID("hive.druid.accurate.cardinality.open.oneid", "false", + "If open oneId service in Druid when using hive2Druid" + ), HIVE_DRUID_MAX_PARTITION_SIZE("hive.druid.indexer.partition.size.max", 5000000, "Maximum number of records per segment partition" ),