Skip to content

Commit 2af4085

Browse files
Nitin-KashyapNitin-Kashyap
committed
[fix](meta) show partitions with Limit for external HMS tables (27835)
Issue:apache#27834
1 parent 66cfcc6 commit 2af4085

File tree

5 files changed

+126
-8
lines changed

5 files changed

+126
-8
lines changed

‎fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPartitionsStmt.java

+10-1
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ public class ShowPartitionsStmt extends ShowStmt {
5858
private static final Logger LOG = LogManager.getLogger(ShowPartitionsStmt.class);
5959

6060
private static final String FILTER_PARTITION_ID = "PartitionId";
61-
private static final String FILTER_PARTITION_NAME = "PartitionName";
61+
public static final String FILTER_PARTITION_NAME = "PartitionName";
6262
private static final String FILTER_STATE = "State";
6363
private static final String FILTER_BUCKETS = "Buckets";
6464
private static final String FILTER_REPLICATION_NUM = "ReplicationNum";
@@ -198,6 +198,11 @@ public void analyzeImpl(Analyzer analyzer) throws UserException {
198198
throw new AnalysisException("Should order by column");
199199
}
200200
SlotRef slotRef = (SlotRef) orderByElement.getExpr();
201+
if (catalog instanceof HMSExternalCatalog
202+
&& !slotRef.getColumnName().equalsIgnoreCase(FILTER_PARTITION_NAME)) {
203+
throw new AnalysisException("External table only support Order By on PartitionName");
204+
}
205+
201206
int index = PartitionsProcDir.analyzeColumn(slotRef.getColumnName());
202207
OrderByPair orderByPair = new OrderByPair(index, !orderByElement.getIsAsc());
203208
orderByPairs.add(orderByPair);
@@ -228,6 +233,10 @@ private void analyzeSubPredicate(Expr subExpr) throws AnalysisException {
228233
}
229234

230235
String leftKey = ((SlotRef) subExpr.getChild(0)).getColumnName();
236+
if (catalog instanceof HMSExternalCatalog && !leftKey.equalsIgnoreCase(FILTER_PARTITION_NAME)) {
237+
throw new AnalysisException(String.format("Only %s column supported in where clause for this catalog",
238+
FILTER_PARTITION_NAME));
239+
}
231240
if (subExpr instanceof BinaryPredicate) {
232241
BinaryPredicate binaryPredicate = (BinaryPredicate) subExpr;
233242
if (leftKey.equalsIgnoreCase(FILTER_PARTITION_NAME) || leftKey.equalsIgnoreCase(FILTER_STATE)) {

‎fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java

+3-2
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,8 @@ public PartitionsProcDir(Database db, OlapTable olapTable, boolean isTempPartiti
8181
this.isTempPartition = isTempPartition;
8282
}
8383

84-
public boolean filter(String columnName, Comparable element, Map<String, Expr> filterMap) throws AnalysisException {
84+
public static boolean filter(String columnName, Comparable element, Map<String, Expr> filterMap)
85+
throws AnalysisException {
8586
if (filterMap == null) {
8687
return true;
8788
}
@@ -142,7 +143,7 @@ public boolean filter(String columnName, Comparable element, Map<String, Expr> f
142143
return true;
143144
}
144145

145-
public boolean like(String str, String expr) {
146+
public static boolean like(String str, String expr) {
146147
expr = expr.toLowerCase();
147148
expr = expr.replace(".", "\\.");
148149
expr = expr.replace("?", ".");

‎fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java

+31-5
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.apache.doris.analysis.AdminShowReplicaStatusStmt;
2525
import org.apache.doris.analysis.AdminShowTabletStorageFormatStmt;
2626
import org.apache.doris.analysis.DescribeStmt;
27+
import org.apache.doris.analysis.Expr;
2728
import org.apache.doris.analysis.HelpStmt;
2829
import org.apache.doris.analysis.LimitElement;
2930
import org.apache.doris.analysis.PartitionNames;
@@ -1691,28 +1692,53 @@ private void handleShowMaxComputeTablePartitions(ShowPartitionsStmt showStmt) {
16911692
resultSet = new ShowResultSet(showStmt.getMetaData(), rows);
16921693
}
16931694

1694-
private void handleShowHMSTablePartitions(ShowPartitionsStmt showStmt) {
1695+
private void handleShowHMSTablePartitions(ShowPartitionsStmt showStmt) throws AnalysisException {
16951696
HMSExternalCatalog catalog = (HMSExternalCatalog) (showStmt.getCatalog());
16961697
List<List<String>> rows = new ArrayList<>();
16971698
String dbName = ClusterNamespace.getNameFromFullName(showStmt.getTableName().getDb());
16981699

16991700
List<String> partitionNames;
17001701
LimitElement limit = showStmt.getLimitElement();
1701-
if (limit != null && limit.hasLimit()) {
1702-
// only limit is valid on Hive
1702+
Map<String, Expr> filterMap = showStmt.getFilterMap();
1703+
List<OrderByPair> orderByPairs = showStmt.getOrderByPairs();
1704+
1705+
if (limit != null && limit.hasLimit() && limit.getOffset() == 0
1706+
&& (orderByPairs == null || !orderByPairs.get(0).isDesc())) {
1707+
// hmsClient returns unordered partition list, hence if offset > 0 cannot pass limit
17031708
partitionNames = catalog.getClient()
1704-
.listPartitionNames(dbName, showStmt.getTableName().getTbl(), limit.getLimit());
1709+
.listPartitionNames(dbName, showStmt.getTableName().getTbl(), limit.getLimit());
17051710
} else {
17061711
partitionNames = catalog.getClient().listPartitionNames(dbName, showStmt.getTableName().getTbl());
17071712
}
1713+
1714+
/* Filter add rows */
17081715
for (String partition : partitionNames) {
17091716
List<String> list = new ArrayList<>();
1717+
1718+
if (filterMap != null && !filterMap.isEmpty()) {
1719+
if (!PartitionsProcDir.filter(ShowPartitionsStmt.FILTER_PARTITION_NAME, partition, filterMap)) {
1720+
continue;
1721+
}
1722+
}
17101723
list.add(partition);
17111724
rows.add(list);
17121725
}
17131726

17141727
// sort by partition name
1715-
rows.sort(Comparator.comparing(x -> x.get(0)));
1728+
if (orderByPairs != null && orderByPairs.get(0).isDesc()) {
1729+
rows.sort(Comparator.comparing(x -> x.get(0), Comparator.reverseOrder()));
1730+
} else {
1731+
rows.sort(Comparator.comparing(x -> x.get(0)));
1732+
}
1733+
1734+
if (limit != null && limit.hasLimit()) {
1735+
int beginIndex = (int) limit.getOffset();
1736+
int endIndex = (int) (beginIndex + limit.getLimit());
1737+
if (endIndex > rows.size()) {
1738+
endIndex = rows.size();
1739+
}
1740+
rows = rows.subList(beginIndex, endIndex);
1741+
}
17161742

17171743
resultSet = new ShowResultSet(showStmt.getMetaData(), rows);
17181744
}

‎regression-test/data/external_table_p0/hive/test_hive_partitions.out

+49
Original file line numberDiff line numberDiff line change
@@ -71,3 +71,52 @@
7171
55 1.22zxy
7272
56 1.22ZXY
7373

74+
-- !q11 --
75+
nation=cn/city=beijing
76+
nation=cn/city=shanghai
77+
nation=jp/city=tokyo
78+
nation=rus/city=moscow
79+
nation=us/city=chicago
80+
nation=us/city=washington
81+
82+
-- !q12 --
83+
nation=cn/city=beijing
84+
85+
-- !q13 --
86+
nation=us/city=chicago
87+
nation=us/city=washington
88+
89+
-- !q14 --
90+
nation=rus/city=moscow
91+
nation=us/city=chicago
92+
nation=us/city=washington
93+
94+
-- !q16 --
95+
nation=cn/city=beijing
96+
nation=cn/city=shanghai
97+
nation=jp/city=tokyo
98+
99+
-- !q17 --
100+
nation=jp/city=tokyo
101+
nation=rus/city=moscow
102+
nation=us/city=chicago
103+
104+
-- !q18 --
105+
nation=us/city=chicago
106+
nation=us/city=washington
107+
108+
-- !q19 --
109+
nation=rus/city=moscow
110+
nation=jp/city=tokyo
111+
nation=cn/city=shanghai
112+
113+
-- !q20 --
114+
nation=cn/city=beijing
115+
nation=cn/city=shanghai
116+
nation=jp/city=tokyo
117+
nation=rus/city=moscow
118+
nation=us/city=chicago
119+
nation=us/city=washington
120+
121+
-- !q21 --
122+

‎regression-test/suites/external_table_p0/hive/test_hive_partitions.groovy

+33
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,39 @@ suite("test_hive_partitions", "p0,external,hive,external_docker,external_docker_
3535
select id, data from table_with_pars where dt_par = '2023-02-01' and time_par = '2023-02-01 01:30:00'
3636
and decimal_par1 = '1' and decimal_par2 = '1.2' and decimal_par3 = '1.22' order by id;
3737
"""
38+
qt_q11 """
39+
show partitions from partition_table;
40+
"""
41+
qt_q12 """
42+
show partitions from partition_table WHERE partitionName='nation=cn/city=beijing';
43+
"""
44+
qt_q13 """
45+
show partitions from partition_table WHERE partitionName like 'nation=us/%';
46+
"""
47+
qt_q14 """
48+
show partitions from partition_table WHERE partitionName like 'nation=%us%';
49+
"""
50+
qt_q16 """
51+
show partitions from partition_table LIMIT 3;
52+
"""
53+
qt_q17 """
54+
show partitions from partition_table LIMIT 3 OFFSET 2;
55+
"""
56+
qt_q18 """
57+
show partitions from partition_table LIMIT 3 OFFSET 4;
58+
"""
59+
qt_q19 """
60+
show partitions from partition_table ORDER BY partitionName desc LIMIT 3 OFFSET 2;
61+
"""
62+
qt_q20 """
63+
show partitions from partition_table ORDER BY partitionName asc;
64+
"""
65+
qt_q21 """
66+
show partitions from partition_table
67+
WHERE partitionName like '%X%'
68+
ORDER BY partitionName DESC
69+
LIMIT 1;
70+
"""
3871
}
3972

4073
String enabled = context.config.otherConfigs.get("enableHiveTest")

0 commit comments

Comments
 (0)