-
Notifications
You must be signed in to change notification settings - Fork 28.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[SPARK-30724][SQL] Support 'LIKE ANY' and 'LIKE ALL' operators
### What changes were proposed in this pull request? `LIKE ANY/SOME` and `LIKE ALL` operators are mostly used when we are matching a text field with numbers of patterns. For example: Teradata / Hive 3.0 / Snowflake: ```sql --like any select 'foo' LIKE ANY ('%foo%','%bar%'); --like all select 'foo' LIKE ALL ('%foo%','%bar%'); ``` PostgreSQL: ```sql -- like any select 'foo' LIKE ANY (array['%foo%','%bar%']); -- like all select 'foo' LIKE ALL (array['%foo%','%bar%']); ``` This PR add support these two operators. More details: https://docs.teradata.com/reader/756LNiPSFdY~4JcCCcR5Cw/4~AyrPNmDN0Xk4SALLo6aQ https://issues.apache.org/jira/browse/HIVE-15229 https://docs.snowflake.net/manuals/sql-reference/functions/like_any.html ### Why are the changes needed? To smoothly migrate SQLs to Spark SQL. ### Does this PR introduce any user-facing change? No ### How was this patch tested? Unit test. Closes #27477 from wangyum/SPARK-30724. Authored-by: Yuming Wang <yumwang@ebay.com> Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
- Loading branch information
Showing
7 changed files
with
405 additions
and
10 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,39 @@ | ||
CREATE OR REPLACE TEMPORARY VIEW like_all_table AS SELECT * FROM (VALUES | ||
('google', '%oo%'), | ||
('facebook', '%oo%'), | ||
('linkedin', '%in')) | ||
as t1(company, pat); | ||
|
||
SELECT company FROM like_all_table WHERE company LIKE ALL ('%oo%', '%go%'); | ||
|
||
SELECT company FROM like_all_table WHERE company LIKE ALL ('microsoft', '%yoo%'); | ||
|
||
SELECT | ||
company, | ||
CASE | ||
WHEN company LIKE ALL ('%oo%', '%go%') THEN 'Y' | ||
ELSE 'N' | ||
END AS is_available, | ||
CASE | ||
WHEN company LIKE ALL ('%oo%', 'go%') OR company LIKE ALL ('%in', 'ms%') THEN 'Y' | ||
ELSE 'N' | ||
END AS mix | ||
FROM like_all_table ; | ||
|
||
-- Mix test with constant pattern and column value | ||
SELECT company FROM like_all_table WHERE company LIKE ALL ('%oo%', pat); | ||
|
||
-- not like all test | ||
SELECT company FROM like_all_table WHERE company NOT LIKE ALL ('%oo%', '%in', 'fa%'); | ||
SELECT company FROM like_all_table WHERE company NOT LIKE ALL ('microsoft', '%yoo%'); | ||
SELECT company FROM like_all_table WHERE company NOT LIKE ALL ('%oo%', 'fa%'); | ||
SELECT company FROM like_all_table WHERE NOT company LIKE ALL ('%oo%', 'fa%'); | ||
|
||
-- null test | ||
SELECT company FROM like_all_table WHERE company LIKE ALL ('%oo%', NULL); | ||
SELECT company FROM like_all_table WHERE company NOT LIKE ALL ('%oo%', NULL); | ||
SELECT company FROM like_all_table WHERE company LIKE ALL (NULL, NULL); | ||
SELECT company FROM like_all_table WHERE company NOT LIKE ALL (NULL, NULL); | ||
|
||
-- negative case | ||
SELECT company FROM like_any_table WHERE company LIKE ALL (); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,39 @@ | ||
CREATE OR REPLACE TEMPORARY VIEW like_any_table AS SELECT * FROM (VALUES | ||
('google', '%oo%'), | ||
('facebook', '%oo%'), | ||
('linkedin', '%in')) | ||
as t1(company, pat); | ||
|
||
SELECT company FROM like_any_table WHERE company LIKE ANY ('%oo%', '%in', 'fa%'); | ||
|
||
SELECT company FROM like_any_table WHERE company LIKE ANY ('microsoft', '%yoo%'); | ||
|
||
select | ||
company, | ||
CASE | ||
WHEN company LIKE ANY ('%oo%', '%in', 'fa%') THEN 'Y' | ||
ELSE 'N' | ||
END AS is_available, | ||
CASE | ||
WHEN company LIKE ANY ('%oo%', 'fa%') OR company LIKE ANY ('%in', 'ms%') THEN 'Y' | ||
ELSE 'N' | ||
END AS mix | ||
FROM like_any_table; | ||
|
||
-- Mix test with constant pattern and column value | ||
SELECT company FROM like_any_table WHERE company LIKE ANY ('%zz%', pat); | ||
|
||
-- not like any test | ||
SELECT company FROM like_any_table WHERE company NOT LIKE ANY ('%oo%', '%in', 'fa%'); | ||
SELECT company FROM like_any_table WHERE company NOT LIKE ANY ('microsoft', '%yoo%'); | ||
SELECT company FROM like_any_table WHERE company NOT LIKE ANY ('%oo%', 'fa%'); | ||
SELECT company FROM like_any_table WHERE NOT company LIKE ANY ('%oo%', 'fa%'); | ||
|
||
-- null test | ||
SELECT company FROM like_any_table WHERE company LIKE ANY ('%oo%', NULL); | ||
SELECT company FROM like_any_table WHERE company NOT LIKE ANY ('%oo%', NULL); | ||
SELECT company FROM like_any_table WHERE company LIKE ANY (NULL, NULL); | ||
SELECT company FROM like_any_table WHERE company NOT LIKE ANY (NULL, NULL); | ||
|
||
-- negative case | ||
SELECT company FROM like_any_table WHERE company LIKE ANY (); |
140 changes: 140 additions & 0 deletions
140
sql/core/src/test/resources/sql-tests/results/like-all.sql.out
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,140 @@ | ||
-- Automatically generated by SQLQueryTestSuite | ||
-- Number of queries: 14 | ||
|
||
|
||
-- !query | ||
CREATE OR REPLACE TEMPORARY VIEW like_all_table AS SELECT * FROM (VALUES | ||
('google', '%oo%'), | ||
('facebook', '%oo%'), | ||
('linkedin', '%in')) | ||
as t1(company, pat) | ||
-- !query schema | ||
struct<> | ||
-- !query output | ||
|
||
|
||
|
||
-- !query | ||
SELECT company FROM like_all_table WHERE company LIKE ALL ('%oo%', '%go%') | ||
-- !query schema | ||
struct<company:string> | ||
-- !query output | ||
|
||
|
||
-- !query | ||
SELECT company FROM like_all_table WHERE company LIKE ALL ('microsoft', '%yoo%') | ||
-- !query schema | ||
struct<company:string> | ||
-- !query output | ||
|
||
|
||
|
||
-- !query | ||
SELECT | ||
company, | ||
CASE | ||
WHEN company LIKE ALL ('%oo%', '%go%') THEN 'Y' | ||
ELSE 'N' | ||
END AS is_available, | ||
CASE | ||
WHEN company LIKE ALL ('%oo%', 'go%') OR company LIKE ALL ('%in', 'ms%') THEN 'Y' | ||
ELSE 'N' | ||
END AS mix | ||
FROM like_all_table | ||
-- !query schema | ||
struct<company:string,is_available:string,mix:string> | ||
-- !query output | ||
facebook N N | ||
google Y Y | ||
linkedin N N | ||
|
||
|
||
-- !query | ||
SELECT company FROM like_all_table WHERE company LIKE ALL ('%oo%', pat) | ||
-- !query schema | ||
struct<company:string> | ||
-- !query output | ||
|
||
|
||
-- !query | ||
SELECT company FROM like_all_table WHERE company NOT LIKE ALL ('%oo%', '%in', 'fa%') | ||
-- !query schema | ||
struct<company:string> | ||
-- !query output | ||
|
||
|
||
|
||
-- !query | ||
SELECT company FROM like_all_table WHERE company NOT LIKE ALL ('microsoft', '%yoo%') | ||
-- !query schema | ||
struct<company:string> | ||
-- !query output | ||
|
||
|
||
-- !query | ||
SELECT company FROM like_all_table WHERE company NOT LIKE ALL ('%oo%', 'fa%') | ||
-- !query schema | ||
struct<company:string> | ||
-- !query output | ||
|
||
|
||
-- !query | ||
SELECT company FROM like_all_table WHERE NOT company LIKE ALL ('%oo%', 'fa%') | ||
-- !query schema | ||
struct<company:string> | ||
-- !query output | ||
|
||
|
||
-- !query | ||
SELECT company FROM like_all_table WHERE company LIKE ALL ('%oo%', NULL) | ||
-- !query schema | ||
struct<company:string> | ||
-- !query output | ||
|
||
|
||
|
||
-- !query | ||
SELECT company FROM like_all_table WHERE company NOT LIKE ALL ('%oo%', NULL) | ||
-- !query schema | ||
struct<company:string> | ||
-- !query output | ||
|
||
|
||
|
||
-- !query | ||
SELECT company FROM like_all_table WHERE company LIKE ALL (NULL, NULL) | ||
-- !query schema | ||
struct<company:string> | ||
-- !query output | ||
|
||
|
||
|
||
-- !query | ||
SELECT company FROM like_all_table WHERE company NOT LIKE ALL (NULL, NULL) | ||
-- !query schema | ||
struct<company:string> | ||
-- !query output | ||
|
||
|
||
|
||
-- !query | ||
SELECT company FROM like_any_table WHERE company LIKE ALL () | ||
-- !query schema | ||
struct<> | ||
-- !query output | ||
org.apache.spark.sql.catalyst.parser.ParseException | ||
|
||
Expected something between '(' and ')'.(line 1, pos 49) | ||
|
||
== SQL == | ||
SELECT company FROM like_any_table WHERE company LIKE ALL () | ||
-------------------------------------------------^^^ |
Oops, something went wrong.