-
Notifications
You must be signed in to change notification settings - Fork 1.1k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat: customize column default values for external tables #8415
Changes from 2 commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -220,8 +220,8 @@ async fn roundtrip_custom_listing_tables() -> Result<()> { | |
// Make sure during round-trip, constraint information is preserved | ||
let query = "CREATE EXTERNAL TABLE multiple_ordered_table_with_pk ( | ||
a0 INTEGER, | ||
a INTEGER, | ||
b INTEGER, | ||
a INTEGER DEFAULT 1*2 + 3, | ||
b INTEGER DEFAULT NULL, | ||
c INTEGER, | ||
d INTEGER, | ||
primary key(c) | ||
|
@@ -232,11 +232,12 @@ async fn roundtrip_custom_listing_tables() -> Result<()> { | |
WITH ORDER (c ASC) | ||
LOCATION '../core/tests/data/window_2.csv';"; | ||
|
||
let plan = ctx.sql(query).await?.into_optimized_plan()?; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It always returned an EmptyRelation before, and did not achieve the testing purpose as expected. |
||
let plan = ctx.state().create_logical_plan(query).await?; | ||
|
||
let bytes = logical_plan_to_bytes(&plan)?; | ||
let logical_round_trip = logical_plan_from_bytes(&bytes, &ctx)?; | ||
assert_eq!(format!("{plan:?}"), format!("{logical_round_trip:?}")); | ||
// Use exact matching to verify everything, such as column defaults | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Nice -- I think this check predated the |
||
assert_eq!(plan, logical_round_trip); | ||
|
||
Ok(()) | ||
} | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -543,3 +543,48 @@ select * from table_without_values; | |
|
||
statement ok | ||
drop table table_without_values; | ||
|
||
|
||
### Test for specifying column's default value | ||
|
||
statement ok | ||
CREATE EXTERNAL TABLE test_column_defaults( | ||
a int, | ||
b int not null default null, | ||
c int default 100*2+300, | ||
d text default lower('DEFAULT_TEXT'), | ||
e timestamp default now() | ||
) STORED AS parquet | ||
LOCATION 'test_files/scratch/insert_to_external/external_parquet_table_q6' | ||
OPTIONS (create_local_path 'true'); | ||
|
||
query IIITP | ||
insert into test_column_defaults values(1, 10, 100, 'ABC', now()) | ||
---- | ||
1 | ||
|
||
statement error DataFusion error: Execution error: Invalid batch column at '1' has null but schema specifies non-nullable | ||
insert into test_column_defaults(a) values(2) | ||
|
||
query IIITP | ||
insert into test_column_defaults(b) values(20) | ||
---- | ||
1 | ||
|
||
query IIIT rowsort | ||
select a,b,c,d from test_column_defaults | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Could you also add a test that the value of Maybe something basic like -- Expect all rows to be true as now() was inserted into the table
select e < now() from test_column_defaults ? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Good idea! Added. |
||
---- | ||
1 10 100 ABC | ||
NULL 20 500 default_text | ||
|
||
statement ok | ||
drop table test_column_defaults | ||
|
||
# test invalid default value | ||
statement error DataFusion error: Error during planning: Column reference is not allowed in the DEFAULT expression : Schema error: No field named a. | ||
CREATE EXTERNAL TABLE test_column_defaults( | ||
a int, | ||
b int default a+1 | ||
) STORED AS parquet | ||
LOCATION 'test_files/scratch/insert_to_external/external_parquet_table_q7' | ||
OPTIONS (create_local_path 'true'); |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I introduced
hashbrown::HashMap
in #8283, now I am replacing it withstd::collections::HashMap
.