Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix decimal scalar dyn kernels #5179

Merged
merged 4 commits into from
Feb 6, 2023
Merged

Fix decimal scalar dyn kernels #5179

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3c6cfaf
Fix decimal scalar dyn kernels
viirya Feb 4, 2023
72367ed
Try fix CI
viirya Feb 4, 2023
365ab59
Add test
viirya Feb 5, 2023
832e273
Revert "Try fix CI"
viirya Feb 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/dev.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,4 +54,5 @@ jobs:
'!datafusion/CHANGELOG.md' \
README.md \
CONTRIBUTING.md
git config --global --add safe.directory "$GITHUB_WORKSPACE"
viirya marked this conversation as resolved.
Show resolved Hide resolved
git diff --exit-code
18 changes: 14 additions & 4 deletions .github/workflows/rust.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,9 @@ jobs:
- name: Run tests (excluding doctests)
run: cargo test --lib --tests --bins --features avro,jit,scheduler,json,dictionary_expressions
- name: Verify Working Directory Clean
run: git diff --exit-code
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
git diff --exit-code

linux-test-example:
name: cargo examples (amd64)
Expand Down Expand Up @@ -132,7 +134,9 @@ jobs:
cargo run --example simple_udf
cargo run --example simple_udaf
- name: Verify Working Directory Clean
run: git diff --exit-code
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
git diff --exit-code

# Run doc tests
linux-test-doc:
Expand All @@ -153,7 +157,9 @@ jobs:
- name: Run doctests
run: cargo test --doc --features avro,jit,scheduler,json
- name: Verify Working Directory Clean
run: git diff --exit-code
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
git diff --exit-code

# verify that the benchmark queries return the correct results
verify-benchmark-results:
Expand Down Expand Up @@ -191,7 +197,9 @@ jobs:
cargo test verify_q --profile release-nonlto --features=ci -- --test-threads=1
cargo test serde_q --profile release-nonlto --features=ci -- --test-threads=1
- name: Verify Working Directory Clean
run: git diff --exit-code
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
git diff --exit-code

sqllogictest-postgres:
name: "Run sqllogictest with Postgres runner"
Expand Down Expand Up @@ -460,6 +468,7 @@ jobs:
#
# ignore ./Cargo.toml because putting workspaces in multi-line lists make it easy to read
ci/scripts/rust_toml_fmt.sh
git config --global --add safe.directory "$GITHUB_WORKSPACE"
git diff --exit-code

config-docs-check:
Expand Down Expand Up @@ -489,4 +498,5 @@ jobs:
run: |
# If you encounter an error, run './dev/update_config_docs.sh' and commit
./dev/update_config_docs.sh
git config --global --add safe.directory "$GITHUB_WORKSPACE"
git diff --exit-code
166 changes: 166 additions & 0 deletions datafusion/physical-expr/src/expressions/binary.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1694,6 +1694,49 @@ mod tests {
Ok(())
}

#[test]
fn plus_op_dict_scalar_decimal() -> Result<()> {
let schema = Schema::new(vec![Field::new(
"a",
DataType::Dictionary(
Box::new(DataType::Int8),
Box::new(DataType::Decimal128(10, 0)),
),
true,
)]);

let value = 123;
let decimal_array = Arc::new(create_decimal_array(
&[Some(value), None, Some(value - 1), Some(value + 1)],
10,
0,
)) as ArrayRef;

let keys = Int8Array::from(vec![0, 2, 1, 3, 0]);
let a = DictionaryArray::try_new(&keys, &decimal_array)?;

let keys = Int8Array::from(vec![0, 2, 1, 3, 0]);
let decimal_array = create_decimal_array(
&[Some(value + 1), None, Some(value), Some(value + 2)],
10,
0,
);
let expected = DictionaryArray::try_new(&keys, &decimal_array)?;

apply_arithmetic_scalar(
Arc::new(schema),
vec![Arc::new(a)],
Operator::Plus,
ScalarValue::Dictionary(
Box::new(DataType::Int8),
Box::new(ScalarValue::Decimal128(Some(1), 10, 0)),
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

without this PR's code, is the actual type of the output array whatever the default precision of Decimal128 is (rather than 10, 0)?

Copy link
Member Author

@viirya viirya Feb 5, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, original code still modifies precision/scale for output array, but it just worked for primitive array not dictionary array.

),
Arc::new(expected),
)?;

Ok(())
}

#[test]
fn minus_op() -> Result<()> {
let schema = Arc::new(Schema::new(vec![
Expand Down Expand Up @@ -1776,6 +1819,49 @@ mod tests {
Ok(())
}

#[test]
fn minus_op_dict_scalar_decimal() -> Result<()> {
let schema = Schema::new(vec![Field::new(
"a",
DataType::Dictionary(
Box::new(DataType::Int8),
Box::new(DataType::Decimal128(10, 0)),
),
true,
)]);

let value = 123;
let decimal_array = Arc::new(create_decimal_array(
&[Some(value), None, Some(value - 1), Some(value + 1)],
10,
0,
)) as ArrayRef;

let keys = Int8Array::from(vec![0, 2, 1, 3, 0]);
let a = DictionaryArray::try_new(&keys, &decimal_array)?;

let keys = Int8Array::from(vec![0, 2, 1, 3, 0]);
let decimal_array = create_decimal_array(
&[Some(value - 1), None, Some(value - 2), Some(value)],
10,
0,
);
let expected = DictionaryArray::try_new(&keys, &decimal_array)?;

apply_arithmetic_scalar(
Arc::new(schema),
vec![Arc::new(a)],
Operator::Minus,
ScalarValue::Dictionary(
Box::new(DataType::Int8),
Box::new(ScalarValue::Decimal128(Some(1), 10, 0)),
),
Arc::new(expected),
)?;

Ok(())
}

#[test]
fn multiply_op() -> Result<()> {
let schema = Arc::new(Schema::new(vec![
Expand Down Expand Up @@ -1850,6 +1936,46 @@ mod tests {
Ok(())
}

#[test]
fn multiply_op_dict_scalar_decimal() -> Result<()> {
let schema = Schema::new(vec![Field::new(
"a",
DataType::Dictionary(
Box::new(DataType::Int8),
Box::new(DataType::Decimal128(10, 0)),
),
true,
)]);

let value = 123;
let decimal_array = Arc::new(create_decimal_array(
&[Some(value), None, Some(value - 1), Some(value + 1)],
10,
0,
)) as ArrayRef;

let keys = Int8Array::from(vec![0, 2, 1, 3, 0]);
let a = DictionaryArray::try_new(&keys, &decimal_array)?;

let keys = Int8Array::from(vec![0, 2, 1, 3, 0]);
let decimal_array =
create_decimal_array(&[Some(246), None, Some(244), Some(248)], 10, 0);
let expected = DictionaryArray::try_new(&keys, &decimal_array)?;

apply_arithmetic_scalar(
Arc::new(schema),
vec![Arc::new(a)],
Operator::Multiply,
ScalarValue::Dictionary(
Box::new(DataType::Int8),
Box::new(ScalarValue::Decimal128(Some(2), 10, 0)),
),
Arc::new(expected),
)?;

Ok(())
}

#[test]
fn divide_op() -> Result<()> {
let schema = Arc::new(Schema::new(vec![
Expand Down Expand Up @@ -1924,6 +2050,46 @@ mod tests {
Ok(())
}

#[test]
fn divide_op_dict_scalar_decimal() -> Result<()> {
let schema = Schema::new(vec![Field::new(
"a",
DataType::Dictionary(
Box::new(DataType::Int8),
Box::new(DataType::Decimal128(10, 0)),
),
true,
)]);

let value = 123;
let decimal_array = Arc::new(create_decimal_array(
&[Some(value), None, Some(value - 1), Some(value + 1)],
10,
0,
)) as ArrayRef;

let keys = Int8Array::from(vec![0, 2, 1, 3, 0]);
let a = DictionaryArray::try_new(&keys, &decimal_array)?;

let keys = Int8Array::from(vec![0, 2, 1, 3, 0]);
let decimal_array =
create_decimal_array(&[Some(61), None, Some(61), Some(62)], 10, 0);
let expected = DictionaryArray::try_new(&keys, &decimal_array)?;

apply_arithmetic_scalar(
Arc::new(schema),
vec![Arc::new(a)],
Operator::Divide,
ScalarValue::Dictionary(
Box::new(DataType::Int8),
Box::new(ScalarValue::Decimal128(Some(2), 10, 0)),
),
Arc::new(expected),
)?;

Ok(())
}

#[test]
fn modulus_op() -> Result<()> {
let schema = Arc::new(Schema::new(vec![
Expand Down
16 changes: 4 additions & 12 deletions datafusion/physical-expr/src/expressions/binary/kernels_arrow.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,28 +189,20 @@ pub(crate) fn add_decimal(
}

pub(crate) fn add_decimal_dyn_scalar(left: &dyn Array, right: i128) -> Result<ArrayRef> {
let left_decimal = left.as_any().downcast_ref::<Decimal128Array>().unwrap();
let (precision, scale) = get_precision_scale(left)?;
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In last PR, forgot to update this and there.


let array = add_scalar_dyn::<Decimal128Type>(left, right)?;
let decimal_array = as_decimal128_array(&array)?;
let decimal_array = decimal_array
.clone()
.with_precision_and_scale(left_decimal.precision(), left_decimal.scale())?;
Ok(Arc::new(decimal_array))
decimal_array_with_precision_scale(array, precision, scale)
}

pub(crate) fn subtract_decimal_dyn_scalar(
left: &dyn Array,
right: i128,
) -> Result<ArrayRef> {
let left_decimal = left.as_any().downcast_ref::<Decimal128Array>().unwrap();
let (precision, scale) = get_precision_scale(left)?;

let array = subtract_scalar_dyn::<Decimal128Type>(left, right)?;
let decimal_array = as_decimal128_array(&array)?;
let decimal_array = decimal_array
.clone()
.with_precision_and_scale(left_decimal.precision(), left_decimal.scale())?;
Ok(Arc::new(decimal_array))
decimal_array_with_precision_scale(array, precision, scale)
}

fn get_precision_scale(left: &dyn Array) -> Result<(u8, i8)> {
Expand Down