From 33e1956f8cf4bf579e357ca076fde5b7b908ace8 Mon Sep 17 00:00:00 2001 From: srib Date: Fri, 9 Sep 2022 20:45:37 -0400 Subject: [PATCH 1/2] include bitwise xor operator --- datafusion/expr/src/binary_rule.rs | 2 + datafusion/expr/src/operator.rs | 4 ++ .../physical-expr/src/expressions/binary.rs | 26 +++++++++- .../src/expressions/binary/kernels.rs | 48 +++++++++++++++++++ datafusion/sql/src/planner.rs | 1 + 5 files changed, 80 insertions(+), 1 deletion(-) diff --git a/datafusion/expr/src/binary_rule.rs b/datafusion/expr/src/binary_rule.rs index 5eeb1867a017..24d1cb32dd9c 100644 --- a/datafusion/expr/src/binary_rule.rs +++ b/datafusion/expr/src/binary_rule.rs @@ -57,6 +57,7 @@ pub fn binary_operator_data_type( // bitwise operations return the common coerced type Operator::BitwiseAnd | Operator::BitwiseOr + | Operator::BitwiseXor | Operator::BitwiseShiftLeft | Operator::BitwiseShiftRight => Ok(result_type), // math operations return the same value as the common coerced type @@ -81,6 +82,7 @@ pub fn coerce_types( let result = match op { Operator::BitwiseAnd | Operator::BitwiseOr + | Operator::BitwiseXor | Operator::BitwiseShiftRight | Operator::BitwiseShiftLeft => bitwise_coercion(lhs_type, rhs_type), Operator::And | Operator::Or => match (lhs_type, rhs_type) { diff --git a/datafusion/expr/src/operator.rs b/datafusion/expr/src/operator.rs index a5b752c64377..7c0c4d76bbe0 100644 --- a/datafusion/expr/src/operator.rs +++ b/datafusion/expr/src/operator.rs @@ -71,6 +71,8 @@ pub enum Operator { BitwiseAnd, /// Bitwise or, like `|` BitwiseOr, + /// Bitwise xor, like `#` + BitwiseXor, /// Bitwise right, like `>>` BitwiseShiftRight, /// Bitwise left, like `<<` @@ -107,6 +109,7 @@ impl Operator { | Operator::RegexNotIMatch | Operator::BitwiseAnd | Operator::BitwiseOr + | Operator::BitwiseXor | Operator::BitwiseShiftRight | Operator::BitwiseShiftLeft | Operator::StringConcat => None, @@ -140,6 +143,7 @@ impl fmt::Display for Operator { Operator::IsNotDistinctFrom => "IS NOT DISTINCT FROM", Operator::BitwiseAnd => "&", Operator::BitwiseOr => "|", + Operator::BitwiseXor => "#", Operator::BitwiseShiftRight => ">>", Operator::BitwiseShiftLeft => "<<", Operator::StringConcat => "||", diff --git a/datafusion/physical-expr/src/expressions/binary.rs b/datafusion/physical-expr/src/expressions/binary.rs index 50d200d4ee9a..e934c259f90a 100644 --- a/datafusion/physical-expr/src/expressions/binary.rs +++ b/datafusion/physical-expr/src/expressions/binary.rs @@ -57,6 +57,7 @@ use arrow::compute::kernels::concat_elements::concat_elements_utf8; use kernels::{ bitwise_and, bitwise_and_scalar, bitwise_or, bitwise_or_scalar, bitwise_shift_left, bitwise_shift_left_scalar, bitwise_shift_right, bitwise_shift_right_scalar, + bitwise_xor, bitwise_xor_scalar, }; use kernels_arrow::{ add_decimal, add_decimal_scalar, divide_decimal, divide_decimal_scalar, @@ -764,6 +765,7 @@ impl BinaryExpr { ), Operator::BitwiseAnd => bitwise_and_scalar(array, scalar.clone()), Operator::BitwiseOr => bitwise_or_scalar(array, scalar.clone()), + Operator::BitwiseXor => bitwise_xor_scalar(array, scalar.clone()), Operator::BitwiseShiftRight => { bitwise_shift_right_scalar(array, scalar.clone()) } @@ -880,6 +882,7 @@ impl BinaryExpr { } Operator::BitwiseAnd => bitwise_and(left, right), Operator::BitwiseOr => bitwise_or(left, right), + Operator::BitwiseXor => bitwise_xor(left, right), Operator::BitwiseShiftRight => bitwise_shift_right(left, right), Operator::BitwiseShiftLeft => bitwise_shift_left(left, right), Operator::StringConcat => { @@ -1295,6 +1298,18 @@ mod tests { DataType::Int64, vec![11i64, 6i64, 7i64] ); + test_coercion!( + Int16Array, + DataType::Int16, + vec![3i16, 2i16, 3i16], + Int64Array, + DataType::Int64, + vec![10i64, 6i64, 5i64], + Operator::BitwiseXor, + Int64Array, + DataType::Int64, + vec![9i64, 4i64, 6i64] + ); Ok(()) } @@ -2511,6 +2526,11 @@ mod tests { result = bitwise_or(left.clone(), right.clone())?; let expected = Int32Array::from(vec![Some(13), None, Some(15)]); assert_eq!(result.as_ref(), &expected); + + result = bitwise_xor(left.clone(), right.clone())?; + let expected = Int32Array::from(vec![Some(13), None, Some(12)]); + assert_eq!(result.as_ref(), &expected); + Ok(()) } @@ -2550,9 +2570,13 @@ mod tests { let expected = Int32Array::from(vec![Some(0), None, Some(3)]); assert_eq!(result.as_ref(), &expected); - result = bitwise_or_scalar(&left, right).unwrap()?; + result = bitwise_or_scalar(&left, right.clone()).unwrap()?; let expected = Int32Array::from(vec![Some(15), None, Some(11)]); assert_eq!(result.as_ref(), &expected); + + result = bitwise_xor_scalar(&left, right.clone()).unwrap()?; + let expected = Int32Array::from(vec![Some(15), None, Some(8)]); + assert_eq!(result.as_ref(), &expected); Ok(()) } diff --git a/datafusion/physical-expr/src/expressions/binary/kernels.rs b/datafusion/physical-expr/src/expressions/binary/kernels.rs index 3ca4a447c875..9414ea11c2cb 100644 --- a/datafusion/physical-expr/src/expressions/binary/kernels.rs +++ b/datafusion/physical-expr/src/expressions/binary/kernels.rs @@ -201,6 +201,28 @@ pub(crate) fn bitwise_or(left: ArrayRef, right: ArrayRef) -> Result { } } +pub(crate) fn bitwise_xor(left: ArrayRef, right: ArrayRef) -> Result { + match &left.data_type() { + DataType::Int8 => { + binary_bitwise_array_op!(left, right, |a, b| a ^ b, Int8Array) + } + DataType::Int16 => { + binary_bitwise_array_op!(left, right, |a, b| a ^ b, Int16Array) + } + DataType::Int32 => { + binary_bitwise_array_op!(left, right, |a, b| a ^ b, Int32Array) + } + DataType::Int64 => { + binary_bitwise_array_op!(left, right, |a, b| a ^ b, Int64Array) + } + other => Err(DataFusionError::Internal(format!( + "Data type {:?} not supported for binary operation '{}' on dyn arrays", + other, + Operator::BitwiseXor + ))), + } +} + pub(crate) fn bitwise_and_scalar( array: &dyn Array, scalar: ScalarValue, @@ -253,6 +275,32 @@ pub(crate) fn bitwise_or_scalar( Some(result) } +pub(crate) fn bitwise_xor_scalar( + array: &dyn Array, + scalar: ScalarValue, +) -> Option> { + let result = match array.data_type() { + DataType::Int8 => { + binary_bitwise_array_scalar!(array, scalar, |a, b| a ^ b, Int8Array, i8) + } + DataType::Int16 => { + binary_bitwise_array_scalar!(array, scalar, |a, b| a ^ b, Int16Array, i16) + } + DataType::Int32 => { + binary_bitwise_array_scalar!(array, scalar, |a, b| a ^ b, Int32Array, i32) + } + DataType::Int64 => { + binary_bitwise_array_scalar!(array, scalar, |a, b| a ^ b, Int64Array, i64) + } + other => Err(DataFusionError::Internal(format!( + "Data type {:?} not supported for binary operation '{}' on dyn arrays", + other, + Operator::BitwiseXor + ))), + }; + Some(result) +} + pub(crate) fn bitwise_shift_right_scalar( array: &dyn Array, scalar: ScalarValue, diff --git a/datafusion/sql/src/planner.rs b/datafusion/sql/src/planner.rs index dd44459f5081..861d89e9cee9 100644 --- a/datafusion/sql/src/planner.rs +++ b/datafusion/sql/src/planner.rs @@ -1589,6 +1589,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { BinaryOperator::PGRegexNotIMatch => Ok(Operator::RegexNotIMatch), BinaryOperator::BitwiseAnd => Ok(Operator::BitwiseAnd), BinaryOperator::BitwiseOr => Ok(Operator::BitwiseOr), + BinaryOperator::BitwiseXor => Ok(Operator::BitwiseXor), BinaryOperator::PGBitwiseShiftRight => Ok(Operator::BitwiseShiftRight), BinaryOperator::PGBitwiseShiftLeft => Ok(Operator::BitwiseShiftLeft), BinaryOperator::StringConcat => Ok(Operator::StringConcat), From 6b4da9a0765290c86847c90f3bc0ccc451ba1c09 Mon Sep 17 00:00:00 2001 From: srib Date: Sat, 10 Sep 2022 08:20:28 -0400 Subject: [PATCH 2/2] fix clippy warning --- datafusion/physical-expr/src/expressions/binary.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/physical-expr/src/expressions/binary.rs b/datafusion/physical-expr/src/expressions/binary.rs index e934c259f90a..2395f85c1eb0 100644 --- a/datafusion/physical-expr/src/expressions/binary.rs +++ b/datafusion/physical-expr/src/expressions/binary.rs @@ -2574,7 +2574,7 @@ mod tests { let expected = Int32Array::from(vec![Some(15), None, Some(11)]); assert_eq!(result.as_ref(), &expected); - result = bitwise_xor_scalar(&left, right.clone()).unwrap()?; + result = bitwise_xor_scalar(&left, right).unwrap()?; let expected = Int32Array::from(vec![Some(15), None, Some(8)]); assert_eq!(result.as_ref(), &expected); Ok(())