Merge pull request #6645 from b41sh/fix-map-access-filter

common/ast/Cargo.toml

@@ -20,7 +20,7 @@ common-meta-types = { path = "../meta/types" }
 # TODO(andylokandy): Use the version from crates.io once
 # https://github.com/brendanzab/codespan/pull/331 is released.
 codespan-reporting = { git = "https://github.com/brendanzab/codespan", rev = "c84116f5" }
-sqlparser = { git = "https://github.com/datafuse-extras/sqlparser-rs", rev = "269dffd" }
+sqlparser = { git = "https://github.com/datafuse-extras/sqlparser-rs", rev = "7f246e3" }
 
 # Crates.io dependencies
 async-trait = "0.1.56"

common/ast/src/parser/expr.rs

@@ -302,7 +302,7 @@ impl<'a, I: Iterator<Item = WithSpan<'a, ExprElement<'a>>>> PrattParser<I> for E
 
     fn query(&mut self, elem: &WithSpan<ExprElement>) -> pratt::Result<Affix> {
         let affix = match &elem.elem {
-            ExprElement::MapAccess { .. } => Affix::Postfix(Precedence(10)),
+            ExprElement::MapAccess { .. } => Affix::Postfix(Precedence(25)),
             ExprElement::IsNull { .. } => Affix::Postfix(Precedence(17)),
             ExprElement::Between { .. } => Affix::Postfix(Precedence(BETWEEN_PREC)),
             ExprElement::IsDistinctFrom { .. } => {

common/exception/Cargo.toml

@@ -25,4 +25,4 @@ tonic = "0.7.2"
 
 # Github dependencies
 bincode = { git = "https://github.com/datafuse-extras/bincode", rev = "fd3f9ff" }
-sqlparser = { git = "https://github.com/datafuse-extras/sqlparser-rs", rev = "269dffd" }
+sqlparser = { git = "https://github.com/datafuse-extras/sqlparser-rs", rev = "7f246e3" }

common/functions/Cargo.toml

@@ -48,7 +48,7 @@ serde_json = "1.0.81"
 sha1 = "0.10.1"
 sha2 = "0.10.2"
 simdutf8 = "0.1.4"
-sqlparser = { git = "https://github.com/datafuse-extras/sqlparser-rs", rev = "269dffd" }
+sqlparser = { git = "https://github.com/datafuse-extras/sqlparser-rs", rev = "7f246e3" }
 strength_reduce = "0.2.3"
 twox-hash = "1.6.3"
 uuid = { version = "1.1.2", features = ["v4"] }

common/functions/src/scalars/comparisons/comparison.rs

@@ -28,6 +28,7 @@ use num::traits::AsPrimitive;
 use super::utils::*;
 use crate::scalars::assert_string;
 use crate::scalars::cast_column_field;
+use crate::scalars::new_mutable_bitmap;
 use crate::scalars::primitive_simd_op_boolean;
 use crate::scalars::scalar_binary_op;
 use crate::scalars::ComparisonEqFunction;
@@ -295,16 +296,39 @@ where
     fn eval(&self, l: &ColumnWithField, r: &ColumnWithField) -> Result<BooleanColumn> {
         let func_ctx = FunctionContext::default();
         let lhs = if self.need_cast && l.data_type() != &self.least_supertype {
-            cast_column_field(l, l.data_type(), &self.least_supertype, &func_ctx)?
+            cast_column_field(
+                l,
+                l.data_type(),
+                &wrap_nullable(&self.least_supertype),
+                &func_ctx,
+            )?
         } else {
             l.column().clone()
         };
         let rhs = if self.need_cast && r.data_type() != &self.least_supertype {
-            cast_column_field(r, r.data_type(), &self.least_supertype, &func_ctx)?
+            cast_column_field(
+                r,
+                r.data_type(),
+                &wrap_nullable(&self.least_supertype),
+                &func_ctx,
+            )?
         } else {
             r.column().clone()
         };
-        scalar_binary_op(&lhs, &rhs, self.func.clone(), &mut EvalContext::default())
+
+        let (_, l_bitmap) = lhs.validity();
+        let (_, r_bitmap) = rhs.validity();
+        let bitmap = combine_validities(l_bitmap, r_bitmap);
+        let lhs = Series::remove_nullable(&lhs);
+        let rhs = Series::remove_nullable(&rhs);
+        let column = scalar_binary_op(&lhs, &rhs, self.func.clone(), &mut EvalContext::default())?;
+        match bitmap {
+            Some(bitmap) => {
+                let bitmap = combine_validities(Some(column.values()), Some(&bitmap)).unwrap();
+                Ok(BooleanColumn::from_arrow_data(bitmap))
+            }
+            None => Ok(column),
+        }
     }
 }
 
@@ -338,17 +362,40 @@ where
     fn eval(&self, l: &ColumnWithField, r: &ColumnWithField) -> Result<BooleanColumn> {
         let func_ctx = FunctionContext::default();
         let lhs = if self.need_cast && l.data_type() != &self.least_supertype {
-            cast_column_field(l, l.data_type(), &self.least_supertype, &func_ctx)?
+            cast_column_field(
+                l,
+                l.data_type(),
+                &wrap_nullable(&self.least_supertype),
+                &func_ctx,
+            )?
         } else {
             l.column().clone()
         };
 
         let rhs = if self.need_cast && r.data_type() != &self.least_supertype {
-            cast_column_field(r, r.data_type(), &self.least_supertype, &func_ctx)?
+            cast_column_field(
+                r,
+                r.data_type(),
+                &wrap_nullable(&self.least_supertype),
+                &func_ctx,
+            )?
         } else {
             r.column().clone()
         };
-        primitive_simd_op_boolean::<T, F>(&lhs, &rhs, self.func.clone())
+
+        let (_, l_bitmap) = lhs.validity();
+        let (_, r_bitmap) = rhs.validity();
+        let bitmap = combine_validities(l_bitmap, r_bitmap);
+        let lhs = Series::remove_nullable(&lhs);
+        let rhs = Series::remove_nullable(&rhs);
+        let column = primitive_simd_op_boolean::<T, F>(&lhs, &rhs, self.func.clone())?;
+        match bitmap {
+            Some(bitmap) => {
+                let bitmap = combine_validities(Some(column.values()), Some(&bitmap)).unwrap();
+                Ok(BooleanColumn::from_arrow_data(bitmap))
+            }
+            None => Ok(column),
+        }
     }
 }
 
@@ -371,7 +418,7 @@ impl<F: BooleanSimdImpl> ComparisonExpression for ComparisonBooleanImpl<F> {
             cast_column_field(
                 l,
                 l.data_type(),
-                &DataTypeImpl::Boolean(BooleanType::default()),
+                &wrap_nullable(&DataTypeImpl::Boolean(BooleanType::default())),
                 &func_ctx,
             )?
         } else {
@@ -381,27 +428,69 @@ impl<F: BooleanSimdImpl> ComparisonExpression for ComparisonBooleanImpl<F> {
             cast_column_field(
                 r,
                 r.data_type(),
-                &DataTypeImpl::Boolean(BooleanType::default()),
+                &wrap_nullable(&DataTypeImpl::Boolean(BooleanType::default())),
                 &func_ctx,
             )?
         } else {
             r.column().clone()
         };
+
         let res = match (lhs.is_const(), rhs.is_const()) {
             (false, false) => {
+                let (_, l_bitmap) = lhs.validity();
+                let (_, r_bitmap) = rhs.validity();
+                let bitmap = combine_validities(l_bitmap, r_bitmap);
+                let lhs = Series::remove_nullable(&lhs);
+                let rhs = Series::remove_nullable(&rhs);
                 let lhs: &BooleanColumn = Series::check_get(&lhs)?;
                 let rhs: &BooleanColumn = Series::check_get(&rhs)?;
-                F::vector_vector(lhs, rhs)
+                let column = F::vector_vector(lhs, rhs);
+                match bitmap {
+                    Some(bitmap) => {
+                        let bitmap =
+                            combine_validities(Some(column.values()), Some(&bitmap)).unwrap();
+                        BooleanColumn::from_arrow_data(bitmap)
+                    }
+                    None => column,
+                }
             }
             (false, true) => {
+                let (_, l_bitmap) = lhs.validity();
+                let lhs = Series::remove_nullable(&lhs);
                 let lhs: &BooleanColumn = Series::check_get(&lhs)?;
-                let r = rhs.get_bool(0)?;
-                F::vector_const(lhs, r)
+                let r = rhs.get_bool(0);
+                if r.is_err() {
+                    let bitmap = new_mutable_bitmap(lhs.len(), false);
+                    return Ok(BooleanColumn::from_arrow_data(bitmap.into()));
+                };
+                let column = F::vector_const(lhs, r.unwrap());
+                match l_bitmap {
+                    Some(bitmap) => {
+                        let bitmap =
+                            combine_validities(Some(column.values()), Some(bitmap)).unwrap();
+                        BooleanColumn::from_arrow_data(bitmap)
+                    }
+                    None => column,
+                }
             }
             (true, false) => {
-                let l = lhs.get_bool(0)?;
+                let (_, r_bitmap) = rhs.validity();
+                let rhs = Series::remove_nullable(&rhs);
                 let rhs: &BooleanColumn = Series::check_get(&rhs)?;
-                F::const_vector(l, rhs)
+                let l = lhs.get_bool(0);
+                if l.is_err() {
+                    let bitmap = new_mutable_bitmap(rhs.len(), false);
+                    return Ok(BooleanColumn::from_arrow_data(bitmap.into()));
+                }
+                let column = F::const_vector(l.unwrap(), rhs);
+                match r_bitmap {
+                    Some(bitmap) => {
+                        let bitmap =
+                            combine_validities(Some(column.values()), Some(bitmap)).unwrap();
+                        BooleanColumn::from_arrow_data(bitmap)
+                    }
+                    None => column,
+                }
             }
             (true, true) => unreachable!(),
         };

common/functions/src/scalars/expressions/cast_from_string.rs

@@ -23,7 +23,6 @@ use common_datavalues::prelude::*;
 use common_exception::Result;
 
 use super::cast_with_type::arrow_cast_compute;
-use super::cast_with_type::new_mutable_bitmap;
 use super::cast_with_type::CastOptions;
 use crate::scalars::FunctionContext;
 
@@ -39,48 +38,59 @@ pub fn cast_from_string(
     let str_column = Series::remove_nullable(column);
     let str_column: &StringColumn = Series::check_get(&str_column)?;
     let size = str_column.len();
-    let mut bitmap = new_mutable_bitmap(size, true);
 
     match data_type.data_type_id() {
         TypeID::Date => {
-            let mut builder = ColumnBuilder::<i32>::with_capacity(size);
+            let mut builder = NullableColumnBuilder::<i32>::with_capacity(size);
 
-            for (row, v) in str_column.iter().enumerate() {
+            for v in str_column.iter() {
                 if let Some(d) = string_to_date(v) {
-                    builder.append((d.num_days_from_ce() - EPOCH_DAYS_FROM_CE) as i32);
+                    builder.append((d.num_days_from_ce() - EPOCH_DAYS_FROM_CE) as i32, true);
                 } else {
-                    bitmap.set(row, false)
+                    builder.append_null();
                 }
             }
-
-            Ok((builder.build(size), Some(bitmap.into())))
+            let column = builder.build(size);
+            let nullable_column: &NullableColumn = Series::check_get(&column)?;
+            Ok((
+                nullable_column.inner().clone(),
+                Some(nullable_column.ensure_validity().clone()),
+            ))
         }
 
         TypeID::Timestamp => {
-            let mut builder = ColumnBuilder::<i64>::with_capacity(size);
+            let mut builder = NullableColumnBuilder::<i64>::with_capacity(size);
             let tz = func_ctx.tz;
-            for (row, v) in str_column.iter().enumerate() {
+            for v in str_column.iter() {
                 match string_to_timestamp(v, &tz) {
-                    Some(d) => {
-                        builder.append(d.timestamp_micros());
-                    }
-                    None => bitmap.set(row, false),
+                    Some(d) => builder.append(d.timestamp_micros(), true),
+                    None => builder.append_null(),
                 }
             }
-            Ok((builder.build(size), Some(bitmap.into())))
+            let column = builder.build(size);
+            let nullable_column: &NullableColumn = Series::check_get(&column)?;
+            Ok((
+                nullable_column.inner().clone(),
+                Some(nullable_column.ensure_validity().clone()),
+            ))
         }
         TypeID::Boolean => {
-            let mut builder = ColumnBuilder::<bool>::with_capacity(size);
-            for (row, v) in str_column.iter().enumerate() {
+            let mut builder = NullableColumnBuilder::<bool>::with_capacity(size);
+            for v in str_column.iter() {
                 if v.eq_ignore_ascii_case("true".as_bytes()) {
-                    builder.append(true);
+                    builder.append(true, true);
                 } else if v.eq_ignore_ascii_case("false".as_bytes()) {
-                    builder.append(false);
+                    builder.append(false, true);
                 } else {
-                    bitmap.set(row, false);
+                    builder.append_null();
                 }
             }
-            Ok((builder.build(size), Some(bitmap.into())))
+            let column = builder.build(size);
+            let nullable_column: &NullableColumn = Series::check_get(&column)?;
+            Ok((
+                nullable_column.inner().clone(),
+                Some(nullable_column.ensure_validity().clone()),
+            ))
         }
         TypeID::Interval => todo!(),
         _ => arrow_cast_compute(column, from_type, data_type, cast_options, func_ctx),