Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add ASCII fast path for ILIKE scalar (90% faster) #3306

Merged
merged 8 commits into from
Dec 9, 2022
Merged

Add ASCII fast path for ILIKE scalar (90% faster) #3306

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
0fb5ea5
Add ASCII fast path for ILIKE scalar
tustvold Dec 9, 2022
616342d
Update starts_with and ends_with
tustvold Dec 9, 2022
faf6eaa
Avoid allocations
tustvold Dec 9, 2022
a47500a
Format
tustvold Dec 9, 2022
7b2c61b
Add more tests
tustvold Dec 9, 2022
3ea923e
Add failing test
tustvold Dec 9, 2022
3d5793f
Merge remote-tracking branch 'upstream/master' into ilike-fast-path
tustvold Dec 9, 2022
3ff4180
Verify is_ascii
tustvold Dec 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions arrow-array/src/array/byte_array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,14 @@ impl<T: ByteArrayType> GenericByteArray<T> {
self.data.buffers()[1].as_slice()
}

/// Returns true if all data within this array is ASCII
pub fn is_ascii(&self) -> bool {
let offsets = self.value_offsets();
let start = offsets.first().unwrap();
let end = offsets.last().unwrap();
self.value_data()[start.as_usize()..end.as_usize()].is_ascii()
}

/// Returns the offset values in the offsets buffer
#[inline]
pub fn value_offsets(&self) -> &[T::Offset] {
Expand Down
207 changes: 150 additions & 57 deletions arrow-string/src/like.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -442,7 +442,10 @@ pub fn nlike_utf8_scalar<OffsetSize: OffsetSizeTrait>(
/// Perform SQL `left ILIKE right` operation on [`StringArray`] /
/// [`LargeStringArray`].
///
/// See the documentation on [`like_utf8`] for more details.
/// Case insensitive version of [`like_utf8`]
///
/// Note: this only implements loose matching as defined by the Unicode standard. For example,
/// the `ff` ligature is not equivalent to `FF` and `ß` is not equivalent to `SS`
pub fn ilike_utf8<OffsetSize: OffsetSizeTrait>(
left: &GenericStringArray<OffsetSize>,
right: &GenericStringArray<OffsetSize>,
Expand Down Expand Up @@ -499,7 +502,7 @@ pub fn ilike_dyn(
/// Perform SQL `left ILIKE right` operation on on [`DictionaryArray`] with values
/// [`StringArray`]/[`LargeStringArray`].
///
/// See the documentation on [`like_utf8`] for more details.
/// See the documentation on [`ilike_utf8`] for more details.
#[cfg(feature = "dyn_cmp_dict")]
fn ilike_dict<K: ArrowPrimitiveType>(
left: &DictionaryArray<K>,
Expand Down Expand Up @@ -540,60 +543,55 @@ fn ilike_dict<K: ArrowPrimitiveType>(
}

#[inline]
fn ilike_scalar_op<'a, F: Fn(bool) -> bool, L: ArrayAccessor<Item = &'a str>>(
left: L,
fn ilike_scalar_op<O: OffsetSizeTrait, F: Fn(bool) -> bool>(
left: &GenericStringArray<O>,
right: &str,
op: F,
) -> Result<BooleanArray, ArrowError> {
if !right.contains(is_like_pattern) {
// fast path, can use equals
let right_uppercase = right.to_uppercase();

Ok(BooleanArray::from_unary(left, |item| {
op(item.to_uppercase() == right_uppercase)
}))
} else if right.ends_with('%')
&& !right.ends_with("\\%")
&& !right[..right.len() - 1].contains(is_like_pattern)
{
// fast path, can use starts_with
let start_str = &right[..right.len() - 1].to_uppercase();
Ok(BooleanArray::from_unary(left, |item| {
op(item.to_uppercase().starts_with(start_str))
}))
} else if right.starts_with('%') && !right[1..].contains(is_like_pattern) {
// fast path, can use ends_with
let ends_str = &right[1..].to_uppercase();
// If not ASCII faster to use case insensitive regex than using to_uppercase
if right.is_ascii() && left.is_ascii() {
if !right.contains(is_like_pattern) {
return Ok(BooleanArray::from_unary(left, |item| {
op(item.eq_ignore_ascii_case(right))
}));
} else if right.ends_with('%')
&& !right.ends_with("\\%")
&& !right[..right.len() - 1].contains(is_like_pattern)
{
// fast path, can use starts_with
let start_str = &right[..right.len() - 1];
return Ok(BooleanArray::from_unary(left, |item| {
let end = item.len().min(start_str.len());
let result = item.is_char_boundary(end)
&& start_str.eq_ignore_ascii_case(&item[..end]);
op(result)
}));
} else if right.starts_with('%') && !right[1..].contains(is_like_pattern) {
// fast path, can use ends_with
let ends_str = &right[1..];
return Ok(BooleanArray::from_unary(left, |item| {
let start = item.len().saturating_sub(ends_str.len());
let result = item.is_char_boundary(start)
&& ends_str.eq_ignore_ascii_case(&item[start..]);
op(result)
}));
}
}

Ok(BooleanArray::from_unary(left, |item| {
op(item.to_uppercase().ends_with(ends_str))
}))
} else if right.starts_with('%')
&& right.ends_with('%')
&& !right.ends_with("\\%")
&& !right[1..right.len() - 1].contains(is_like_pattern)
{
// fast path, can use contains
let contains = &right[1..right.len() - 1].to_uppercase();
Ok(BooleanArray::from_unary(left, |item| {
op(item.to_uppercase().contains(contains))
}))
} else {
let re_pattern = replace_like_wildcards(right)?;
let re = Regex::new(&format!("(?i)^{}$", re_pattern)).map_err(|e| {
ArrowError::ComputeError(format!(
"Unable to build regex from ILIKE pattern: {}",
e
))
})?;
let re_pattern = replace_like_wildcards(right)?;
let re = Regex::new(&format!("(?i)^{}$", re_pattern)).map_err(|e| {
ArrowError::ComputeError(format!(
"Unable to build regex from ILIKE pattern: {}",
e
))
})?;

Ok(BooleanArray::from_unary(left, |item| op(re.is_match(item))))
}
Ok(BooleanArray::from_unary(left, |item| op(re.is_match(item))))
}

#[inline]
fn ilike_scalar<'a, L: ArrayAccessor<Item = &'a str>>(
left: L,
fn ilike_scalar<O: OffsetSizeTrait>(
left: &GenericStringArray<O>,
right: &str,
) -> Result<BooleanArray, ArrowError> {
ilike_scalar_op(left, right, |x| x)
Expand All @@ -603,7 +601,7 @@ fn ilike_scalar<'a, L: ArrayAccessor<Item = &'a str>>(
/// [`LargeStringArray`], or [`DictionaryArray`] with values
/// [`StringArray`]/[`LargeStringArray`] and a scalar.
///
/// See the documentation on [`like_utf8`] for more details.
/// See the documentation on [`ilike_utf8`] for more details.
pub fn ilike_utf8_scalar_dyn(
left: &dyn Array,
right: &str,
Expand Down Expand Up @@ -641,7 +639,7 @@ pub fn ilike_utf8_scalar_dyn(
/// Perform SQL `left ILIKE right` operation on [`StringArray`] /
/// [`LargeStringArray`] and a scalar.
///
/// See the documentation on [`like_utf8`] for more details.
/// See the documentation on [`ilike_utf8`] for more details.
pub fn ilike_utf8_scalar<OffsetSize: OffsetSizeTrait>(
left: &GenericStringArray<OffsetSize>,
right: &str,
Expand All @@ -652,7 +650,7 @@ pub fn ilike_utf8_scalar<OffsetSize: OffsetSizeTrait>(
/// Perform SQL `left NOT ILIKE right` operation on [`StringArray`] /
/// [`LargeStringArray`].
///
/// See the documentation on [`like_utf8`] for more details.
/// See the documentation on [`ilike_utf8`] for more details.
pub fn nilike_utf8<OffsetSize: OffsetSizeTrait>(
left: &GenericStringArray<OffsetSize>,
right: &GenericStringArray<OffsetSize>,
Expand All @@ -670,7 +668,7 @@ pub fn nilike_utf8<OffsetSize: OffsetSizeTrait>(
/// Perform SQL `left NOT ILIKE right` operation on on [`DictionaryArray`] with values
/// [`StringArray`]/[`LargeStringArray`].
///
/// See the documentation on [`like_utf8`] for more details.
/// See the documentation on [`ilike_utf8`] for more details.
pub fn nilike_dyn(
left: &dyn Array,
right: &dyn Array,
Expand Down Expand Up @@ -709,7 +707,7 @@ pub fn nilike_dyn(
/// Perform SQL `left NOT ILIKE right` operation on on [`DictionaryArray`] with values
/// [`StringArray`]/[`LargeStringArray`].
///
/// See the documentation on [`like_utf8`] for more details.
/// See the documentation on [`ilike_utf8`] for more details.
#[cfg(feature = "dyn_cmp_dict")]
fn nilike_dict<K: ArrowPrimitiveType>(
left: &DictionaryArray<K>,
Expand Down Expand Up @@ -750,8 +748,8 @@ fn nilike_dict<K: ArrowPrimitiveType>(
}

#[inline]
fn nilike_scalar<'a, L: ArrayAccessor<Item = &'a str>>(
left: L,
fn nilike_scalar<O: OffsetSizeTrait>(
left: &GenericStringArray<O>,
right: &str,
) -> Result<BooleanArray, ArrowError> {
ilike_scalar_op(left, right, |x| !x)
Expand All @@ -761,7 +759,7 @@ fn nilike_scalar<'a, L: ArrayAccessor<Item = &'a str>>(
/// [`LargeStringArray`], or [`DictionaryArray`] with values
/// [`StringArray`]/[`LargeStringArray`] and a scalar.
///
/// See the documentation on [`like_utf8`] for more details.
/// See the documentation on [`ilike_utf8`] for more details.
pub fn nilike_utf8_scalar_dyn(
left: &dyn Array,
right: &str,
Expand Down Expand Up @@ -799,7 +797,7 @@ pub fn nilike_utf8_scalar_dyn(
/// Perform SQL `left NOT ILIKE right` operation on [`StringArray`] /
/// [`LargeStringArray`] and a scalar.
///
/// See the documentation on [`like_utf8`] for more details.
/// See the documentation on [`ilike_utf8`] for more details.
pub fn nilike_utf8_scalar<OffsetSize: OffsetSizeTrait>(
left: &GenericStringArray<OffsetSize>,
right: &str,
Expand Down Expand Up @@ -1272,6 +1270,101 @@ mod tests {
vec![true, false, false, false]
);

// We only implement loose matching
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When I ran these tests without the changes in this PR I got a bunch of failures. Is that expected?

failures:

---- like::tests::test_utf8_array_ilike_unicode stdout ----
thread 'like::tests::test_utf8_array_ilike_unicode' panicked at 'assertion failed: `(left == right)`
  left: `true`,
 right: `false`: unexpected result when comparing FFkoß at position 0 to FFkoSS ', arrow-string/src/like.rs:1279:5
stack backtrace:
   0: rust_begin_unwind
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/std/src/panicking.rs:584:5
   1: core::panicking::panic_fmt
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/panicking.rs:142:14
   2: core::panicking::assert_failed_inner
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/panicking.rs:218:23
   3: core::panicking::assert_failed
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/panicking.rs:181:5
   4: arrow_string::like::tests::test_utf8_array_ilike_unicode
             at ./src/like.rs:1279:5
   5: arrow_string::like::tests::test_utf8_array_ilike_unicode::{{closure}}
             at ./src/like.rs:917:13
   6: core::ops::function::FnOnce::call_once
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/ops/function.rs:248:5
   7: core::ops::function::FnOnce::call_once
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/ops/function.rs:248:5
note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.

---- like::tests::test_utf8_array_ilike_unicode_contains stdout ----
thread 'like::tests::test_utf8_array_ilike_unicode_contains' panicked at 'assertion failed: `(left == right)`
  left: `true`,
 right: `false`: unexpected result when comparing sdlkdfFkoßsdfs at position 0 to %FFkoSS% ', arrow-string/src/like.rs:1331:5
stack backtrace:
   0: rust_begin_unwind
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/std/src/panicking.rs:584:5
   1: core::panicking::panic_fmt
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/panicking.rs:142:14
   2: core::panicking::assert_failed_inner
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/panicking.rs:218:23
   3: core::panicking::assert_failed
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/panicking.rs:181:5
   4: arrow_string::like::tests::test_utf8_array_ilike_unicode_contains
             at ./src/like.rs:1331:5
   5: arrow_string::like::tests::test_utf8_array_ilike_unicode_contains::{{closure}}
             at ./src/like.rs:917:13
   6: core::ops::function::FnOnce::call_once
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/ops/function.rs:248:5
   7: core::ops::function::FnOnce::call_once
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/ops/function.rs:248:5
note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.

---- like::tests::test_utf8_array_ilike_unicode_contains_dyn stdout ----
thread 'like::tests::test_utf8_array_ilike_unicode_contains_dyn' panicked at 'assertion failed: `(left == right)`
  left: `true`,
 right: `false`: unexpected result when comparing sdlkdfFkoßsdfs at position 0 to %FFkoSS% ', arrow-string/src/like.rs:1331:5
stack backtrace:
   0: rust_begin_unwind
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/std/src/panicking.rs:584:5
   1: core::panicking::panic_fmt
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/panicking.rs:142:14
   2: core::panicking::assert_failed_inner
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/panicking.rs:218:23
   3: core::panicking::assert_failed
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/panicking.rs:181:5
   4: arrow_string::like::tests::test_utf8_array_ilike_unicode_contains_dyn
             at ./src/like.rs:1331:5
   5: arrow_string::like::tests::test_utf8_array_ilike_unicode_contains_dyn::{{closure}}
             at ./src/like.rs:917:13
   6: core::ops::function::FnOnce::call_once
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/ops/function.rs:248:5
   7: core::ops::function::FnOnce::call_once
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/ops/function.rs:248:5
note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.

---- like::tests::test_utf8_array_ilike_unicode_dyn stdout ----
thread 'like::tests::test_utf8_array_ilike_unicode_dyn' panicked at 'assertion failed: `(left == right)`
  left: `true`,
 right: `false`: unexpected result when comparing FFkoß at position 0 to FFkoSS ', arrow-string/src/like.rs:1279:5
stack backtrace:
   0: rust_begin_unwind
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/std/src/panicking.rs:584:5
   1: core::panicking::panic_fmt
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/panicking.rs:142:14
   2: core::panicking::assert_failed_inner
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/panicking.rs:218:23
   3: core::panicking::assert_failed
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/panicking.rs:181:5
   4: arrow_string::like::tests::test_utf8_array_ilike_unicode_dyn
             at ./src/like.rs:1279:5
   5: arrow_string::like::tests::test_utf8_array_ilike_unicode_dyn::{{closure}}
             at ./src/like.rs:917:13
   6: core::ops::function::FnOnce::call_once
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/ops/function.rs:248:5
   7: core::ops::function::FnOnce::call_once
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/ops/function.rs:248:5
note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.

---- like::tests::test_utf8_array_ilike_unicode_ends stdout ----
thread 'like::tests::test_utf8_array_ilike_unicode_ends' panicked at 'assertion failed: `(left == right)`
  left: `true`,
 right: `false`: unexpected result when comparing sdlkdfFFkoß at position 0 to %FFkoSS ', arrow-string/src/like.rs:1311:5
stack backtrace:
   0: rust_begin_unwind
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/std/src/panicking.rs:584:5
   1: core::panicking::panic_fmt
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/panicking.rs:142:14
   2: core::panicking::assert_failed_inner
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/panicking.rs:218:23
   3: core::panicking::assert_failed
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/panicking.rs:181:5
   4: arrow_string::like::tests::test_utf8_array_ilike_unicode_ends
             at ./src/like.rs:924:21
   5: arrow_string::like::tests::test_utf8_array_ilike_unicode_ends::{{closure}}
             at ./src/like.rs:917:13
   6: core::ops::function::FnOnce::call_once
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/ops/function.rs:248:5
   7: core::ops::function::FnOnce::call_once
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/ops/function.rs:248:5
note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.

---- like::tests::test_utf8_array_ilike_unicode_ends_dyn stdout ----
thread 'like::tests::test_utf8_array_ilike_unicode_ends_dyn' panicked at 'assertion failed: `(left == right)`
  left: `true`,
 right: `false`: unexpected result when comparing sdlkdfFFkoß at position 0 to %FFkoSS ', arrow-string/src/like.rs:1311:5
stack backtrace:
   0: rust_begin_unwind
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/std/src/panicking.rs:584:5
   1: core::panicking::panic_fmt
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/panicking.rs:142:14
   2: core::panicking::assert_failed_inner
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/panicking.rs:218:23
   3: core::panicking::assert_failed
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/panicking.rs:181:5
   4: arrow_string::like::tests::test_utf8_array_ilike_unicode_ends_dyn
             at ./src/like.rs:924:21
   5: arrow_string::like::tests::test_utf8_array_ilike_unicode_ends_dyn::{{closure}}
             at ./src/like.rs:917:13
   6: core::ops::function::FnOnce::call_once
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/ops/function.rs:248:5
   7: core::ops::function::FnOnce::call_once
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/ops/function.rs:248:5
note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.

---- like::tests::test_utf8_array_ilike_unicode_starts stdout ----
thread 'like::tests::test_utf8_array_ilike_unicode_starts' panicked at 'assertion failed: `(left == right)`
  left: `true`,
 right: `false`: unexpected result when comparing FFkoßsdlkdf at position 0 to FFkoSS% ', arrow-string/src/like.rs:1291:5
stack backtrace:
   0: rust_begin_unwind
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/std/src/panicking.rs:584:5
   1: core::panicking::panic_fmt
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/panicking.rs:142:14
   2: core::panicking::assert_failed_inner
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/panicking.rs:218:23
   3: core::panicking::assert_failed
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/panicking.rs:181:5
   4: arrow_string::like::tests::test_utf8_array_ilike_unicode_starts
             at ./src/like.rs:1291:5
   5: arrow_string::like::tests::test_utf8_array_ilike_unicode_starts::{{closure}}
             at ./src/like.rs:917:13
   6: core::ops::function::FnOnce::call_once
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/ops/function.rs:248:5
   7: core::ops::function::FnOnce::call_once
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/ops/function.rs:248:5
note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.

---- like::tests::test_utf8_array_ilike_unicode_start_dyn stdout ----
thread 'like::tests::test_utf8_array_ilike_unicode_start_dyn' panicked at 'assertion failed: `(left == right)`
  left: `true`,
 right: `false`: unexpected result when comparing FFkoßsdlkdf at position 0 to FFkoSS% ', arrow-string/src/like.rs:1291:5
stack backtrace:
   0: rust_begin_unwind
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/std/src/panicking.rs:584:5
   1: core::panicking::panic_fmt
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/panicking.rs:142:14
   2: core::panicking::assert_failed_inner
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/panicking.rs:218:23
   3: core::panicking::assert_failed
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/panicking.rs:181:5
   4: arrow_string::like::tests::test_utf8_array_ilike_unicode_start_dyn
             at ./src/like.rs:1291:5
   5: arrow_string::like::tests::test_utf8_array_ilike_unicode_start_dyn::{{closure}}
             at ./src/like.rs:917:13
   6: core::ops::function::FnOnce::call_once
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/ops/function.rs:248:5
   7: core::ops::function::FnOnce::call_once
             at /rustc/897e37553bba8b42751c67658967889d11ecd120/library/core/src/ops/function.rs:248:5
note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.


failures:
    like::tests::test_utf8_array_ilike_unicode
    like::tests::test_utf8_array_ilike_unicode_contains
    like::tests::test_utf8_array_ilike_unicode_contains_dyn
    like::tests::test_utf8_array_ilike_unicode_dyn
    like::tests::test_utf8_array_ilike_unicode_ends
    like::tests::test_utf8_array_ilike_unicode_ends_dyn
    like::tests::test_utf8_array_ilike_unicode_start_dyn
    like::tests::test_utf8_array_ilike_unicode_starts

Copy link
Contributor Author

@tustvold tustvold Dec 9, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes see #3311 the previous special case logic was incorrect as it changed the semantics from the regex

test_utf8_scalar!(
test_utf8_array_ilike_unicode,
test_utf8_array_ilike_unicode_dyn,
vec![
"FFkoß", "FFkoSS", "FFkoss", "FFkoS", "FFkos", "ffkoSS", "ffkoß", "FFKoSS"
],
"FFkoSS",
ilike_utf8_scalar,
ilike_utf8_scalar_dyn,
vec![false, true, true, false, false, false, false, true]
);

test_utf8_scalar!(
test_utf8_array_ilike_unicode_starts,
test_utf8_array_ilike_unicode_start_dyn,
vec![
"FFkoßsdlkdf",
"FFkoSSsdlkdf",
"FFkosssdlkdf",
"FFkoS",
"FFkos",
"ffkoSS",
"ffkoß",
"FfkosSsdfd",
"FFKoSS",
],
"FFkoSS%",
ilike_utf8_scalar,
ilike_utf8_scalar_dyn,
vec![false, true, true, false, false, false, false, true, true]
);

test_utf8_scalar!(
test_utf8_array_ilike_unicode_ends,
test_utf8_array_ilike_unicode_ends_dyn,
vec![
"sdlkdfFFkoß",
"sdlkdfFFkoSS",
"sdlkdfFFkoss",
"FFkoS",
"FFkos",
"ffkoSS",
"ffkoß",
"h😃klFfkosS",
"FFKoSS",
],
"%FFkoSS",
ilike_utf8_scalar,
ilike_utf8_scalar_dyn,
vec![false, true, true, false, false, false, false, true, true]
);

test_utf8_scalar!(
test_utf8_array_ilike_unicode_contains,
test_utf8_array_ilike_unicode_contains_dyn,
vec![
"sdlkdfFkoßsdfs",
"sdlkdfFkoSSdggs",
"sdlkdfFkosssdsd",
"FkoS",
"Fkos",
"ffkoSS",
"ffkoß",
"😃sadlksffkosSsh😃klF",
"😱slgffkosSsh😃klF",
"FFKoSS",
],
"%FFkoSS%",
ilike_utf8_scalar,
ilike_utf8_scalar_dyn,
vec![false, true, true, false, false, false, false, true, true, true]
);

test_utf8_scalar!(
test_utf8_array_ilike_unicode_complex,
test_utf8_array_ilike_unicode_complex_dyn,
vec![
"sdlkdfFooßsdfs",
"sdlkdfFooSSdggs",
"sdlkdfFoosssdsd",
"FooS",
"Foos",
"ffooSS",
"ffooß",
"😃sadlksffofsSsh😃klF",
"😱slgffoesSsh😃klF",
"FFKoSS",
],
"%FF__SS%",
ilike_utf8_scalar,
ilike_utf8_scalar_dyn,
vec![false, true, true, false, false, false, false, true, true, true]
);

test_utf8_scalar!(
test_utf8_array_ilike_scalar_one,
test_utf8_array_ilike_scalar_dyn_one,
Expand Down