-
Notifications
You must be signed in to change notification settings - Fork 750
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Faster ListArray to StringArray conversion #3593
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -45,50 +45,6 @@ impl<OffsetSize: OffsetSizeTrait> GenericStringArray<OffsetSize> { | |
self.value(i).chars().count() | ||
} | ||
|
||
/// Convert a list array to a string array. | ||
/// | ||
/// Note: this performs potentially expensive UTF-8 validation, consider using | ||
/// [`StringBuilder`][crate::builder::StringBuilder] to avoid this | ||
/// | ||
/// # Panics | ||
/// | ||
/// This method panics if the array contains non-UTF-8 data | ||
fn from_list(v: GenericListArray<OffsetSize>) -> Self { | ||
assert_eq!( | ||
v.data_ref().child_data().len(), | ||
1, | ||
"StringArray can only be created from list array of u8 values \ | ||
(i.e. List<PrimitiveArray<u8>>)." | ||
); | ||
let child_data = &v.data_ref().child_data()[0]; | ||
|
||
assert_eq!( | ||
child_data.child_data().len(), | ||
0, | ||
"StringArray can only be created from list array of u8 values \ | ||
(i.e. List<PrimitiveArray<u8>>)." | ||
); | ||
assert_eq!( | ||
child_data.data_type(), | ||
&DataType::UInt8, | ||
"StringArray can only be created from List<u8> arrays, mismatched data types." | ||
); | ||
assert_eq!( | ||
child_data.null_count(), | ||
0, | ||
"The child array cannot contain null values." | ||
); | ||
|
||
let builder = ArrayData::builder(Self::DATA_TYPE) | ||
.len(v.len()) | ||
.offset(v.offset()) | ||
.add_buffer(v.data().buffers()[0].clone()) | ||
.add_buffer(child_data.buffers()[0].slice(child_data.offset())) | ||
.null_bit_buffer(v.data().null_buffer().cloned()); | ||
|
||
Self::from(builder.build().unwrap()) | ||
} | ||
|
||
/// Creates a [`GenericStringArray`] based on an iterator of values without nulls | ||
pub fn from_iter_values<Ptr, I>(iter: I) -> Self | ||
where | ||
|
@@ -208,7 +164,7 @@ impl<OffsetSize: OffsetSizeTrait> From<GenericListArray<OffsetSize>> | |
for GenericStringArray<OffsetSize> | ||
{ | ||
fn from(v: GenericListArray<OffsetSize>) -> Self { | ||
GenericStringArray::<OffsetSize>::from_list(v) | ||
GenericBinaryArray::<OffsetSize>::from(v).into() | ||
} | ||
} | ||
|
||
|
@@ -282,7 +238,8 @@ pub type LargeStringArray = GenericStringArray<i64>; | |
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
use crate::builder::{ListBuilder, StringBuilder}; | ||
use crate::builder::{ListBuilder, PrimitiveBuilder, StringBuilder}; | ||
use crate::types::UInt8Type; | ||
use arrow_buffer::Buffer; | ||
use arrow_schema::Field; | ||
|
||
|
@@ -663,18 +620,28 @@ mod tests { | |
|
||
#[test] | ||
#[should_panic( | ||
expected = "StringArray can only be created from List<u8> arrays, mismatched data types." | ||
expected = "BinaryArray can only be created from List<u8> arrays, mismatched data types." | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is a change, but I'm inclined to think it doesn't matter There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Might be a bit confusing but seems minor. |
||
)] | ||
fn test_string_array_from_list_array_wrong_type() { | ||
_test_generic_string_array_from_list_array_wrong_type::<i32>(); | ||
} | ||
|
||
#[test] | ||
#[should_panic( | ||
expected = "StringArray can only be created from List<u8> arrays, mismatched data types." | ||
expected = "BinaryArray can only be created from List<u8> arrays, mismatched data types." | ||
)] | ||
fn test_large_string_array_from_list_array_wrong_type() { | ||
_test_generic_string_array_from_list_array_wrong_type::<i32>(); | ||
_test_generic_string_array_from_list_array_wrong_type::<i64>(); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This was a mistake it wasn't actually testing large string array 😱 |
||
} | ||
|
||
#[test] | ||
#[should_panic(expected = "Invalid UTF-8 sequence: Utf8Error")] | ||
fn test_list_array_utf8_validation() { | ||
let mut builder = ListBuilder::new(PrimitiveBuilder::<UInt8Type>::new()); | ||
builder.values().append_value(0xFF); | ||
builder.append(true); | ||
let list = builder.finish(); | ||
let _ = StringArray::from(list); | ||
} | ||
|
||
#[test] | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is what previously performed UTF-8 validation