Skip to content

Commit

Permalink
Clear string-tracking hash table when ByteView deduplication is enabl…
Browse files Browse the repository at this point in the history
…ed (#6385)
  • Loading branch information
shanesveller committed Sep 13, 2024
1 parent 889a904 commit ba85fa3
Showing 1 changed file with 17 additions and 0 deletions.
17 changes: 17 additions & 0 deletions arrow-array/src/builder/generic_bytes_view_builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -368,6 +368,9 @@ impl<T: ByteViewType + ?Sized> GenericByteViewBuilder<T> {
let len = self.views_builder.len();
let views = ScalarBuffer::new(self.views_builder.finish(), 0, len);
let nulls = self.null_buffer_builder.finish();
if let Some((ref mut ht, _)) = self.string_tracker.as_mut() {
ht.clear();
}
// SAFETY: valid by construction
unsafe { GenericByteViewArray::new_unchecked(views, completed, nulls) }
}
Expand Down Expand Up @@ -590,6 +593,20 @@ mod tests {
assert_eq!(array.views().get(1), array.views().get(5));
}

#[test]
fn test_string_view_deduplicate_after_finish() {
let mut builder = StringViewBuilder::new().with_deduplicate_strings();

let value_1 = "long string to test string view";
let value_2 = "not so similar string but long";
builder.append_value(value_1);
let _array = builder.finish();
builder.append_value(value_2);
let _array = builder.finish();
builder.append_value(value_1);
let _array = builder.finish();
}

#[test]
fn test_string_view() {
let b1 = Buffer::from(b"world\xFFbananas\xF0\x9F\x98\x81");
Expand Down

0 comments on commit ba85fa3

Please sign in to comment.