Skip to content

Commit ba85fa3

Browse files
authored
Clear string-tracking hash table when ByteView deduplication is enabled (#6385)
1 parent 889a904 commit ba85fa3

File tree

1 file changed

+17
-0
lines changed

1 file changed

+17
-0
lines changed

arrow-array/src/builder/generic_bytes_view_builder.rs

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -368,6 +368,9 @@ impl<T: ByteViewType + ?Sized> GenericByteViewBuilder<T> {
368368
let len = self.views_builder.len();
369369
let views = ScalarBuffer::new(self.views_builder.finish(), 0, len);
370370
let nulls = self.null_buffer_builder.finish();
371+
if let Some((ref mut ht, _)) = self.string_tracker.as_mut() {
372+
ht.clear();
373+
}
371374
// SAFETY: valid by construction
372375
unsafe { GenericByteViewArray::new_unchecked(views, completed, nulls) }
373376
}
@@ -590,6 +593,20 @@ mod tests {
590593
assert_eq!(array.views().get(1), array.views().get(5));
591594
}
592595

596+
#[test]
597+
fn test_string_view_deduplicate_after_finish() {
598+
let mut builder = StringViewBuilder::new().with_deduplicate_strings();
599+
600+
let value_1 = "long string to test string view";
601+
let value_2 = "not so similar string but long";
602+
builder.append_value(value_1);
603+
let _array = builder.finish();
604+
builder.append_value(value_2);
605+
let _array = builder.finish();
606+
builder.append_value(value_1);
607+
let _array = builder.finish();
608+
}
609+
593610
#[test]
594611
fn test_string_view() {
595612
let b1 = Buffer::from(b"world\xFFbananas\xF0\x9F\x98\x81");

0 commit comments

Comments
 (0)