From 1558b70316e154666a6a9b435b70223404931b59 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Thu, 8 Dec 2022 01:23:08 -0800 Subject: [PATCH 1/2] Fix null buffer import/export behavior --- arrow/src/ffi.rs | 42 ++++++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/arrow/src/ffi.rs b/arrow/src/ffi.rs index fc8dc654af0c..3c4c7a5080ca 100644 --- a/arrow/src/ffi.rs +++ b/arrow/src/ffi.rs @@ -462,12 +462,18 @@ impl FFI_ArrowArray { /// This method releases `buffers`. Consumers of this struct *must* call `release` before /// releasing this struct, or contents in `buffers` leak. pub fn new(data: &ArrayData) -> Self { - // * insert the null buffer at the start - // * make all others `Option`. - let buffers = iter::once(data.null_buffer().cloned()) - .chain(data.buffers().iter().map(|b| Some(b.clone()))) - .collect::>(); let data_layout = layout(data.data_type()); + + let buffers = if data_layout.can_contain_null_mask { + // * insert the null buffer at the start + // * make all others `Option`. + iter::once(data.null_buffer().cloned()) + .chain(data.buffers().iter().map(|b| Some(b.clone()))) + .collect::>() + } else { + data.buffers().iter().map(|b| Some(b.clone())).collect() + }; + // `n_buffers` is the number of buffers by the spec. let n_buffers = { data_layout.buffers.len() + { @@ -616,8 +622,15 @@ pub trait ArrowArrayRef { let len = self.array().len(); let offset = self.array().offset(); let null_count = self.array().null_count(); - let buffers = self.buffers()?; - let null_bit_buffer = self.null_bit_buffer(); + + let data_layout = layout(&data_type); + let buffers = self.buffers(data_layout.can_contain_null_mask)?; + + let null_bit_buffer = if data_layout.can_contain_null_mask { + self.null_bit_buffer() + } else { + None + }; let mut child_data: Vec = (0..self.array().n_children as usize) .map(|i| { @@ -649,11 +662,16 @@ pub trait ArrowArrayRef { } /// returns all buffers, as organized by Rust (i.e. null buffer is skipped) - fn buffers(&self) -> Result> { - (0..self.array().n_buffers - 1) + fn buffers(&self, can_contain_null_mask: bool) -> Result> { + let buffer_begin = if can_contain_null_mask { + // + 1: skip null buffer + 1 + } else { + 0 + }; + (buffer_begin..self.array().n_buffers) .map(|index| { - // + 1: skip null buffer - let index = (index + 1) as usize; + let index = index as usize; let len = self.buffer_len(index)?; @@ -661,7 +679,7 @@ pub trait ArrowArrayRef { .ok_or_else(|| { ArrowError::CDataInterface(format!( "The external buffer at position {} is null.", - index - 1 + index )) }) }) From 34dcb7092b71fcbbe758317e4a1bfad449941dbb Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies Date: Thu, 8 Dec 2022 14:38:00 +0000 Subject: [PATCH 2/2] Clippy --- arrow/src/ffi.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/arrow/src/ffi.rs b/arrow/src/ffi.rs index 68a9bca7cea1..abb53dff68bd 100644 --- a/arrow/src/ffi.rs +++ b/arrow/src/ffi.rs @@ -663,12 +663,8 @@ pub trait ArrowArrayRef { /// returns all buffers, as organized by Rust (i.e. null buffer is skipped) fn buffers(&self, can_contain_null_mask: bool) -> Result> { - let buffer_begin = if can_contain_null_mask { - // + 1: skip null buffer - 1 - } else { - 0 - }; + // + 1: skip null buffer + let buffer_begin = can_contain_null_mask as i64; (buffer_begin..self.array().n_buffers) .map(|index| { let index = index as usize;