Skip to content

Commit

Permalink
Support concat_batches for 0 columns (#4662)
Browse files Browse the repository at this point in the history
* Support `concat_batches` for 0 columns

* Add doc comment
  • Loading branch information
Dandandan authored Aug 8, 2023
1 parent 50f161e commit 696cbdb
Showing 1 changed file with 23 additions and 0 deletions.
23 changes: 23 additions & 0 deletions arrow-select/src/concat.rs
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,14 @@ pub fn concat_batches<'a>(
schema: &SchemaRef,
input_batches: impl IntoIterator<Item = &'a RecordBatch>,
) -> Result<RecordBatch, ArrowError> {
// When schema is empty, sum the number of the rows of all batches
if schema.fields().is_empty() {
let num_rows: usize = input_batches.into_iter().map(RecordBatch::num_rows).sum();
let mut options = RecordBatchOptions::default();
options.row_count = Some(num_rows);
return RecordBatch::try_new_with_options(schema.clone(), vec![], &options);
}

let batches: Vec<&RecordBatch> = input_batches.into_iter().collect();
if batches.is_empty() {
return Ok(RecordBatch::new_empty(schema.clone()));
Expand Down Expand Up @@ -142,6 +150,21 @@ mod tests {
assert!(re.is_err());
}

#[test]
fn test_concat_batches_no_columns() {
// Test concat using empty schema / batches without columns
let schema = Arc::new(Schema::empty());

let mut options = RecordBatchOptions::default();
options.row_count = Some(100);
let batch =
RecordBatch::try_new_with_options(schema.clone(), vec![], &options).unwrap();
// put in 2 batches of 100 rows each
let re = concat_batches(&schema, &[batch.clone(), batch]).unwrap();

assert_eq!(re.num_rows(), 200);
}

#[test]
fn test_concat_one_element_vec() {
let arr = Arc::new(PrimitiveArray::<Int64Type>::from(vec![
Expand Down

0 comments on commit 696cbdb

Please sign in to comment.