Skip to content

Commit

Permalink
Minor: Clarify documentation on PruningStatistics and make test match (
Browse files Browse the repository at this point in the history
  • Loading branch information
alamb authored Apr 9, 2024
1 parent 78f8ef1 commit eb05741
Showing 1 changed file with 16 additions and 11 deletions.
27 changes: 16 additions & 11 deletions datafusion/core/src/physical_optimizer/pruning.rs
Original file line number Diff line number Diff line change
Expand Up @@ -105,19 +105,23 @@ pub trait PruningStatistics {
fn num_containers(&self) -> usize;

/// Return the number of null values for the named column as an
/// `Option<UInt64Array>`.
/// [`UInt64Array`]
///
/// See [`Self::min_values`] for when to return `None` and null values.
///
/// Note: the returned array must contain [`Self::num_containers`] rows
///
/// [`UInt64Array`]: arrow::array::UInt64Array
fn null_counts(&self, column: &Column) -> Option<ArrayRef>;

/// Return the number of rows for the named column in each container
/// as an `Option<UInt64Array>`.
/// as an [`UInt64Array`].
///
/// See [`Self::min_values`] for when to return `None` and null values.
///
/// Note: the returned array must contain [`Self::num_containers`] rows
///
/// [`UInt64Array`]: arrow::array::UInt64Array
fn row_counts(&self, column: &Column) -> Option<ArrayRef>;

/// Returns [`BooleanArray`] where each row represents information known
Expand Down Expand Up @@ -1519,6 +1523,7 @@ mod tests {
array::{BinaryArray, Int32Array, Int64Array, StringArray},
datatypes::{DataType, TimeUnit},
};
use arrow_array::UInt64Array;
use datafusion_common::{ScalarValue, ToDFSchema};
use datafusion_expr::execution_props::ExecutionProps;
use datafusion_expr::expr::InList;
Expand Down Expand Up @@ -1684,10 +1689,10 @@ mod tests {
/// there are containers
fn with_null_counts(
mut self,
counts: impl IntoIterator<Item = Option<i64>>,
counts: impl IntoIterator<Item = Option<u64>>,
) -> Self {
let null_counts: ArrayRef =
Arc::new(counts.into_iter().collect::<Int64Array>());
Arc::new(counts.into_iter().collect::<UInt64Array>());

self.assert_invariants();
self.null_counts = Some(null_counts);
Expand All @@ -1698,10 +1703,10 @@ mod tests {
/// there are containers
fn with_row_counts(
mut self,
counts: impl IntoIterator<Item = Option<i64>>,
counts: impl IntoIterator<Item = Option<u64>>,
) -> Self {
let row_counts: ArrayRef =
Arc::new(counts.into_iter().collect::<Int64Array>());
Arc::new(counts.into_iter().collect::<UInt64Array>());

self.assert_invariants();
self.row_counts = Some(row_counts);
Expand Down Expand Up @@ -1753,13 +1758,13 @@ mod tests {
self
}

/// Add null counts for the specified columm.
/// Add null counts for the specified column.
/// There must be the same number of null counts as
/// there are containers
fn with_null_counts(
mut self,
name: impl Into<String>,
counts: impl IntoIterator<Item = Option<i64>>,
counts: impl IntoIterator<Item = Option<u64>>,
) -> Self {
let col = Column::from_name(name.into());

Expand All @@ -1775,13 +1780,13 @@ mod tests {
self
}

/// Add row counts for the specified columm.
/// Add row counts for the specified column.
/// There must be the same number of row counts as
/// there are containers
fn with_row_counts(
mut self,
name: impl Into<String>,
counts: impl IntoIterator<Item = Option<i64>>,
counts: impl IntoIterator<Item = Option<u64>>,
) -> Self {
let col = Column::from_name(name.into());

Expand All @@ -1797,7 +1802,7 @@ mod tests {
self
}

/// Add contained information for the specified columm.
/// Add contained information for the specified column.
fn with_contained(
mut self,
name: impl Into<String>,
Expand Down

0 comments on commit eb05741

Please sign in to comment.