Skip to content

Commit

Permalink
expose and features
Browse files Browse the repository at this point in the history
  • Loading branch information
ritchie46 committed Dec 21, 2024
1 parent 9346989 commit fea15f0
Show file tree
Hide file tree
Showing 7 changed files with 47 additions and 4 deletions.
7 changes: 7 additions & 0 deletions crates/polars-lazy/src/frame/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,13 @@ impl LazyFrame {
self
}

/// Check if operations are order dependent and unset maintaining_order if
/// the order would not be observed.
pub fn with_check_order(mut self, toggle: bool) -> Self {
self.opt_state.set(OptFlags::CHECK_ORDER_OBSERVE, toggle);
self
}

/// Toggle predicate pushdown optimization.
pub fn with_predicate_pushdown(mut self, toggle: bool) -> Self {
self.opt_state.set(OptFlags::PREDICATE_PUSHDOWN, toggle);
Expand Down
3 changes: 3 additions & 0 deletions crates/polars-plan/src/frame/opt_state.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@ bitflags! {
const FAST_PROJECTION = 1 << 14;
/// Collapse slower joins with filters into faster joins.
const COLLAPSE_JOINS = 1 << 15;
/// Check if operations are order dependent and unset maintaining_order if
/// the order would not be observed.
const CHECK_ORDER_OBSERVE = 1 << 16;
}
}

Expand Down
4 changes: 3 additions & 1 deletion crates/polars-plan/src/plans/optimizer/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,9 @@ pub fn optimize(
cache_states::set_cache_states(lp_top, lp_arena, expr_arena, scratch, expr_eval, verbose)?;
}

if members.has_group_by | members.has_sort | members.has_distinct {
if opt_state.contains(OptFlags::CHECK_ORDER_OBSERVE)
&& members.has_group_by | members.has_sort | members.has_distinct
{
set_order_flags(lp_top, lp_arena, expr_arena, scratch);
}

Expand Down
4 changes: 2 additions & 2 deletions crates/polars-plan/src/plans/optimizer/set_order.rs
Original file line number Diff line number Diff line change
Expand Up @@ -105,8 +105,8 @@ pub(super) fn set_order_flags(

if apply.is_some()
|| *maintain_order
|| options.rolling.is_some()
|| options.dynamic.is_some()
|| options.is_rolling()
|| options.is_dynamic()
{
maintain_order_above = true;
continue;
Expand Down
24 changes: 24 additions & 0 deletions crates/polars-plan/src/plans/options.rs
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,30 @@ pub struct GroupbyOptions {
pub slice: Option<(i64, usize)>,
}

impl GroupbyOptions {
pub(crate) fn is_rolling(&self) -> bool {
#[cfg(feature = "dynamic_group_by")]
{
self.rolling.is_some()
}
#[cfg(not(feature = "dynamic_group_by"))]
{
false
}
}

pub(crate) fn is_dynamic(&self) -> bool {
#[cfg(feature = "dynamic_group_by")]
{
self.dynamic.is_some()
}
#[cfg(not(feature = "dynamic_group_by"))]
{
false
}
}
}

#[derive(Clone, Debug, Eq, PartialEq, Default, Hash)]
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
pub struct DistinctOptionsDSL {
Expand Down
2 changes: 2 additions & 0 deletions crates/polars-python/src/lazyframe/general.rs
Original file line number Diff line number Diff line change
Expand Up @@ -494,6 +494,7 @@ impl PyLazyFrame {
collapse_joins: bool,
streaming: bool,
_eager: bool,
_check_order: bool,
#[allow(unused_variables)] new_streaming: bool,
) -> Self {
let ldf = self.ldf.clone();
Expand All @@ -504,6 +505,7 @@ impl PyLazyFrame {
.with_slice_pushdown(slice_pushdown)
.with_cluster_with_columns(cluster_with_columns)
.with_collapse_joins(collapse_joins)
.with_check_order(_check_order)
._with_eager(_eager)
.with_projection_pushdown(projection_pushdown);

Expand Down
7 changes: 6 additions & 1 deletion py-polars/polars/lazyframe/frame.py
Original file line number Diff line number Diff line change
Expand Up @@ -986,7 +986,7 @@ def skip_minmax(dt: PolarsDataType) -> bool:
# reshape wide result
n_metrics = len(metrics)
column_metrics = [
df_metrics.row(0)[(n * n_metrics) : (n + 1) * n_metrics]
df_metrics.row(0)[(n * n_metrics): (n + 1) * n_metrics]
for n in range(schema.len())
]
summary = dict(zip(schema, column_metrics))
Expand Down Expand Up @@ -1768,6 +1768,7 @@ def collect(
engine: EngineType = "cpu",
background: Literal[True],
_eager: bool = False,
_check_order: bool = True,
) -> InProcessQuery: ...

@overload
Expand All @@ -1787,6 +1788,7 @@ def collect(
streaming: bool = False,
engine: EngineType = "cpu",
background: Literal[False] = False,
_check_order: bool = True,
_eager: bool = False,
) -> DataFrame: ...

Expand All @@ -1806,6 +1808,7 @@ def collect(
streaming: bool = False,
engine: EngineType = "cpu",
background: bool = False,
_check_order: bool = True,
_eager: bool = False,
**_kwargs: Any,
) -> DataFrame | InProcessQuery:
Expand Down Expand Up @@ -1974,6 +1977,7 @@ def collect(
comm_subexpr_elim = False
cluster_with_columns = False
collapse_joins = False
_check_order = False

if streaming:
issue_unstable_warning("Streaming mode is considered unstable.")
Expand Down Expand Up @@ -2005,6 +2009,7 @@ def collect(
collapse_joins,
streaming,
_eager,
_check_order,
new_streaming,
)

Expand Down

0 comments on commit fea15f0

Please sign in to comment.