refactor: Refactor parts of IR. (#16899)

crates/polars-io/src/csv/read/options.rs

@@ -21,7 +21,7 @@ pub struct CsvReadOptions {
     pub n_rows: Option<usize>,
     pub row_index: Option<RowIndex>,
     // Column-wise options
-    pub columns: Option<Arc<Vec<String>>>,
+    pub columns: Option<Arc<[String]>>,
     pub projection: Option<Arc<Vec<usize>>>,
     pub schema: Option<SchemaRef>,
     pub schema_overwrite: Option<SchemaRef>,
@@ -145,7 +145,7 @@ impl CsvReadOptions {
     }
 
     /// Which columns to select.
-    pub fn with_columns(mut self, columns: Option<Arc<Vec<String>>>) -> Self {
+    pub fn with_columns(mut self, columns: Option<Arc<[String]>>) -> Self {
         self.columns = columns;
         self
     }

crates/polars-io/src/csv/read/read_impl.rs

@@ -144,7 +144,7 @@ impl<'a> CoreReader<'a> {
         has_header: bool,
         ignore_errors: bool,
         schema: Option<SchemaRef>,
-        columns: Option<Arc<Vec<String>>>,
+        columns: Option<Arc<[String]>>,
         encoding: CsvEncoding,
         mut n_threads: Option<usize>,
         schema_overwrite: Option<SchemaRef>,

crates/polars-io/src/ipc/ipc_reader_async.rs

@@ -25,7 +25,7 @@ pub struct IpcReaderAsync {
 #[derive(Default, Clone)]
 pub struct IpcReadOptions {
     // Names of the columns to include in the output.
-    projection: Option<Vec<String>>,
+    projection: Option<Arc<[String]>>,
 
     // The maximum number of rows to include in the output.
     row_limit: Option<usize>,
@@ -38,8 +38,8 @@ pub struct IpcReadOptions {
 }
 
 impl IpcReadOptions {
-    pub fn with_projection(mut self, indices: impl Into<Option<Vec<String>>>) -> Self {
-        self.projection = indices.into();
+    pub fn with_projection(mut self, projection: Option<Arc<[String]>>) -> Self {
+        self.projection = projection;
         self
     }
 

crates/polars-lazy/src/physical_plan/executors/python_scan.rs

@@ -26,8 +26,7 @@ impl Executor for PythonScanExec {
 
             let python_scan_function = self.options.scan_fn.take().unwrap().0;
 
-            let with_columns =
-                with_columns.map(|mut cols| std::mem::take(Arc::make_mut(&mut cols)));
+            let with_columns = with_columns.map(|cols| cols.iter().cloned().collect::<Vec<_>>());
 
             let out = callable
                 .call1((

crates/polars-lazy/src/physical_plan/executors/scan/ipc.rs

@@ -62,10 +62,7 @@ impl IpcExec {
         }
 
         let projection = materialize_projection(
-            self.file_options
-                .with_columns
-                .as_deref()
-                .map(|cols| cols.deref()),
+            self.file_options.with_columns.as_deref(),
             &self.schema,
             None,
             self.file_options.row_index.is_some(),
@@ -193,9 +190,7 @@ impl IpcExec {
                                     }),
                                 )
                                 .with_row_index(this.file_options.row_index.clone())
-                                .with_projection(
-                                    this.file_options.with_columns.as_deref().cloned(),
-                                ),
+                                .with_projection(this.file_options.with_columns.as_ref().cloned()),
                             verbose,
                         )
                         .await?;

crates/polars-lazy/src/physical_plan/executors/scan/mod.rs

@@ -10,8 +10,6 @@ mod parquet;
 #[cfg(feature = "ipc")]
 mod support;
 use std::mem;
-#[cfg(any(feature = "parquet", feature = "ipc", feature = "cse"))]
-use std::ops::Deref;
 
 #[cfg(feature = "csv")]
 pub(crate) use csv::CsvExec;
@@ -38,7 +36,7 @@ type Predicate = Option<Arc<dyn PhysicalIoExpr>>;
 #[cfg(any(feature = "ipc", feature = "parquet"))]
 fn prepare_scan_args(
     predicate: Option<Arc<dyn PhysicalExpr>>,
-    with_columns: &mut Option<Arc<Vec<String>>>,
+    with_columns: &mut Option<Arc<[String]>>,
     schema: &mut SchemaRef,
     has_row_index: bool,
     hive_partitions: Option<&[Series]>,
@@ -47,7 +45,7 @@ fn prepare_scan_args(
     let schema = mem::take(schema);
 
     let projection = materialize_projection(
-        with_columns.as_deref().map(|cols| cols.deref()),
+        with_columns.as_deref(),
         &schema,
         hive_partitions,
         has_row_index,
@@ -62,7 +60,7 @@ fn prepare_scan_args(
 pub struct DataFrameExec {
     pub(crate) df: Arc<DataFrame>,
     pub(crate) selection: Option<Arc<dyn PhysicalExpr>>,
-    pub(crate) projection: Option<Arc<Vec<String>>>,
+    pub(crate) projection: Option<Arc<[String]>>,
     pub(crate) predicate_has_windows: bool,
 }
 

crates/polars-lazy/src/physical_plan/executors/scan/parquet.rs

@@ -173,11 +173,7 @@ impl ParquetExec {
             .unwrap_left();
         let first_metadata = &self.metadata;
         let cloud_options = self.cloud_options.as_ref();
-        let with_columns = self
-            .file_options
-            .with_columns
-            .as_ref()
-            .map(|v| v.as_slice());
+        let with_columns = self.file_options.with_columns.as_ref().map(|v| v.as_ref());
 
         let mut result = vec![];
         let batch_size = get_file_prefetch_size();

crates/polars-lazy/src/physical_plan/planner/lp.rs

@@ -351,7 +351,7 @@ fn create_physical_plan_impl(
         DataFrameScan {
             df,
             projection,
-            selection: predicate,
+            filter: predicate,
             schema,
             ..
         } => {

crates/polars-lazy/src/physical_plan/streaming/construct_pipeline.rs

@@ -242,7 +242,7 @@ fn get_pipeline_node(
         schema: Arc::new(Schema::new()),
         output_schema: None,
         projection: None,
-        selection: None,
+        filter: None,
     });
 
     IR::MapFunction {

crates/polars-lazy/src/tests/optimization_checks.rs

@@ -31,7 +31,7 @@ pub(crate) fn predicate_at_scan(q: LazyFrame) -> bool {
         matches!(
             lp,
             DataFrameScan {
-                selection: Some(_),
+                filter: Some(_),
                 ..
             } | Scan {
                 predicate: Some(_),
@@ -50,7 +50,7 @@ pub(crate) fn predicate_at_all_scans(q: LazyFrame) -> bool {
         matches!(
             lp,
             DataFrameScan {
-                selection: Some(_),
+                filter: Some(_),
                 ..
             } | Scan {
                 predicate: Some(_),
@@ -482,7 +482,7 @@ fn test_with_column_prune() -> PolarsResult<()> {
         match lp {
             DataFrameScan { projection, .. } => {
                 let projection = projection.as_ref().unwrap();
-                let projection = projection.as_slice();
+                let projection = projection.as_ref();
                 assert_eq!(projection.len(), 1);
                 let name = &projection[0];
                 assert_eq!(name, "c1");

crates/polars-pipe/src/executors/sources/parquet.rs

@@ -1,5 +1,5 @@
 use std::collections::VecDeque;
-use std::ops::{Deref, Range};
+use std::ops::Range;
 use std::path::PathBuf;
 use std::sync::Arc;
 
@@ -86,10 +86,7 @@ impl ParquetSource {
             .map(|hive| hive.materialize_partition_columns());
 
         let projection = materialize_projection(
-            file_options
-                .with_columns
-                .as_deref()
-                .map(|cols| cols.deref()),
+            file_options.with_columns.as_deref(),
             &schema,
             hive_partitions.as_deref(),
             false,
@@ -144,11 +141,7 @@ impl ParquetSource {
 
     fn finish_init_reader(&mut self, batched_reader: BatchedParquetReader) -> PolarsResult<()> {
         if self.processed_paths >= 1 {
-            let with_columns = self
-                .file_options
-                .with_columns
-                .as_ref()
-                .map(|v| v.as_slice());
+            let with_columns = self.file_options.with_columns.as_ref().map(|v| v.as_ref());
             check_projected_arrow_schema(
                 batched_reader.schema().as_ref(),
                 self.file_info

crates/polars-pipe/src/pipeline/convert.rs

@@ -54,7 +54,7 @@ where
         DataFrameScan {
             df,
             projection,
-            selection,
+            filter: selection,
             output_schema,
             ..
         } => {
@@ -68,7 +68,7 @@ where
                 }
                 // projection is free
                 if let Some(projection) = projection {
-                    df = df.select(projection.as_slice())?;
+                    df = df.select(projection.as_ref())?;
                 }
             }
             Ok(Box::new(sources::DataFrameSource::from_df(df)) as Box<dyn Source>)

crates/polars-plan/src/dsl/meta.rs

@@ -2,8 +2,8 @@ use std::fmt::Display;
 use std::ops::BitAnd;
 
 use super::*;
-use crate::logical_plan::alp::tree_format::TreeFmtVisitor;
 use crate::logical_plan::conversion::is_regex_projection;
+use crate::logical_plan::ir::tree_format::TreeFmtVisitor;
 use crate::logical_plan::visitor::{AexprNode, TreeWalker};
 
 /// Specialized expressions for Categorical dtypes.

crates/polars-plan/src/logical_plan/anonymous_scan.rs

@@ -8,7 +8,7 @@ use crate::dsl::Expr;
 
 pub struct AnonymousScanArgs {
     pub n_rows: Option<usize>,
-    pub with_columns: Option<Arc<Vec<String>>>,
+    pub with_columns: Option<Arc<[String]>>,
     pub schema: SchemaRef,
     pub output_schema: Option<SchemaRef>,
     pub predicate: Option<Expr>,

crates/polars-plan/src/logical_plan/builder_dsl.rs

@@ -313,7 +313,7 @@ impl DslBuilder {
             schema,
             output_schema: None,
             projection: None,
-            selection: None,
+            filter: None,
         }
         .into()
     }

crates/polars-plan/src/logical_plan/conversion/dsl_to_ir.rs

@@ -20,7 +20,7 @@ fn empty_df() -> IR {
         schema: Arc::new(Default::default()),
         output_schema: None,
         projection: None,
-        selection: None,
+        filter: None,
     }
 }
 
@@ -204,13 +204,13 @@ pub fn to_alp_impl(
             schema,
             output_schema,
             projection,
-            selection,
+            filter: selection,
         } => IR::DataFrameScan {
             df,
             schema,
             output_schema,
             projection,
-            selection: selection.map(|expr| to_expr_ir(expr, expr_arena)),
+            filter: selection.map(|expr| to_expr_ir(expr, expr_arena)),
         },
         DslPlan::Select {
             expr,

crates/polars-plan/src/logical_plan/conversion/mod.rs

@@ -100,13 +100,13 @@ impl IR {
                 schema,
                 output_schema,
                 projection,
-                selection,
+                filter: selection,
             } => DslPlan::DataFrameScan {
                 df,
                 schema,
                 output_schema,
                 projection,
-                selection: selection.map(|e| e.to_expr(expr_arena)),
+                filter: selection.map(|e| e.to_expr(expr_arena)),
             },
             IR::Select {
                 expr,

crates/polars-plan/src/logical_plan/alp/dot.rs → crates/polars-plan/src/logical_plan/ir/dot.rs

@@ -3,7 +3,7 @@ use std::path::PathBuf;
 
 use super::format::ExprIRSliceDisplay;
 use crate::constants::UNLIMITED_CACHE;
-use crate::prelude::alp::format::ColumnsDisplay;
+use crate::prelude::ir::format::ColumnsDisplay;
 use crate::prelude::*;
 
 pub struct IRDotDisplay<'a> {
@@ -153,7 +153,7 @@ impl<'a> IRDotDisplay<'a> {
             #[cfg(feature = "python")]
             PythonScan { predicate, options } => {
                 let predicate = predicate.as_ref().map(|e| self.display_expr(e));
-                let with_columns = NumColumns(options.with_columns.as_ref().map(|s| s.as_slice()));
+                let with_columns = NumColumns(options.with_columns.as_ref().map(|s| s.as_ref()));
                 let total_columns = options.schema.len();
                 let predicate = OptionExprIRDisplay(predicate);
 
@@ -228,10 +228,10 @@ impl<'a> IRDotDisplay<'a> {
             DataFrameScan {
                 schema,
                 projection,
-                selection,
+                filter: selection,
                 ..
             } => {
-                let num_columns = NumColumns(projection.as_ref().map(|p| p.as_ref().as_ref()));
+                let num_columns = NumColumns(projection.as_ref().map(|p| p.as_ref()));
                 let selection = selection.as_ref().map(|e| self.display_expr(e));
                 let selection = OptionExprIRDisplay(selection);
                 let total_columns = schema.len();
@@ -250,7 +250,7 @@ impl<'a> IRDotDisplay<'a> {
             } => {
                 let name: &str = scan_type.into();
                 let path = PathsDisplay(paths.as_ref());
-                let with_columns = options.with_columns.as_ref().map(|cols| cols.as_slice());
+                let with_columns = options.with_columns.as_ref().map(|cols| cols.as_ref());
                 let with_columns = NumColumns(with_columns);
                 let total_columns = file_info.schema.len();
                 let predicate = predicate.as_ref().map(|e| self.display_expr(e));

crates/polars-plan/src/logical_plan/alp/format.rs → crates/polars-plan/src/logical_plan/ir/format.rs

@@ -250,7 +250,7 @@ impl<'a> IRDisplay<'a> {
             DataFrameScan {
                 schema,
                 projection,
-                selection,
+                filter: selection,
                 ..
             } => {
                 let total_columns = schema.len();

crates/polars-plan/src/logical_plan/alp/inputs.rs → crates/polars-plan/src/logical_plan/ir/inputs.rs

@@ -127,7 +127,7 @@ impl IR {
                 schema,
                 output_schema,
                 projection,
-                selection,
+                filter: selection,
             } => {
                 let mut new_selection = None;
                 if selection.is_some() {
@@ -139,7 +139,7 @@ impl IR {
                     schema: schema.clone(),
                     output_schema: output_schema.clone(),
                     projection: projection.clone(),
-                    selection: new_selection,
+                    filter: new_selection,
                 }
             },
             MapFunction { function, .. } => MapFunction {
@@ -188,7 +188,9 @@ impl IR {
                     container.push(pred.clone())
                 }
             },
-            DataFrameScan { selection, .. } => {
+            DataFrameScan {
+                filter: selection, ..
+            } => {
                 if let Some(expr) = selection {
                     container.push(expr.clone())
                 }

crates/polars-plan/src/logical_plan/alp/mod.rs → crates/polars-plan/src/logical_plan/ir/mod.rs

@@ -62,8 +62,8 @@ pub enum IR {
         schema: SchemaRef,
         // schema of the projected file
         output_schema: Option<SchemaRef>,
-        projection: Option<Arc<Vec<String>>>,
-        selection: Option<ExprIR>,
+        projection: Option<Arc<[String]>>,
+        filter: Option<ExprIR>,
     },
     // Only selects columns (semantically only has row access).
     // This is a more restricted operation than `Select`.