diff --git a/extendr-polars/Cargo.toml b/extendr-polars/Cargo.toml index e3bc70a..af471af 100644 --- a/extendr-polars/Cargo.toml +++ b/extendr-polars/Cargo.toml @@ -10,8 +10,8 @@ description = "extendr bindings to polars" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -polars = { version = "*", default_features = false, features = ["dtype-struct"]} -polars-core = { version = "*", default_features = false } +polars = { version = "*", features = ["dtype-struct"]} +polars-core = { version = "*" } polars-plan = { version = "*", default_features = false, optional = true } polars-lazy = { version = "*", default_features = false, optional = true } thiserror = "1" diff --git a/extendr-polars/src/export_dataframe.rs b/extendr-polars/src/export_dataframe.rs index 0ee8753..4494671 100644 --- a/extendr-polars/src/export_dataframe.rs +++ b/extendr-polars/src/export_dataframe.rs @@ -25,7 +25,7 @@ pub fn to_rpolars_dataframe(df: pl::DataFrame) -> EResult { // safety: requires a valid array stream pointer, robj_str_ref unsafe fn export_df_as_stream(df: pl::DataFrame, robj_str_ref: &Robj) -> EResult<()> { let stream_ptr = rptr::robj_str_ptr_to_usize(robj_str_ref)? as *mut ffi::ArrowArrayStream; - let schema = df.schema().to_arrow(); + let schema = df.schema().to_arrow(true); let data_type = pl::ArrowDataType::Struct(schema.fields); let field = pl::ArrowField::new("", data_type, false); let iter_boxed = Box::new(odi::OwnedDataFrameIterator::new(df)); diff --git a/extendr-polars/src/odi.rs b/extendr-polars/src/odi.rs index 80e3304..1175741 100644 --- a/extendr-polars/src/odi.rs +++ b/extendr-polars/src/odi.rs @@ -1,7 +1,9 @@ // OwnedDataFrameIterator written by @paleolimbot / nanoarrow in PR for pola-rs/r-polars +use polars::error::PolarsError; use polars_core::utils::arrow; -use polars_core::utils::arrow::datatypes::DataType as ADataType; +use polars_core::utils::arrow::datatypes::ArrowDataType as ADataType; +use polars_core::utils::arrow::array::Array; pub struct OwnedDataFrameIterator { columns: Vec, @@ -12,7 +14,7 @@ pub struct OwnedDataFrameIterator { impl OwnedDataFrameIterator { pub fn new(df: polars::frame::DataFrame) -> Self { - let schema = df.schema().to_arrow(); + let schema = df.schema().to_arrow(true); let data_type = ADataType::Struct(schema.fields); let vs = df.get_columns().to_vec(); Self { @@ -25,20 +27,19 @@ impl OwnedDataFrameIterator { } impl Iterator for OwnedDataFrameIterator { - type Item = std::result::Result, arrow::error::Error>; + type Item = std::result::Result, PolarsError>; fn next(&mut self) -> Option { if self.idx >= self.n_chunks { None } else { // create a batch of the columns with the same chunk no. - let batch_cols = self.columns.iter().map(|s| s.to_arrow(self.idx)).collect(); + let batch_cols: Vec> = self.columns.iter().map(|s| s.to_arrow(self.idx, true)).collect(); self.idx += 1; - let chunk = polars::frame::ArrowChunk::new(batch_cols); let array = arrow::array::StructArray::new( self.data_type.clone(), - chunk.into_arrays(), + batch_cols, std::option::Option::None, ); Some(std::result::Result::Ok(Box::new(array)))