feat: More memmaps through semaphore

crates/polars-io/src/csv/read/parser.rs

@@ -6,6 +6,7 @@ use polars_core::prelude::*;
 use polars_core::{config, POOL};
 use polars_error::feature_gated;
 use polars_utils::index::Bounded;
+use polars_utils::mmap::MMapSemaphore;
 use rayon::prelude::*;
 
 use super::buffer::Buffer;
@@ -38,7 +39,7 @@ pub fn count_rows(
         polars_utils::open_file(path)?
     };
 
-    let mmap = unsafe { memmap::Mmap::map(&file).unwrap() };
+    let mmap = MMapSemaphore::new_from_file(&file).unwrap();
     let owned = &mut vec![];
     let reader_bytes = maybe_decompress_bytes(mmap.as_ref(), owned)?;
 

crates/polars-io/src/ipc/ipc_reader_async.rs

@@ -7,6 +7,7 @@ use polars_core::datatypes::IDX_DTYPE;
 use polars_core::frame::DataFrame;
 use polars_core::schema::{Schema, SchemaExt};
 use polars_error::{polars_bail, polars_err, to_compute_err, PolarsResult};
+use polars_utils::mmap::MMapSemaphore;
 use polars_utils::pl_str::PlSmallStr;
 
 use crate::cloud::{
@@ -136,7 +137,7 @@ impl IpcReaderAsync {
         // TODO: Only download what is needed rather than the entire file by
         // making use of the projection, row limit, predicate and such.
         let file = tokio::task::block_in_place(|| self.cache_entry.try_open_check_latest())?;
-        let bytes = unsafe { memmap::Mmap::map(&file) }.unwrap();
+        let bytes = MMapSemaphore::new_from_file(&file).unwrap();
 
         let projection = match options.projection.as_deref() {
             Some(projection) => {
@@ -185,7 +186,7 @@ impl IpcReaderAsync {
         // TODO: Only download what is needed rather than the entire file by
         // making use of the projection, row limit, predicate and such.
         let file = tokio::task::block_in_place(|| self.cache_entry.try_open_check_latest())?;
-        let bytes = unsafe { memmap::Mmap::map(&file) }.unwrap();
+        let bytes = MMapSemaphore::new_from_file(&file).unwrap();
         get_row_count(&mut std::io::Cursor::new(bytes.as_ref()))
     }
 }

crates/polars-utils/src/mmap.rs

@@ -282,8 +282,16 @@ pub struct MMapSemaphore {
 impl MMapSemaphore {
     pub fn new_from_file_with_options(
         file: &File,
-        options: MmapOptions,
+        #[allow(unused_mut)] mut options: MmapOptions,
     ) -> PolarsResult<MMapSemaphore> {
+        // Set mmap size based on seek to end when running under Emscripten
+        #[cfg(target_os = "emscripten")]
+        {
+            let mut file = file;
+            let size = file.seek(SeekFrom::End(0)).unwrap();
+            options.len((size - offset) as usize);
+        }
+
         let mmap = unsafe { options.map(file) }?;
 
         #[cfg(target_family = "unix")]