From e668d8f10837ed813b616a80c229b8a52a89710f Mon Sep 17 00:00:00 2001 From: fxpineau Date: Tue, 7 May 2024 15:22:07 +0200 Subject: [PATCH] Add fonctions to get the sum of a multi-order map in a given S-MOC --- src/deser/fits/mod.rs | 10 ++ src/deser/fits/multiordermap.rs | 287 ++++++++++++++++++++++---------- src/deser/fits/skymap.rs | 2 +- src/deser/stcs.rs | 2 +- src/elemset/range/mod.rs | 7 + src/lib.rs | 2 + src/moc/mod.rs | 8 +- src/moc/range/mod.rs | 21 ++- src/mom/mod.rs | 103 ++++++++++++ src/ranges/mod.rs | 84 ++++++++-- src/storage/u64idx/mod.rs | 42 +++++ src/storage/u64idx/op1.rs | 56 ++++++- 12 files changed, 521 insertions(+), 103 deletions(-) create mode 100644 src/mom/mod.rs diff --git a/src/deser/fits/mod.rs b/src/deser/fits/mod.rs index db31ddd..a3622d7 100644 --- a/src/deser/fits/mod.rs +++ b/src/deser/fits/mod.rs @@ -24,6 +24,7 @@ use crate::elemset::{ range::MocRanges, }; use crate::idx::Idx; +use crate::moc::range::op::convert::convert_to_u64; use crate::moc::{ cell::CellMOC, range::{RangeMOC, RangeMocIter}, @@ -103,6 +104,15 @@ impl, R: BufRead> MocType { MocType::Cells(cells) => cells.into_cell_moc_iter().ranges().into_range_moc(), } } + /// WARNING: you have to be sure that 'P' is the same qty as 'Q'!! + pub fn collect_to_u64>(self) -> RangeMOC { + match self { + MocType::Ranges(ranges) => convert_to_u64::(ranges).into_range_moc(), + MocType::Cells(cells) => { + convert_to_u64::(cells.into_cell_moc_iter().ranges()).into_range_moc() + } + } + } } #[derive(Debug)] diff --git a/src/deser/fits/multiordermap.rs b/src/deser/fits/multiordermap.rs index a4b1918..97f984a 100644 --- a/src/deser/fits/multiordermap.rs +++ b/src/deser/fits/multiordermap.rs @@ -1,23 +1,28 @@ -use std::f64::consts::PI; -use std::io::{BufRead, BufReader, Read, Seek}; +use std::{ + f64::consts::PI, + io::{BufRead, BufReader, Read, Seek}, +}; use byteorder::{BigEndian, ReadBytesExt}; -use crate::deser::{ - fits::{ - common::{ - check_keyword_and_parse_uint_val, check_keyword_and_val, consume_primary_hdu, - next_36_chunks_of_80_bytes, +use crate::{ + deser::{ + fits::{ + common::{ + check_keyword_and_parse_uint_val, check_keyword_and_val, consume_primary_hdu, + next_36_chunks_of_80_bytes, + }, + error::FitsError, + keywords::{FitsCard, MocKeywords, MocKeywordsMap, MocOrder, Ordering}, }, - error::FitsError, - keywords::{FitsCard, MocKeywords, MocKeywordsMap, MocOrder, Ordering}, + gz::{is_gz, uncompress}, }, - gz::{is_gz, uncompress}, + elem::valuedcell::valued_cells_to_moc_with_opt, + idx::Idx, + moc::range::RangeMOC, + mom::{HpxMOMIterator, HpxMomIter}, + qty::Hpx, }; -use crate::elem::valuedcell::valued_cells_to_moc_with_opt; -use crate::idx::Idx; -use crate::moc::range::RangeMOC; -use crate::qty::Hpx; /// We expect the FITS file to be a BINTABLE containing a multi-order map. /// To be fast (in execution and development), we start by a non-flexible approach in which we @@ -92,7 +97,7 @@ pub fn from_fits_multiordermap( } fn from_fits_multiordermap_internal( - mut reader: R, + reader: R, cumul_from: f64, cumul_to: f64, asc: bool, @@ -100,72 +105,19 @@ fn from_fits_multiordermap_internal( no_split: bool, reverse_decent: bool, ) -> Result>, FitsError> { - let mut header_block = [b' '; 2880]; - consume_primary_hdu(&mut reader, &mut header_block)?; - // Read the extention HDU - let mut it80 = next_36_chunks_of_80_bytes(&mut reader, &mut header_block)?; - // See Table 10 and 17 in https://fits.gsfc.nasa.gov/standard40/fits_standard40aa-le.pdf - check_keyword_and_val(it80.next().unwrap(), b"XTENSION", b"'BINTABLE'")?; - check_keyword_and_val(it80.next().unwrap(), b"BITPIX ", b"8")?; - check_keyword_and_val(it80.next().unwrap(), b"NAXIS ", b"2")?; - let n_bytes_per_row = check_keyword_and_parse_uint_val::(it80.next().unwrap(), b"NAXIS1 ")?; - let n_rows = check_keyword_and_parse_uint_val::(it80.next().unwrap(), b"NAXIS2 ")?; - check_keyword_and_val(it80.next().unwrap(), b"PCOUNT ", b"0")?; - check_keyword_and_val(it80.next().unwrap(), b"GCOUNT ", b"1")?; - let _n_cols = check_keyword_and_parse_uint_val::(it80.next().unwrap(), b"TFIELDS ")?; - check_keyword_and_val(it80.next().unwrap(), b"TTYPE1 ", b"'UNIQ '")?; - check_keyword_and_val(it80.next().unwrap(), b"TFORM1 ", b"'K '")?; - check_keyword_and_val(it80.next().unwrap(), b"TTYPE2 ", b"'PROBDENSITY'")?; - check_keyword_and_val(it80.next().unwrap(), b"TFORM2 ", b"'D '")?; - // nbits = |BITPIX|xGCOUNTx(PCOUNT+NAXIS1xNAXIS2x...xNAXISn) - // In our case (bitpix = 8, GCOUNT = 1, PCOUNT = 0) => nbytes = n_cells * size_of(T) - // let data_size n_bytes as usize * n_cells as usize; // N_BYTES ok since BITPIX = 8 - // Read MOC keywords - let mut moc_kws = MocKeywordsMap::new(); - 'hr: loop { - for kw_record in &mut it80 { - // Parse only MOC related keywords and ignore others - if let Some(mkw) = MocKeywords::is_moc_kw(kw_record) { - if let Some(previous_mkw) = moc_kws.insert(mkw?) { - // A FITS keyword MUST BE uniq (I may be more relax here, taking the last one and not complaining) - // return Err(FitsError::MultipleKeyword(previous_mkw.keyword_str().to_string())) - eprintln!("WARNING: Keyword '{}' found more than once in a same HDU! We use the first occurrence.", previous_mkw.keyword_str()); - moc_kws.insert(previous_mkw); - } - // else keyword added without error - } else if &kw_record[0..4] == b"END " { - break 'hr; - } - } - // Read next 2880 bytes - it80 = next_36_chunks_of_80_bytes(&mut reader, &mut header_block)?; - } - // Check header params - moc_kws.check_pixtype()?; - moc_kws.check_ordering(Ordering::Nuniq)?; - moc_kws.check_coordsys()?; - // - get MOC depth - let depth_max = match moc_kws.get::() { - Some(MocKeywords::MOCOrder(MocOrder { depth })) => *depth, - _ => return Err(FitsError::MissingKeyword(MocOrder::keyword_string())), - }; - // Read data - let n_byte_skip = (n_bytes_per_row - 16) as usize; - let mut sink = vec![0; n_byte_skip]; - let area_per_cell = (PI / 3.0) / (1_u64 << (depth_max << 1) as u32) as f64; // = 4pi / (12*4^depth) - let mut uniq_val_dens: Vec<(u64, f64, f64)> = Vec::with_capacity(n_rows as usize); - for _ in 0..n_rows { - let uniq = u64::read::<_, BigEndian>(&mut reader)?; - let dens = reader.read_f64::()?; - let (cdepth, _ipix) = Hpx::::from_uniq_hpx(uniq); - let n_sub_cells = (1_u64 << (((depth_max - cdepth) << 1) as u32)) as f64; - uniq_val_dens.push((uniq, dens * n_sub_cells * area_per_cell, dens)); - /*{ - // Discard remaining row bytes - io::copy(&mut reader.by_ref().take(n_byte_skip), &mut io::sink()); - }*/ - reader.read_exact(&mut sink)?; - } + let data_it = MultiOrderMapIterator::open(reader)?; + let depth_max = data_it.depth_max; + let area_per_cell = data_it.area_per_cell; + let uniq_val_dens = data_it + .map(|res_uniq_dens| { + res_uniq_dens.map(|(uniq, dens)| { + let (cdepth, _ipix) = Hpx::::from_uniq_hpx(uniq); + let n_sub_cells = (1_u64 << (((depth_max - cdepth) << 1) as u32)) as f64; + let value = dens * n_sub_cells * area_per_cell; + (uniq, value, dens) + }) + }) + .collect::, FitsError>>()?; // Build the MOC let ranges = valued_cells_to_moc_with_opt( depth_max, @@ -180,13 +132,157 @@ fn from_fits_multiordermap_internal( Ok(RangeMOC::new(depth_max, ranges)) } +/// Returns the sum of the multi-order map values associated with cells inside the given MOC. +/// If a cell is partially covered by the MOC, we apply on the value a factor equals to the ratio +/// of the cell area covered by the MOC over the total cell area. +pub fn sum_from_fits_multiordermap( + mut reader: BufReader, + moc: &RangeMOC>, +) -> Result { + if is_gz(&mut reader)? { + let reader = uncompress(reader); + sum_from_fits_multiordermap_internal(reader, moc) + } else { + sum_from_fits_multiordermap_internal(reader, moc) + } +} + +fn sum_from_fits_multiordermap_internal( + reader: R, + moc: &RangeMOC>, +) -> Result { + let data_it = MultiOrderMapIterator::open(reader)?; + let depth_max = data_it.depth_max; + let area_per_cell = data_it.area_per_cell; + let mom = data_it + .map(|res_uniq_dens| { + res_uniq_dens.map(|(uniq, dens)| { + let (cdepth, _ipix) = Hpx::::from_uniq_hpx(uniq); + let n_sub_cells = (1_u64 << (((depth_max - cdepth) << 1) as u32)) as f64; + let value = dens * n_sub_cells * area_per_cell; + (uniq, value) + }) + }) + .collect::, FitsError>>()?; + let mom_it = HpxMomIter::, f64, _>::new(mom.into_iter()); + Ok(mom_it.sum_values_in_hpxmoc(moc)) +} + +struct MultiOrderMapIterator { + /// Reader + reader: R, + /// MOM depth + depth_max: u8, + /// Area of a cell at the max MOM depth + area_per_cell: f64, + /// Number of rows to be read + n_rows: u64, + /// Number of rows already returned + n_rows_consumed: u64, + /// Used to consume row bytes we are not interested in + sink: Vec, +} + +impl MultiOrderMapIterator { + fn open(mut reader: R) -> Result { + let mut header_block = [b' '; 2880]; + consume_primary_hdu(&mut reader, &mut header_block)?; + // Read the extention HDU + let mut it80 = next_36_chunks_of_80_bytes(&mut reader, &mut header_block)?; + // See Table 10 and 17 in https://fits.gsfc.nasa.gov/standard40/fits_standard40aa-le.pdf + check_keyword_and_val(it80.next().unwrap(), b"XTENSION", b"'BINTABLE'")?; + check_keyword_and_val(it80.next().unwrap(), b"BITPIX ", b"8")?; + check_keyword_and_val(it80.next().unwrap(), b"NAXIS ", b"2")?; + let n_bytes_per_row = + check_keyword_and_parse_uint_val::(it80.next().unwrap(), b"NAXIS1 ")?; + let n_rows = check_keyword_and_parse_uint_val::(it80.next().unwrap(), b"NAXIS2 ")?; + check_keyword_and_val(it80.next().unwrap(), b"PCOUNT ", b"0")?; + check_keyword_and_val(it80.next().unwrap(), b"GCOUNT ", b"1")?; + let _n_cols = check_keyword_and_parse_uint_val::(it80.next().unwrap(), b"TFIELDS ")?; + check_keyword_and_val(it80.next().unwrap(), b"TTYPE1 ", b"'UNIQ '")?; + check_keyword_and_val(it80.next().unwrap(), b"TFORM1 ", b"'K '")?; + check_keyword_and_val(it80.next().unwrap(), b"TTYPE2 ", b"'PROBDENSITY'")?; + check_keyword_and_val(it80.next().unwrap(), b"TFORM2 ", b"'D '")?; + // nbits = |BITPIX|xGCOUNTx(PCOUNT+NAXIS1xNAXIS2x...xNAXISn) + // In our case (bitpix = 8, GCOUNT = 1, PCOUNT = 0) => nbytes = n_cells * size_of(T) + // let data_size n_bytes as usize * n_cells as usize; // N_BYTES ok since BITPIX = 8 + // Read MOC keywords + let mut moc_kws = MocKeywordsMap::new(); + 'hr: loop { + for kw_record in &mut it80 { + // Parse only MOC related keywords and ignore others + if let Some(mkw) = MocKeywords::is_moc_kw(kw_record) { + if let Some(previous_mkw) = moc_kws.insert(mkw?) { + // A FITS keyword MUST BE uniq (I may be more relax here, taking the last one and not complaining) + // return Err(FitsError::MultipleKeyword(previous_mkw.keyword_str().to_string())) + eprintln!("WARNING: Keyword '{}' found more than once in a same HDU! We use the first occurrence.", previous_mkw.keyword_str()); + moc_kws.insert(previous_mkw); + } + // else keyword added without error + } else if &kw_record[0..4] == b"END " { + break 'hr; + } + } + // Read next 2880 bytes + it80 = next_36_chunks_of_80_bytes(&mut reader, &mut header_block)?; + } + // Check header params + moc_kws.check_pixtype()?; + moc_kws.check_ordering(Ordering::Nuniq)?; + moc_kws.check_coordsys()?; + // - get MOC depth + let depth_max = match moc_kws.get::() { + Some(MocKeywords::MOCOrder(MocOrder { depth })) => Ok(*depth), + _ => Err(FitsError::MissingKeyword(MocOrder::keyword_string())), + }?; + let n_byte_skip = (n_bytes_per_row - 16) as usize; + let sink = vec![0; n_byte_skip]; + let area_per_cell = (PI / 3.0) / (1_u64 << (depth_max << 1) as u32) as f64; // = 4pi / (12*4^depth) + Ok(Self { + reader, + depth_max, + area_per_cell, + n_rows, + n_rows_consumed: 0, + sink, + }) + } +} +impl Iterator for MultiOrderMapIterator { + type Item = Result<(u64, f64), FitsError>; + + fn next(&mut self) -> Option { + if self.n_rows_consumed < self.n_rows { + self.n_rows_consumed += 1; + Some( + u64::read::<_, BigEndian>(&mut self.reader) + .and_then(|uniq| { + self.reader.read_f64::().and_then(|dens| { + self + .reader + .read_exact(&mut self.sink) + .map(|()| (uniq, dens)) + }) + }) + .map_err(FitsError::Io), + ) + } else { + None + } + } + + fn size_hint(&self) -> (usize, Option) { + let n_rows_remaining = (self.n_rows - self.n_rows_consumed) as usize; + (n_rows_remaining, Some(n_rows_remaining)) + } +} + #[cfg(test)] mod tests { - use super::from_fits_multiordermap; - use std::fs::File; - use std::io::BufReader; - use std::path::PathBuf; + use std::{fs::File, io::BufReader, path::PathBuf}; + + use super::{from_fits_multiordermap, sum_from_fits_multiordermap}; #[test] fn test_mutliordermap() { @@ -208,4 +304,25 @@ mod tests { } } } + + #[test] + fn test_mutliordermap_sum() { + let path_buf1 = PathBuf::from("resources/LALInference.multiorder.fits"); + let path_buf2 = PathBuf::from("../resources/LALInference.multiorder.fits"); + let file = File::open(&path_buf1) + .or_else(|_| File::open(&path_buf2)) + .unwrap(); + let reader = BufReader::new(file); + // First create MOC + let moc = from_fits_multiordermap(reader, 0.0, 0.9, false, true, true, false).unwrap(); + + // Then compute the sum inside the MOC (should be 90%, i.e, 0.9). + let file = File::open(&path_buf1) + .or_else(|_| File::open(&path_buf2)) + .unwrap(); + let reader = BufReader::new(file); + let sum = sum_from_fits_multiordermap(reader, &moc).unwrap(); + println!("value: {}", sum); + assert!((0.8999..0.9001).contains(&sum)); + } } diff --git a/src/deser/fits/skymap.rs b/src/deser/fits/skymap.rs index fe319fa..593fbb7 100644 --- a/src/deser/fits/skymap.rs +++ b/src/deser/fits/skymap.rs @@ -152,7 +152,7 @@ fn from_fits_skymap_internal( // TODO: ALSO SUPPORT B or 1B and TTYPE = M (for MASK)!! Err(FitsError::UnexpectedValue( String::from("TFORM1"), - String::from("'D', '1D', 'E', '1E' or '1024E'"), + String::from("['D', '1D', 'E', '1E' or '1024E']"), String::from(tform1), )) }?; diff --git a/src/deser/stcs.rs b/src/deser/stcs.rs index e445885..0c99703 100644 --- a/src/deser/stcs.rs +++ b/src/deser/stcs.rs @@ -329,7 +329,7 @@ impl SpaceVisitor for Stc2Moc { self, interval: &PositionInterval, ) -> Result { - // We use compound visitor onyl to check interval parameters + // We use compound visitor only to check interval parameters self.new_compound_visitor(&interval.pre, &interval.post)?; let depth = self.depth; let corners = interval diff --git a/src/elemset/range/mod.rs b/src/elemset/range/mod.rs index 490dd7e..dd12c61 100644 --- a/src/elemset/range/mod.rs +++ b/src/elemset/range/mod.rs @@ -95,6 +95,9 @@ impl<'a, T: Idx, Q: MocQty> SNORanges<'a, T> for MocRanges { self.0.contains_range(x) } + fn range_fraction(&self, x: &Range) -> f64 { + self.0.range_fraction(x) + } fn intersects(&self, rhs: &Self) -> bool { self.0.intersects(&rhs.0) } @@ -292,6 +295,10 @@ impl<'a, T: Idx, Q: MocQty> SNORanges<'a, T> for BorrowedMocRanges<'a, T, Q> self.0.contains_range(x) } + fn range_fraction(&self, x: &Range) -> f64 { + self.0.range_fraction(x) + } + fn intersects(&self, rhs: &Self) -> bool { self.0.intersects(&rhs.0) } diff --git a/src/lib.rs b/src/lib.rs index 0b71144..55db32b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,6 +25,8 @@ pub mod mocranges2d; pub mod deser; +pub mod mom; + #[cfg(feature = "storage")] pub mod storage; pub mod utils; diff --git a/src/moc/mod.rs b/src/moc/mod.rs index 74d3030..b408529 100644 --- a/src/moc/mod.rs +++ b/src/moc/mod.rs @@ -263,14 +263,14 @@ pub trait RangeMOCIterator: Sized + MOCProperties + Iterator f64 { - let rsum = self.range_sum(); - let tot = Self::Qty::upper_bound_exclusive(); + let mut rsum = self.range_sum(); + let mut tot = Self::Qty::upper_bound_exclusive(); if T::N_BITS > 52 { // 52 = n mantissa bits in a f64 // Divide by the same power of 2, dropping the LSBs let shift = (T::N_BITS - 52) as u32; - rsum.unsigned_shr(shift); - tot.unsigned_shr(shift); + rsum = rsum.unsigned_shr(shift); + tot = tot.unsigned_shr(shift); } rsum.cast_to_f64() / tot.cast_to_f64() } diff --git a/src/moc/range/mod.rs b/src/moc/range/mod.rs index 34e19c3..5992ef1 100644 --- a/src/moc/range/mod.rs +++ b/src/moc/range/mod.rs @@ -195,6 +195,25 @@ impl> RangeMOC { self.ranges.contains_range(&range.0) } + /// Returns the fraction of the given cell covered by this MOC. + /// So: + /// * if the given cell if fully covered, the result is 1.0. + /// * if half of the given cell is covered, the result is 0.5. + /// * if the given cell is **not** covered, the result is 0.0. + pub fn cell_fraction(&self, depth: u8, idx: T) -> f64 { + let range = MocRange::::from((depth, idx)); + self.range_fraction(&range) + } + + /// Returns the fraction of the given range `x` covered by this (`self`) MOC. + /// So: + /// * if the given range `x` if fully covered, the result is 1.0. + /// * if half of the given range `x` is covered, the result is 0.5. + /// * if the given range is **not** covered, the result is 0.0. + pub fn range_fraction(&self, x: &MocRange) -> f64 { + self.ranges.range_fraction(&x.0) + } + pub fn append_fixed_depth_cells>( self, depth: u8, @@ -418,7 +437,7 @@ impl RangeMOC> { ))) } - /// Fill the possible holes in MOC whoch are smaller than the given `sky_fraction` (in `[0, 1]) + /// Fill the possible holes in MOC which are smaller than the given `sky_fraction` (in `[0, 1]) /// This operation may be an heavy operation, here the algorithm we use: /// * perform a `split_into_joint_mocs` operation on the moc `complement` /// * remove the sub-mocs covering more than the given sky fraction diff --git a/src/mom/mod.rs b/src/mom/mod.rs new file mode 100644 index 0000000..0ddf631 --- /dev/null +++ b/src/mom/mod.rs @@ -0,0 +1,103 @@ +//! Multi-Ordered healpix Map (MOM) +//! Here we assume that a MOM a simply a set of of `(key, value)` pairs in which: +//! * the `key` is a UNIQ NESTED HEALPix cell number +//! * all the `keys` in the set are non-overlapping +//! * we assume nothing on the `key` ordering +//! * the `value` (e.g. a probability) is proportional to the cell area, i.e. we can split +//! a `(key, value)` at the order N+1 into the 4 `(key, value)` pairs: +//! + `(key << 2 + 0, value / 4)` +//! + `(key << 2 + 1, value / 4)` +//! + `(key << 2 + 2, value / 4)` +//! + `(key << 2 + 3, value / 4)` + +use std::{ + f64, + marker::PhantomData, + ops::{AddAssign, Mul}, +}; + +use num::Num; + +use crate::{ + idx::Idx, + moc::range::RangeMOC, + qty::{Hpx, MocQty}, +}; + +// 'static mean that Idx does not contains any reference +pub trait Value: + 'static + + Num + + PartialOrd + + Mul + + AddAssign + + Copy + + Send + + Sync + + std::fmt::Debug +{ +} + +impl Value for f64 {} + +/* +trait MOMIterOp { + pub fn sum_values_in_moc +}*/ + +pub trait MOMIterator, V: Value>: Sized + Iterator { + fn sum_values_in_moc(self, moc: &RangeMOC) -> V { + let mut sum = V::zero(); + for (zuniq, value) in self { + let (depth, ipix) = Q::from_zuniq(zuniq); + let cell_fraction = moc.cell_fraction(depth, ipix); + sum += value * cell_fraction; + } + sum + } +} + +pub trait HpxMOMIterator>: MOMIterator, V> { + fn sum_values_in_hpxmoc(self, moc: &RangeMOC>) -> V { + let mut sum = V::zero(); + for (hpx_uniq, value) in self { + let (depth, ipix) = Hpx::::from_uniq_hpx(hpx_uniq); + let cell_fraction = moc.cell_fraction(depth, ipix); + sum += value * cell_fraction; + } + sum + } +} + +pub struct HpxMomIter, V: Value, I: Sized + Iterator> { + it: I, + _phantom: PhantomData, +} +impl, V: Value, I: Sized + Iterator> HpxMomIter { + pub fn new(it: I) -> Self { + Self { + it, + _phantom: PhantomData, + } + } +} + +impl, V: Value, I: Sized + Iterator> Iterator + for HpxMomIter +{ + type Item = (T, V); + + fn next(&mut self) -> Option { + self.it.next() + } +} + +impl, V: Value, I: Sized + Iterator> + MOMIterator, V> for HpxMomIter +{ +} + +impl, V: Value, I: Sized + Iterator> HpxMOMIterator + for HpxMomIter +{ +} diff --git a/src/ranges/mod.rs b/src/ranges/mod.rs index 8882ec8..ff82e52 100644 --- a/src/ranges/mod.rs +++ b/src/ranges/mod.rs @@ -1,15 +1,15 @@ //! Very generic ranges operations -use std::cmp; -use std::cmp::Ordering; -use std::collections::VecDeque; -use std::mem; -use std::ops::{Index, Range}; -use std::ptr::slice_from_raw_parts; -use std::slice::Iter; +use std::{ + cmp::{self, Ordering}, + collections::VecDeque, + mem, + ops::{Index, Range}, + ptr::slice_from_raw_parts, + slice::Iter, +}; use num::{Integer, One, PrimInt, Zero}; - #[cfg(not(target_arch = "wasm32"))] use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; #[cfg(not(target_arch = "wasm32"))] @@ -56,6 +56,13 @@ pub trait SNORanges<'a, T: Idx>: Sized { .any(|a| a) } + /// Returns the fraction of the given range `x` covered by this (`self`) set of ranges. + /// So: + /// * if the given range `x` if fully covered, the result is 1.0. + /// * if half of the given range `x` is covered, the result is 0.5. + /// * if the given range is **not** covered, the result is 0.0. + fn range_fraction(&self, x: &Range) -> f64; + fn contains(&self, rhs: &'a Self) -> bool { // TODO: implement a more efficient algo, avoiding to re-explore the sub-part of self // already explored (since rhs ranges are also ordered!) @@ -225,6 +232,10 @@ impl<'a, T: Idx> SNORanges<'a, T> for Ranges { BorrowedRanges(&self.0).contains_range(x) } + fn range_fraction(&self, x: &Range) -> f64 { + BorrowedRanges(&self.0).range_fraction(x) + } + fn intersects(&self, rhs: &Self) -> bool { BorrowedRanges(&self.0).intersects(&BorrowedRanges(&rhs.0)) } @@ -370,8 +381,61 @@ impl<'a, T: Idx> SNORanges<'a, T> for BorrowedRanges<'a, T> { } } + // The MOC **MUST BE** consistent! + /// Warning: if the order difference is larger that 26, the result may be approximate (52 bit + /// in a f64 mantissa!) + fn range_fraction(&self, x: &Range) -> f64 { + let mut width = T::zero(); + let ranges = self.0; + if ranges.is_empty() || x.end <= ranges[0].start || ranges[ranges.len() - 1].end <= x.start { + // quick rejection test + 0.0 + } else { + // Find the starting range + let i = match ranges.binary_search_by(|range| range.start.cmp(&x.start)) { + Ok(i) => i, + Err(i) => { + if i > 0 && ranges[i - 1].end > x.start { + i - 1 + } else { + i + } + } + }; + // Iterate from the starting element + for range in ranges[i..].iter() { + if x.end <= range.start { + // |--x--| |--range--| + break; + } else { + let start = range.start.max(x.start); + let end = range.end.min(x.end); + width += end - start; + } + } + // Compute fraction + let mut tot = x.end - x.start; + if width == T::zero() { + 0.0 + } else if width == tot { + 1.0 + } else { + // Deal with numerical precision... + if tot.unsigned_shr(52) > T::zero() { + // 52 = n mantissa bits in a f64 + // Divide by the same power of 2, dropping the LSBs + // Shift chosen so that 'tot' leading 1 is lower than 1^52 + let shift = T::N_BITS as u32 - tot.unsigned_shr(52).leading_zeros(); + width = width.unsigned_shr(shift); + tot = tot.unsigned_shr(shift); + } + width.cast_to_f64() / tot.cast_to_f64() + } + } + } + fn intersects(&self, rhs: &Self) -> bool { - // Quickly adaptedd from "intersection", we may find a better option + // Quickly adapted from "intersection", we may find a better option let l = &self.0; let r = &rhs.0; // Quick rejection test @@ -623,7 +687,7 @@ impl<'a, T: Idx> SNORanges<'a, T> for BorrowedRanges<'a, T> { } fn intersection(&self, other: &Self) -> Self::OwnedRanges { - // utils.flatten()/unfallten() + // utils.flatten()/unflaten() let l = &self.0; let r = &other.0; // Quick rejection test diff --git a/src/storage/u64idx/mod.rs b/src/storage/u64idx/mod.rs index 0a34ef3..c7c5b77 100644 --- a/src/storage/u64idx/mod.rs +++ b/src/storage/u64idx/mod.rs @@ -22,6 +22,7 @@ use rayon::iter::{ IndexedParallelIterator, IntoParallelIterator, IntoParallelRefMutIterator, ParallelIterator, }; +use crate::storage::u64idx::op1::{op1_mom_sum, op1_mom_sum_from_data, op1_mom_sum_from_path}; use crate::{ deser::{ ascii::{from_ascii_ivoa, moc2d_from_ascii_ivoa}, @@ -1797,6 +1798,47 @@ impl U64MocStore { op1_count_split(index, true) } + /// Sum the value of the given multi-order map which are in the given MOC. + /// Remark: we have no information and cannot make any guess on the order if te `UNIQ` cell + /// in the iterator. + /// # Params + /// * `index`: index pf the S-MOC in the storage + /// * `mom_it`: iterator on non-overlapping `(uniq, value)` pairs. + pub fn multiordermap_sum_in_moc(&self, index: usize, mom_it: I) -> Result + where + I: Sized + Iterator, + { + op1_mom_sum(index, mom_it) + } + + /// Sum the value of the multi-order map in the given path which are in the given MOC. + /// Remark: we have no information and cannot make any guess on the order if te `UNIQ` cell + /// in the iterator. + /// # Params + /// * `index`: index pf the S-MOC in the storage + /// * `mom_path`: path of the MOM FITS file. + pub fn multiordermap_sum_in_moc_from_path>( + &self, + index: usize, + mom_path: P, + ) -> Result { + op1_mom_sum_from_path(index, mom_path) + } + + /// Sum the value of the multi-order map in the given FITS data which are in the given MOC. + /// Remark: we have no information and cannot make any guess on the order if te `UNIQ` cell + /// in the iterator. + /// # Params + /// * `index`: index pf the S-MOC in the storage + /// * `data`: raw FITS file containing the MOM + pub fn multiordermap_sum_in_moc_from_data( + &self, + index: usize, + mom_data: &[u8], + ) -> Result { + op1_mom_sum_from_data(index, mom_data) + } + pub fn degrade(&self, index: usize, new_depth: u8) -> Result { Op1::Degrade { new_depth }.exec(index) } diff --git a/src/storage/u64idx/op1.rs b/src/storage/u64idx/op1.rs index 9dae0c7..6b2505f 100644 --- a/src/storage/u64idx/op1.rs +++ b/src/storage/u64idx/op1.rs @@ -1,4 +1,15 @@ -use crate::moc::{CellMOCIntoIterator, CellMOCIterator, RangeMOCIntoIterator, RangeMOCIterator}; +use std::{ + fs::File, + io::{BufReader, Cursor}, + path::Path, +}; + +use crate::{ + deser::fits::multiordermap::sum_from_fits_multiordermap, + moc::{CellMOCIntoIterator, CellMOCIterator, RangeMOCIntoIterator, RangeMOCIterator}, + mom::{HpxMOMIterator, HpxMomIter}, + qty::Hpx, +}; use super::{ common::{InternalMoc, FMOC, SMOC, STMOC, TMOC}, @@ -241,3 +252,46 @@ pub(crate) fn op1_1st_axis_max(index: usize) -> Result, String> { }) }) } + +pub(crate) fn op1_mom_sum(index: usize, it: I) -> Result +where + I: Sized + Iterator, +{ + store::exec_on_one_readonly_moc(index, move |moc| match moc { + InternalMoc::Space(moc) => { + let mom_it = HpxMomIter::, f64, _>::new(it); + Ok(mom_it.sum_values_in_hpxmoc(&moc)) + } + InternalMoc::Time(_) => Err(String::from("MOM Sum not implemented for T-MOCs.")), + InternalMoc::Frequency(_) => Err(String::from("MOM Sum not implemented for F-MOCs.")), + InternalMoc::TimeSpace(_) => Err(String::from("MOM Sum not implemented for ST-MOCs.")), + }) +} + +pub(crate) fn op1_mom_sum_from_path>( + index: usize, + mom_path: P, +) -> Result { + store::exec_on_one_readonly_moc(index, move |moc| match moc { + InternalMoc::Space(moc) => { + let file = File::open(&mom_path).map_err(|e| e.to_string())?; + let reader = BufReader::new(file); + sum_from_fits_multiordermap(reader, &moc).map_err(|e| e.to_string()) + } + InternalMoc::Time(_) => Err(String::from("MOM Sum not implemented for T-MOCs.")), + InternalMoc::Frequency(_) => Err(String::from("MOM Sum not implemented for F-MOCs.")), + InternalMoc::TimeSpace(_) => Err(String::from("MOM Sum not implemented for ST-MOCs.")), + }) +} + +pub(crate) fn op1_mom_sum_from_data(index: usize, mom_data: &[u8]) -> Result { + store::exec_on_one_readonly_moc(index, move |moc| match moc { + InternalMoc::Space(moc) => { + sum_from_fits_multiordermap(BufReader::new(Cursor::new(mom_data)), &moc) + .map_err(|e| e.to_string()) + } + InternalMoc::Time(_) => Err(String::from("MOM Sum not implemented for T-MOCs.")), + InternalMoc::Frequency(_) => Err(String::from("MOM Sum not implemented for F-MOCs.")), + InternalMoc::TimeSpace(_) => Err(String::from("MOM Sum not implemented for ST-MOCs.")), + }) +}