diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 281114f7..8d67d1eb 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -158,7 +158,7 @@ jobs: MATURIN_PASSWORD: ${{ secrets.PYPI_PASSWORD_FXP }} run: | #python3 -m pip install maturin - rustup target add aarch64-apple-darwin + rustup target add x86_64-apple-darwin pip install --upgrade pip pip install maturin maturin -V diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d163c62b..32e8e654 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -63,7 +63,7 @@ jobs: - name: "Build and test wheel for Python ${{ matrix.python-version }} on MacOS" run: | # Install, create and activate a python virtualenv - rustup target add aarch64-apple-darwin + rustup target add x86_64-apple-darwin pip install virtualenv virtualenv mocpy-env source mocpy-env/bin/activate diff --git a/CHANGELOG.md b/CHANGELOG.md index b1dd57e8..56be294c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [unreleased] +### Added + +* `MOC.sum_in_multiordermap` takes an astropy table with a `UNIQ` column and +a column name to sum. It returns the sum of the column in the intersection between the +MOC and the Multi-order-map. `MOC.probability_in_multiordermap` has a similar +behavior but also converts a probability-density into a probability. +* `STMOC.new_empty()` allows to create a new empty Space-Time MOC. + ## [0.13.1] ### Changed diff --git a/Cargo.toml b/Cargo.toml index 4a1e5210..1d3b4144 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,13 +27,13 @@ bench = true crate-type = ["cdylib"] [dependencies] -moc = { version = "0.12", features = ["storage"] } -healpix = { package = "cdshealpix", version = "0.6" } +moc = { version = "0.14", features = ["storage"] } # moc = { git = 'https://github.com/cds-astro/cds-moc-rust', branch = 'main', features = ["storage"] } +healpix = { package = "cdshealpix", version = "0.6" } # healpix = { package = "cdshealpix", git = 'https://github.com/cds-astro/cds-healpix-rust', branch = 'master' } [dependencies.numpy] -version = "0.17" +version = "0.19" [dependencies.ndarray] version = "0.15" @@ -42,7 +42,7 @@ default-features = false # do not include the default features, and optionally features = ["rayon"] [dependencies.pyo3] -version = "0.17.3" +version = "0.19" features = ["extension-module"] [profile.release] diff --git a/docs/examples/bayestar.py b/docs/examples/bayestar.py index 7b4c4030..f286f7b9 100644 --- a/docs/examples/bayestar.py +++ b/docs/examples/bayestar.py @@ -4,12 +4,12 @@ import numpy as np from astropy.coordinates import Angle, SkyCoord from astropy.io import fits - from mocpy import MOC, WCS +# this can be found in mocpy's repository +# https://github.com/cds-astro/mocpy/blob/master/resources/bayestar.multiorder.fits fits_image_filename = "./../../resources/bayestar.multiorder.fits" -max_order = None with fits.open(fits_image_filename) as hdul: hdul.info() data = hdul[1].data @@ -18,13 +18,12 @@ uniq = data["UNIQ"] probdensity = data["PROBDENSITY"] - +# let's convert the probability density into a probability level, ipix = ah.uniq_to_level_ipix(uniq) area = ah.nside_to_pixel_area(ah.level_to_nside(level)).to_value(u.steradian) - prob = probdensity * area - +# now we create the mocs corresponding to different probability thresholds cumul_to = np.linspace(0.5, 0.9, 5)[::-1] colors = ["blue", "green", "yellow", "orange", "red"] mocs = [ @@ -33,8 +32,6 @@ # Plot the MOC using matplotlib - - fig = plt.figure(111, figsize=(15, 10)) # Define a astropy WCS easily with WCS( @@ -58,9 +55,7 @@ label="confidence probability " + str(round(c * 100)) + "%", ) moc.border(ax=ax, wcs=wcs, alpha=0.5, color=col) - ax.legend() - plt.xlabel("ra") plt.ylabel("dec") plt.title("Bayestar") diff --git a/docs/examples/user_documentation.rst b/docs/examples/user_documentation.rst index 25aa134c..26bc4a45 100644 --- a/docs/examples/user_documentation.rst +++ b/docs/examples/user_documentation.rst @@ -82,9 +82,13 @@ In this example: Gravitational Waves MOCs ~~~~~~~~~~~~~~~~~~~~~~~~ -This example shows the probability confidence regions of gravitational waves. HEALPix cells are given under the `uniq pixel notation `__. Each -pixel is associated with a specific value. We can create a MOC from which a GW has x% of change of being localized in it. By definition the MOC which has 100% of chance -of containing a GW is the full sky MOC. +This example shows the probability confidence regions of gravitational waves. +HEALPix cells are given under the +`uniq pixel notation `__. +Each pixel is associated with a specific probability density value. We convert this into +a probability by multiplying it with the area of each cell. +Then, we can create a MOC from which a GW has x% of chance of being localized in it. +By definition the MOC which has 100% of chance of containing a GW is the full sky MOC. .. plot:: examples/bayestar.py :include-source: diff --git a/python/mocpy/moc/moc.py b/python/mocpy/moc/moc.py index b87453e6..2c57d676 100644 --- a/python/mocpy/moc/moc.py +++ b/python/mocpy/moc/moc.py @@ -1,6 +1,7 @@ import contextlib import functools from io import BytesIO +from pathlib import Path from urllib.parse import urlencode import numpy as np @@ -16,6 +17,7 @@ SkyCoord, ) from astropy.io import fits +from astropy.table import Table from astropy.utils.data import download_file with contextlib.suppress(ImportError): @@ -717,6 +719,120 @@ def from_multiordermap_fits_file( ) return cls(index) + def probability_in_multiordermap(self, multiordermap): + """Calculate the probability in the intersection between the multiordermap and the MOC. + + ``PROBDENSITY`` values are multiplied by the area of their associated HEALPix + cell before summing them. For cells that are not complete, the ratio of the area + is used. + + Parameters + ---------- + multiordermap : str, pathlib.Path, astropy.table.Table, or astropy.table.QTable + If ``multiordermap`` is given as a string or `~pathlib.Path`, the probability + will be read from the column ``PROBDENSITY`` of the FITS file. + If it is an `~astropy.table.Table`, then it should have a column ``UNIQ`` that + corresponds to HEALPix cells and a ``PROBDENSITY`` column. + + Returns + ------- + float + The probability in the intersection between the MOC and the Multi-Order-Map + coverages. + + Examples + -------- + >>> from mocpy import MOC + >>> import numpy as np + >>> from astropy.table import Table + >>> all_sky = MOC.from_str("0/0-11") + >>> # Let's create a meaningless multiorder map + >>> uniq = [4 * 4**4 + x for x in range(20)] + >>> rng = np.random.default_rng(0) + >>> probdensity = rng.random(20) / 100 + >>> multi_order_map = Table([uniq, probdensity], names=("UNIQ", "PROBDENSITY")) + >>> # The probability to be in the intersection with the all sky is + >>> round(all_sky.probability_in_multiordermap(multi_order_map), 4) + 0.0004 + + """ + index = self.store_index + + if isinstance(multiordermap, Table): + uniq = multiordermap["UNIQ"] + probdensity = multiordermap["PROBDENSITY"] + try: + uniq_mask = uniq.data.mask + probdensity_mask = probdensity.data.mask + except AttributeError: + uniq_mask = np.zeros(uniq.shape) + probdensity_mask = np.zeros(probdensity.shape) + + return mocpy.multiorder_probdens_map_sum_in_smoc( + index, + np.array(uniq.data, dtype="uint64"), + np.array(uniq_mask, dtype="bool"), + np.array(probdensity.data, dtype="float"), + np.array(probdensity_mask, dtype="bool"), + ) + if isinstance(multiordermap, (Path, str)): + return mocpy.multiordermap_sum_in_smoc_from_file(index, str(multiordermap)) + + raise ValueError( + "An argument of type 'str', 'pathlib.Path', or 'astropy.table.Table'" + f" is expected. Got '{type(multiordermap)}'", + ) + + def sum_in_multiordermap(self, multiordermap: Table, column: str): + """Calculate the sum of a column from a multiordermap in the intersection with the MOC. + + Parameters + ---------- + multiordermap : astropy.table.Table + The table should have a ``UNIQ`` that corresponds to HEALPix cells in the uniq + notation. + column : str + The name of the column to sum. + + Returns + ------- + float + The sum of the values in the intersection between the MOC and the + multiorder map coverages. + + Examples + -------- + >>> from mocpy import MOC + >>> import numpy as np + >>> from astropy.table import Table + >>> all_sky = MOC.from_str("0/0-11") + >>> # Let's create a meaningless multiorder map + >>> uniq = [4 * 4**5 + x for x in np.arange(200)] + >>> rng = np.random.default_rng(0) + >>> column = rng.random(200) + >>> multi_order_map = Table([uniq, column], names=("UNIQ", "column")) + >>> round(all_sky.sum_in_multiordermap(multi_order_map, "column"), 4) + 107.9259 + + """ + index = self.store_index + uniq = multiordermap["UNIQ"] + values_to_sum = multiordermap[column] + try: + uniq_mask = uniq.data.mask + values_to_sum_mask = values_to_sum.data.mask + except AttributeError: + uniq_mask = np.zeros(uniq.shape) + values_to_sum_mask = uniq_mask + + return mocpy.multiordermap_sum_in_smoc( + index, + np.array(uniq.data, dtype="uint64"), + np.array(uniq_mask, dtype="bool"), + np.array(values_to_sum.data, dtype="float"), + np.array(values_to_sum_mask, dtype="bool"), + ) + @classmethod def from_valued_healpix_cells( cls, diff --git a/python/mocpy/stmoc/stmoc.py b/python/mocpy/stmoc/stmoc.py index d637ac4c..7f4f8b7e 100644 --- a/python/mocpy/stmoc/stmoc.py +++ b/python/mocpy/stmoc/stmoc.py @@ -39,7 +39,7 @@ def max_depth(self): @property def max_order(self): """Is a clone of max_depth, to preserve the api between moc types.""" - return self.max_depth() + return self.max_depth @property def max_time(self): @@ -85,10 +85,32 @@ def n_cells(cls, depth, dimension): ) @classmethod - def new_empty(cls): - """Create a new empty STMOC.""" - # TODO - raise NotImplementedError("This method is not implemented yet.") + def new_empty(cls, max_depth_time, max_depth_space): + """Create a new empty STMOC. + + Parameters + ---------- + max_depth_time : int + The time resolution of the STMOC. Should be comprised between 0 and 61. + max_depth_space : int + The space resolution of the STMOC. Should be comprised between 0 and 29. + + Returns + ------- + `~mocpy.moc.STMOC` + + Examples + -------- + >>> from mocpy import STMOC + >>> STMOC.new_empty(42, 12) + t42/ s12/ + + """ + index = mocpy.new_empty_stmoc( + np.uint8(max_depth_time), + np.uint8(max_depth_space), + ) + return cls(index) def is_empty(self): """Check whether the Space-Time coverage is empty.""" diff --git a/python/mocpy/tests/test_moc.py b/python/mocpy/tests/test_moc.py index 64bf701e..e3488764 100644 --- a/python/mocpy/tests/test_moc.py +++ b/python/mocpy/tests/test_moc.py @@ -8,6 +8,7 @@ from astropy.coordinates import Angle, SkyCoord from astropy.io import fits from astropy.io.votable import parse_single_table +from astropy.table import QTable from ..moc import MOC, WCS @@ -740,6 +741,38 @@ def test_from_valued_healpix_cells_bayestar_and_split(): assert moc.max_order == 11 +def test_probability_in_multiordermap(): + # from path + moc = MOC.from_str("0/4") + fits_mom_filename = "./resources/bayestar.multiorder.fits" + proba = moc.probability_in_multiordermap(fits_mom_filename) + assert np.isclose(proba, 0.20877154164727782) + + # has no mask + mom = QTable() + mom["UNIQ"] = [4 + x for x in range(20)] + mom["PROBDENSITY"] = [x / 10 for x in range(20)] + proba = moc.probability_in_multiordermap(mom) + assert np.isclose(proba, 0.41887902047863906) + + # is not a valid mom or path + with pytest.raises( + ValueError, + match="An argument of type 'str', 'pathlib.Path', or " + "'astropy.table.Table' is expected. Got ''", + ): + moc.probability_in_multiordermap(1) + + +def test_sum_in_multiordermap(): + all_sky = MOC.from_str("0/0-11") + mom = QTable() + range_20 = range(20) + mom["UNIQ"] = [4 * 4**3 + x for x in range_20] + mom["TO_SUM"] = range_20 + assert all_sky.sum_in_multiordermap(mom, "TO_SUM") == sum(range_20) + + def test_from_stcs(): moc1 = MOC.from_stcs("Circle ICRS 147.6 69.9 0.4", 14, 2) moc2 = MOC.from_cone( diff --git a/python/mocpy/tests/test_stmoc.py b/python/mocpy/tests/test_stmoc.py index 06c046a2..30ef6d1d 100644 --- a/python/mocpy/tests/test_stmoc.py +++ b/python/mocpy/tests/test_stmoc.py @@ -52,6 +52,10 @@ def stmoc_xmm_dr8(): ) +def test_new_empty(): + assert STMOC.new_empty(0, 0).is_empty() + + def test_n_cells(): assert STMOC.n_cells(0, "time") == 2 assert STMOC.n_cells(0, "space") == 12 diff --git a/src/lib.rs b/src/lib.rs index fd7af245..a14e5ab6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,6 +12,7 @@ use pyo3::{ }; use moc::{ + moc::range::CellSelection, qty::{Frequency, Hpx, MocQty, Time}, storage::u64idx::U64MocStore, utils, @@ -146,7 +147,14 @@ fn mocpy(_py: Python, m: &PyModule) -> PyResult<()> { delta_depth: u8, ) -> PyResult { U64MocStore::get_global_store() - .from_cone(lon_deg, lat_deg, radius_deg, depth, delta_depth) + .from_cone( + lon_deg, + lat_deg, + radius_deg, + depth, + delta_depth, + CellSelection::All, + ) .map_err(PyValueError::new_err) } @@ -180,7 +188,15 @@ fn mocpy(_py: Python, m: &PyModule) -> PyResult<()> { delta_depth: u8, ) -> PyResult { U64MocStore::get_global_store() - .from_ring(lon_deg, lat_deg, r_int_deg, r_ext_deg, depth, delta_depth) + .from_ring( + lon_deg, + lat_deg, + r_int_deg, + r_ext_deg, + depth, + delta_depth, + CellSelection::All, + ) .map_err(PyValueError::new_err) } @@ -195,7 +211,16 @@ fn mocpy(_py: Python, m: &PyModule) -> PyResult<()> { delta_depth: u8, ) -> PyResult { U64MocStore::get_global_store() - .from_elliptical_cone(lon_deg, lat_deg, a_deg, b_deg, pa_deg, depth, delta_depth) + .from_elliptical_cone( + lon_deg, + lat_deg, + a_deg, + b_deg, + pa_deg, + depth, + delta_depth, + CellSelection::All, + ) .map_err(PyValueError::new_err) } @@ -209,7 +234,7 @@ fn mocpy(_py: Python, m: &PyModule) -> PyResult<()> { let lon = lon_deg.as_array().into_iter().cloned(); let lat = lat_deg.as_array().into_iter().cloned(); U64MocStore::get_global_store() - .from_polygon(lon.zip(lat), complement, depth) + .from_polygon(lon.zip(lat), complement, depth, CellSelection::All) .map_err(PyValueError::new_err) } @@ -1342,6 +1367,146 @@ fn mocpy(_py: Python, m: &PyModule) -> PyResult<()> { .map_err(PyIOError::new_err) } + /// Get the sum of the given multiorder map values that are in the S-MOC + /// of given index. + /// + /// # Arguments + /// + /// * ``index`` - The index of the coverage in the store. + /// * ``path`` - The path of the multi-order map file. + /// + /// # Warning + /// * `PROBDENSITY` values are multiplied by the area of the HEALPix + /// cell they are associated with to compute the values that are summed. + /// + /// # Info + /// + /// We expect the FITS file to be a BINTABLE containing a multi-order map. + /// In this non-flexible approach, we expect the BINTABLE extension to contains: + /// + /// ```bash + /// XTENSION= 'BINTABLE' / binary table extension + /// BITPIX = 8 / array data type + /// NAXIS = 2 / number of array dimensions + /// AXIS1 = ?? / length of dimension 1 + /// NAXIS2 = ?? / length of dimension 2 + /// PCOUNT = 0 / number of group parameters + /// GCOUNT = 1 / number of groups + /// TFIELDS = xx / number of table fields + /// TTYPE1 = 'UNIQ ' + /// TFORM1 = 'K ' + /// TTYPE2 = 'PROBDENSITY' + /// TFORM2 = 'D ' + /// TUNIT2 = 'sr-1 ' + /// ... + /// MOC = T + /// PIXTYPE = 'HEALPIX ' / HEALPIX pixelisation + /// ORDERING= 'NUNIQ ' / Pixel ordering scheme: RING, NESTED, or NUNIQ + /// COORDSYS= 'C ' / Ecliptic, Galactic or Celestial (equatorial) + /// MOCORDER= xx / MOC resolution (best order) + /// ... + /// END + /// ``` + #[pyfn(m)] + fn multiordermap_sum_in_smoc_from_file(index: usize, path: String) -> PyResult { + U64MocStore::get_global_store() + .multiordermap_sum_in_moc_from_path(index, path) + .map_err(PyIOError::new_err) + } + + /// Get the sum of the value for the UNIQ HEAPix cells which are in + /// the S-MOC of given index. + /// + /// # Arguments + /// + /// * ``index`` - The index of the coverage in the store. + /// * ``uniq`` - UNIQ HEALPix values. + /// * ``uniq_mask`` - Mask on the UNIQ HEALPix values. + /// * ``value`` - Values to be summed. + /// * ``value_mask`` - mask on the values to be summed. + /// + /// # Warning + /// * contrary to `multiordermap_sum_in_smoc_from_file`, nothing is known + /// about the values that are summed, so we do not multiply them be the area + /// of the cells + #[pyfn(m)] + fn multiordermap_sum_in_smoc( + index: usize, + uniq: PyReadonlyArrayDyn, + uniq_mask: PyReadonlyArrayDyn, + value: PyReadonlyArrayDyn, + value_mask: PyReadonlyArrayDyn, + ) -> PyResult { + let it = uniq + .as_array() + .into_iter() + .cloned() + .zip(value.as_array().into_iter().cloned()) + .zip( + uniq_mask + .as_array() + .into_iter() + .cloned() + .zip(value_mask.as_array().into_iter().cloned()), + ) + .filter_map(|(key_val, (mask_key, mask_val))| { + if mask_key | mask_val { + None + } else { + Some(key_val) + } + }); + U64MocStore::get_global_store() + .multiordermap_sum_in_moc(index, it) + .map_err(PyIOError::new_err) + } + + /// Get the sum of the probability for the UNIQ HEAPix cells which are in + /// the S-MOC of given index. We assume that input values are probability densities + /// that we have to multiply by the cell area to get the probability value. + /// + /// # Arguments + /// + /// * ``index`` - The index of the coverage in the store. + /// * ``uniq`` - UNIQ HEALPix values. + /// * ``uniq_mask`` - Mask on the UNIQ HEALPix values. + /// * ``probdens`` - Probability densities to be converted into probabilities before being summed. + /// * ``probdens_mask`` - mask on the probability densities. + #[pyfn(m)] + fn multiorder_probdens_map_sum_in_smoc( + index: usize, + uniq: PyReadonlyArrayDyn, + uniq_mask: PyReadonlyArrayDyn, + probdens: PyReadonlyArrayDyn, + probdens_mask: PyReadonlyArrayDyn, + ) -> PyResult { + let it = uniq + .as_array() + .into_iter() + .cloned() + .zip(probdens.as_array().into_iter().cloned()) + .zip( + uniq_mask + .as_array() + .into_iter() + .cloned() + .zip(probdens_mask.as_array().into_iter().cloned()), + ) + .filter_map(|((uniq, dens), (mask_key, mask_val))| { + if mask_key | mask_val { + None + } else { + let (depth, _ipix) = Hpx::::from_uniq_hpx(uniq); + let area = (std::f64::consts::PI / 3.0) / (1_u64 << (depth << 1) as u32) as f64; + let proba = dens * area; + Some((uniq, proba)) + } + }); + U64MocStore::get_global_store() + .multiordermap_sum_in_moc(index, it) + .map_err(PyIOError::new_err) + } + /// Deserialize a spatial MOC from a FITS file. /// /// # Arguments @@ -2026,6 +2191,13 @@ fn mocpy(_py: Python, m: &PyModule) -> PyResult<()> { .map_err(PyIOError::new_err) } + #[pyfn(m)] + fn new_empty_stmoc(depth_time: u8, depth_space: u8) -> PyResult { + U64MocStore::get_global_store() + .new_empty_stmoc(depth_time, depth_space) + .map_err(PyIOError::new_err) + } + /// Gives the number of cells for a specific depth in a spatial MOC /// /// # Arguments