From f7614136bd45762f140800d8ca38082a9aac12ab Mon Sep 17 00:00:00 2001 From: Erin Sheldon Date: Fri, 16 Aug 2024 13:21:50 -0400 Subject: [PATCH 01/12] add ability to send dither seed Added test for write() read() --- fitsio/fitsio_pywrap.c | 11 ++++- fitsio/fitslib.py | 31 ++++++++++++- fitsio/tests/test_image_compression.py | 60 ++++++++++++++++++++++++++ 3 files changed, 100 insertions(+), 2 deletions(-) diff --git a/fitsio/fitsio_pywrap.c b/fitsio/fitsio_pywrap.c index 06eab60e..66548b91 100644 --- a/fitsio/fitsio_pywrap.c +++ b/fitsio/fitsio_pywrap.c @@ -1395,6 +1395,7 @@ PyFITSObject_create_image_hdu(struct PyFITSObject* self, PyObject* args, PyObjec int extver=0; float qlevel=0; int qmethod=0; + int dither_seed=0; float hcomp_scale=0; int hcomp_smooth=0; @@ -1411,6 +1412,7 @@ PyFITSObject_create_image_hdu(struct PyFITSObject* self, PyObject* args, PyObjec "qlevel", "qmethod", + "dither_seed", "hcomp_scale", "hcomp_smooth", @@ -1419,7 +1421,7 @@ PyFITSObject_create_image_hdu(struct PyFITSObject* self, PyObject* args, PyObjec "extver", NULL, }; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "Oi|OiOfifisi", kwlist, + if (!PyArg_ParseTupleAndKeywords(args, kwds, "Oi|OiOfiifisi", kwlist, &array_obj, &nkeys, &dims_obj, &comptype, @@ -1427,6 +1429,7 @@ PyFITSObject_create_image_hdu(struct PyFITSObject* self, PyObject* args, PyObjec &qlevel, &qmethod, + &dither_seed, &hcomp_scale, &hcomp_smooth, @@ -1495,6 +1498,12 @@ PyFITSObject_create_image_hdu(struct PyFITSObject* self, PyObject* args, PyObjec goto create_image_hdu_cleanup; } + if (dither_seed >= 0) { + if (fits_set_dither_seed(self->fits, dither_seed, &status)) { + goto create_image_hdu_cleanup; + } + } + if (comptype == HCOMPRESS_1) { if (fits_set_hcomp_scale(self->fits, hcomp_scale, &status)) { diff --git a/fitsio/fitslib.py b/fitsio/fitslib.py index 3d435491..c1f69697 100644 --- a/fitsio/fitslib.py +++ b/fitsio/fitslib.py @@ -288,6 +288,7 @@ def write(filename, data, extname=None, extver=None, header=None, names=None, write_bitcols=False, compress=None, tile_dims=None, qlevel=DEFAULT_QLEVEL, qmethod=DEFAULT_QMETHOD, + dither_seed=None, hcomp_scale=DEFAULT_HCOMP_SCALE, hcomp_smooth=False, **keys): @@ -376,6 +377,9 @@ def write(filename, data, extname=None, extver=None, header=None, Preserves zeros Defaults to 'SUBTRACTIVE_DITHER_1' which follows the fpack defaults + dither_seed: int + Seed for the subtractive dither. Seeding makes the lossy + compression reproducible. hcomp_scale: float Scale value for HCOMPRESS, 0.0 means lossless compression. Default is @@ -409,6 +413,7 @@ def write(filename, data, extname=None, extver=None, header=None, tile_dims=tile_dims, qlevel=qlevel, qmethod=qmethod, + dither_seed=dither_seed, hcomp_scale=hcomp_scale, hcomp_smooth=hcomp_smooth, ) @@ -624,6 +629,7 @@ def write(self, data, units=None, extname=None, extver=None, tile_dims=None, qlevel=DEFAULT_QLEVEL, qmethod=DEFAULT_QMETHOD, + dither_seed=None, hcomp_scale=DEFAULT_HCOMP_SCALE, hcomp_smooth=False, header=None, names=None, @@ -686,6 +692,9 @@ def write(self, data, units=None, extname=None, extver=None, Preserves zeros Defaults to 'SUBTRACTIVE_DITHER_1' which follows the fpack defaults + dither_seed: int + Seed for the subtractive dither. Seeding makes the lossy + compression reproducible. hcomp_scale: float Scale value for HCOMPRESS, 0.0 means lossless compression. Default @@ -731,6 +740,7 @@ def write(self, data, units=None, extname=None, extver=None, tile_dims=tile_dims, qlevel=qlevel, qmethod=qmethod, + dither_seed=dither_seed, hcomp_scale=hcomp_scale, hcomp_smooth=hcomp_smooth, header=header) @@ -745,6 +755,7 @@ def write_image(self, img, extname=None, extver=None, compress=None, tile_dims=None, qlevel=DEFAULT_QLEVEL, qmethod=DEFAULT_QMETHOD, + dither_seed=None, hcomp_scale=DEFAULT_HCOMP_SCALE, hcomp_smooth=False, header=None): @@ -792,6 +803,9 @@ def write_image(self, img, extname=None, extver=None, Preserves zeros Defaults to 'SUBTRACTIVE_DITHER_1' which follows the fpack defaults + dither_seed: int + Seed for the subtractive dither. Seeding makes the lossy + compression reproducible. hcomp_scale: float Scale value for HCOMPRESS, 0.0 means lossless compression. Default @@ -823,6 +837,7 @@ def write_image(self, img, extname=None, extver=None, tile_dims=tile_dims, qlevel=qlevel, qmethod=qmethod, + dither_seed=dither_seed, hcomp_scale=hcomp_scale, hcomp_smooth=hcomp_smooth, ) @@ -844,6 +859,7 @@ def create_image_hdu(self, tile_dims=None, qlevel=DEFAULT_QLEVEL, qmethod=DEFAULT_QMETHOD, + dither_seed=None, hcomp_scale=DEFAULT_HCOMP_SCALE, hcomp_smooth=False, header=None): @@ -917,7 +933,9 @@ def create_image_hdu(self, Preserves zeros Defaults to 'SUBTRACTIVE_DITHER_1' which follows the fpack defaults - + dither_seed: int + Seed for the subtractive dither. Seeding makes the lossy + compression reproducible. hcomp_scale: float Scale value for HCOMPRESS, 0.0 means lossless compression. Default is 0.0 following the fpack defaults. @@ -1002,6 +1020,7 @@ def create_image_hdu(self, comptype = get_compress_type(compress) qmethod = get_qmethod(qmethod) + dither_seed = get_dither_seed(dither_seed) tile_dims = get_tile_dims(tile_dims, dims) if qlevel is None: @@ -1032,6 +1051,7 @@ def create_image_hdu(self, qlevel=qlevel, qmethod=qmethod, + dither_seed=dither_seed, hcomp_scale=hcomp_scale, hcomp_smooth=hcomp_smooth, @@ -1800,6 +1820,15 @@ def get_qmethod(qmethod): return _qmethod_map[qmethod] +def get_dither_seed(dither_seed): + if dither_seed is None: + # -1 means do not set the seed + return -1 + else: + # must fit in an int + return numpy.int32(dither_seed) + + def check_comptype_img(comptype, dtype_str): if comptype == NOCOMPRESS: diff --git a/fitsio/tests/test_image_compression.py b/fitsio/tests/test_image_compression.py index 1f7a1bb2..87debf42 100644 --- a/fitsio/tests/test_image_compression.py +++ b/fitsio/tests/test_image_compression.py @@ -219,3 +219,63 @@ def test_compress_preserve_zeros(): for zind in zinds: assert rdata[zind[0], zind[1]] == 0.0 + + +@pytest.mark.parametrize( + 'compress', + [ + 'rice', + 'hcompress', + 'plio', + ] +) +@pytest.mark.parametrize( + 'use_seed', + [False, True], +) +def test_compressed_seed(compress, use_seed): + """ + Test writing and reading a rice compressed image + """ + nrows = 5 + ncols = 20 + dtypes = ['f4', 'f8'] + + qlevel = 16 + + seed = 1919 + rng = np.random.RandomState(seed) + + if use_seed: + dither_seed = 9881 + else: + dither_seed = None + + with tempfile.TemporaryDirectory() as tmpdir: + fname1 = os.path.join(tmpdir, 'test1.fits') + fname2 = os.path.join(tmpdir, 'test2.fits') + + for ext, dtype in enumerate(dtypes): + data = rng.normal(size=(nrows, ncols)) + if compress == 'plio': + data = data.clip(min=0) + data = data.astype(dtype) + + write( + fname1, data, compress=compress, qlevel=qlevel, + dither_seed=dither_seed, + ) + rdata1 = read(fname1, ext=ext+1) + + write( + fname2, data, compress=compress, qlevel=qlevel, + dither_seed=dither_seed, + ) + rdata2 = read(fname2, ext=ext+1) + + mess = "%s compressed images ('%s')" % (compress, dtype) + + if use_seed: + assert np.all(rdata1 == rdata2), mess + else: + assert np.all(rdata1 != rdata2), mess From 25a11c77a3ea1f483eb4abd2526868408d57c2b1 Mon Sep 17 00:00:00 2001 From: Erin Sheldon Date: Fri, 16 Aug 2024 13:24:22 -0400 Subject: [PATCH 02/12] add dither_seed test using FITS object --- fitsio/tests/test_image_compression.py | 43 +++++++++++++++++++------- 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/fitsio/tests/test_image_compression.py b/fitsio/tests/test_image_compression.py index 87debf42..6367c2a6 100644 --- a/fitsio/tests/test_image_compression.py +++ b/fitsio/tests/test_image_compression.py @@ -233,7 +233,11 @@ def test_compress_preserve_zeros(): 'use_seed', [False, True], ) -def test_compressed_seed(compress, use_seed): +@pytest.mark.parametrize( + 'use_fits_object', + [False, True], +) +def test_compressed_seed(compress, use_seed, use_fits_object): """ Test writing and reading a rice compressed image """ @@ -261,17 +265,32 @@ def test_compressed_seed(compress, use_seed): data = data.clip(min=0) data = data.astype(dtype) - write( - fname1, data, compress=compress, qlevel=qlevel, - dither_seed=dither_seed, - ) - rdata1 = read(fname1, ext=ext+1) - - write( - fname2, data, compress=compress, qlevel=qlevel, - dither_seed=dither_seed, - ) - rdata2 = read(fname2, ext=ext+1) + if use_fits_object: + with FITS(fname1, 'rw') as fits1: + fits1.write( + data, compress=compress, qlevel=qlevel, + dither_seed=dither_seed, + ) + rdata1 = fits1[-1].read() + + with FITS(fname2, 'rw') as fits2: + fits2.write( + data, compress=compress, qlevel=qlevel, + dither_seed=dither_seed, + ) + rdata2 = fits2[-1].read() + else: + write( + fname1, data, compress=compress, qlevel=qlevel, + dither_seed=dither_seed, + ) + rdata1 = read(fname1, ext=ext+1) + + write( + fname2, data, compress=compress, qlevel=qlevel, + dither_seed=dither_seed, + ) + rdata2 = read(fname2, ext=ext+1) mess = "%s compressed images ('%s')" % (compress, dtype) From ce5f4d50014ebcbc97329c3233c8741a7c6b1084 Mon Sep 17 00:00:00 2001 From: Erin Sheldon Date: Fri, 16 Aug 2024 13:34:58 -0400 Subject: [PATCH 03/12] temporarily turn off test of external cfitsio --- .github/workflows/tests.yml | 40 ++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9b0fe651..10f2a825 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -77,26 +77,26 @@ jobs: cd .. cd .. - - name: test non-bundled build - shell: bash -l {0} - run: | - pip install -vv -e . \ - --global-option="build_ext" \ - --global-option="--use-system-fitsio" \ - --global-option="--system-fitsio-includedir=$HOME/cfitsio-static-install/include" \ - --global-option="--system-fitsio-libdir=$HOME/cfitsio-static-install/lib" - SKIP_BZIP_TEST=true pytest -vv fitsio - - - name: test non-bundled build w/ env vars - shell: bash -l {0} - run: | - rm -rf build* - find . -name "*.so" -type f -delete - export FITSIO_USE_SYSTEM_FITSIO=1 - export FITSIO_SYSTEM_FITSIO_INCLUDEDIR=$HOME/cfitsio-static-install/include - export FITSIO_SYSTEM_FITSIO_LIBDIR=$HOME/cfitsio-static-install/lib - pip install -vv -e . - SKIP_BZIP_TEST=true pytest -vv fitsio + # - name: test non-bundled build + # shell: bash -l {0} + # run: | + # pip install -vv -e . \ + # --global-option="build_ext" \ + # --global-option="--use-system-fitsio" \ + # --global-option="--system-fitsio-includedir=$HOME/cfitsio-static-install/include" \ + # --global-option="--system-fitsio-libdir=$HOME/cfitsio-static-install/lib" + # SKIP_BZIP_TEST=true pytest -vv fitsio + # + # - name: test non-bundled build w/ env vars + # shell: bash -l {0} + # run: | + # rm -rf build* + # find . -name "*.so" -type f -delete + # export FITSIO_USE_SYSTEM_FITSIO=1 + # export FITSIO_SYSTEM_FITSIO_INCLUDEDIR=$HOME/cfitsio-static-install/include + # export FITSIO_SYSTEM_FITSIO_LIBDIR=$HOME/cfitsio-static-install/lib + # pip install -vv -e . + # SKIP_BZIP_TEST=true pytest -vv fitsio - name: test bundled build shell: bash -l {0} From 3b04c9c988eb1a02dbe74c713f1aef3e7d79af60 Mon Sep 17 00:00:00 2001 From: Erin Sheldon Date: Fri, 16 Aug 2024 13:55:36 -0400 Subject: [PATCH 04/12] fix tests for new numpy --- fitsio/tests/test_image_compression.py | 103 +++++++++++++++---------- 1 file changed, 61 insertions(+), 42 deletions(-) diff --git a/fitsio/tests/test_image_compression.py b/fitsio/tests/test_image_compression.py index 6367c2a6..23c8eac4 100644 --- a/fitsio/tests/test_image_compression.py +++ b/fitsio/tests/test_image_compression.py @@ -230,71 +230,90 @@ def test_compress_preserve_zeros(): ] ) @pytest.mark.parametrize( - 'use_seed', + 'match_seed', [False, True], ) @pytest.mark.parametrize( 'use_fits_object', [False, True], ) -def test_compressed_seed(compress, use_seed, use_fits_object): +@pytest.mark.parametrize( + 'dtype', + ['f4', 'f8'], +) +def test_compressed_seed(compress, match_seed, use_fits_object, dtype): """ Test writing and reading a rice compressed image """ nrows = 5 ncols = 20 - dtypes = ['f4', 'f8'] qlevel = 16 seed = 1919 rng = np.random.RandomState(seed) - if use_seed: - dither_seed = 9881 + if match_seed: + # dither_seed = 9881 + dither_seed1 = 9881 + dither_seed2 = 9881 else: - dither_seed = None + # dither_seed = None + dither_seed1 = 3 + dither_seed2 = 4 with tempfile.TemporaryDirectory() as tmpdir: fname1 = os.path.join(tmpdir, 'test1.fits') fname2 = os.path.join(tmpdir, 'test2.fits') - for ext, dtype in enumerate(dtypes): - data = rng.normal(size=(nrows, ncols)) - if compress == 'plio': - data = data.clip(min=0) - data = data.astype(dtype) - - if use_fits_object: - with FITS(fname1, 'rw') as fits1: - fits1.write( - data, compress=compress, qlevel=qlevel, - dither_seed=dither_seed, - ) - rdata1 = fits1[-1].read() - - with FITS(fname2, 'rw') as fits2: - fits2.write( - data, compress=compress, qlevel=qlevel, - dither_seed=dither_seed, - ) - rdata2 = fits2[-1].read() - else: - write( - fname1, data, compress=compress, qlevel=qlevel, - dither_seed=dither_seed, + data = rng.normal(size=(nrows, ncols)) + if compress == 'plio': + data = data.clip(min=0) + data = data.astype(dtype) + + if use_fits_object: + with FITS(fname1, 'rw') as fits1: + fits1.write( + data, compress=compress, qlevel=qlevel, + # dither_seed=dither_seed, + dither_seed=dither_seed1, ) - rdata1 = read(fname1, ext=ext+1) + rdata1 = fits1[-1].read() - write( - fname2, data, compress=compress, qlevel=qlevel, - dither_seed=dither_seed, + with FITS(fname2, 'rw') as fits2: + fits2.write( + data, compress=compress, qlevel=qlevel, + # dither_seed=dither_seed, + dither_seed=dither_seed2, ) - rdata2 = read(fname2, ext=ext+1) - - mess = "%s compressed images ('%s')" % (compress, dtype) - - if use_seed: - assert np.all(rdata1 == rdata2), mess - else: - assert np.all(rdata1 != rdata2), mess + rdata2 = fits2[-1].read() + else: + write( + fname1, data, compress=compress, qlevel=qlevel, + # dither_seed=dither_seed, + dither_seed=dither_seed1, + ) + rdata1 = read(fname1) + + write( + fname2, data, compress=compress, qlevel=qlevel, + # dither_seed=dither_seed, + dither_seed=dither_seed2, + ) + rdata2 = read(fname2) + + mess = "%s compressed images ('%s')" % (compress, dtype) + + if match_seed: + assert np.all(rdata1 == rdata2), mess + else: + assert np.all(rdata1 != rdata2), mess + + +if __name__ == '__main__': + test_compressed_seed( + compress='rice', + match_seed=False, + use_fits_object=True, + dtype='f4', + ) From b87c8b31c865e7d126b278879b2727157daa8401 Mon Sep 17 00:00:00 2001 From: Erin Sheldon Date: Fri, 16 Aug 2024 13:58:52 -0400 Subject: [PATCH 05/12] put back non-bundled tests --- .github/workflows/tests.yml | 40 ++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 10f2a825..9b0fe651 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -77,26 +77,26 @@ jobs: cd .. cd .. - # - name: test non-bundled build - # shell: bash -l {0} - # run: | - # pip install -vv -e . \ - # --global-option="build_ext" \ - # --global-option="--use-system-fitsio" \ - # --global-option="--system-fitsio-includedir=$HOME/cfitsio-static-install/include" \ - # --global-option="--system-fitsio-libdir=$HOME/cfitsio-static-install/lib" - # SKIP_BZIP_TEST=true pytest -vv fitsio - # - # - name: test non-bundled build w/ env vars - # shell: bash -l {0} - # run: | - # rm -rf build* - # find . -name "*.so" -type f -delete - # export FITSIO_USE_SYSTEM_FITSIO=1 - # export FITSIO_SYSTEM_FITSIO_INCLUDEDIR=$HOME/cfitsio-static-install/include - # export FITSIO_SYSTEM_FITSIO_LIBDIR=$HOME/cfitsio-static-install/lib - # pip install -vv -e . - # SKIP_BZIP_TEST=true pytest -vv fitsio + - name: test non-bundled build + shell: bash -l {0} + run: | + pip install -vv -e . \ + --global-option="build_ext" \ + --global-option="--use-system-fitsio" \ + --global-option="--system-fitsio-includedir=$HOME/cfitsio-static-install/include" \ + --global-option="--system-fitsio-libdir=$HOME/cfitsio-static-install/lib" + SKIP_BZIP_TEST=true pytest -vv fitsio + + - name: test non-bundled build w/ env vars + shell: bash -l {0} + run: | + rm -rf build* + find . -name "*.so" -type f -delete + export FITSIO_USE_SYSTEM_FITSIO=1 + export FITSIO_SYSTEM_FITSIO_INCLUDEDIR=$HOME/cfitsio-static-install/include + export FITSIO_SYSTEM_FITSIO_LIBDIR=$HOME/cfitsio-static-install/lib + pip install -vv -e . + SKIP_BZIP_TEST=true pytest -vv fitsio - name: test bundled build shell: bash -l {0} From eba647fa023ad1d4e8605c6f0cdacfdb406e5263 Mon Sep 17 00:00:00 2001 From: Erin Sheldon Date: Fri, 16 Aug 2024 15:10:38 -0400 Subject: [PATCH 06/12] update CHANGES.md --- CHANGES.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 73c68e72..dc52e5fd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,11 @@ version 1.2.5 (not yet released) ------------- +New Features + + - writing images supports the dither_seed keyword, to seed + the random number generator for subtractive dithering + Bug Fixes - Fig bug slicing tables that have TBIT columns From 7dcda44684153dacf1dfe5eda6d17b3f8a746133 Mon Sep 17 00:00:00 2001 From: Erin Sheldon Date: Tue, 20 Aug 2024 11:39:35 -0400 Subject: [PATCH 07/12] allow different dither seeds Also restrict range of seeds --- fitsio/fitsio_pywrap.c | 3 +- fitsio/fitslib.py | 62 ++++++++++++++++++++------ fitsio/tests/test_image_compression.py | 49 +++++++++++++++++--- 3 files changed, 94 insertions(+), 20 deletions(-) diff --git a/fitsio/fitsio_pywrap.c b/fitsio/fitsio_pywrap.c index 66548b91..61934846 100644 --- a/fitsio/fitsio_pywrap.c +++ b/fitsio/fitsio_pywrap.c @@ -1498,7 +1498,8 @@ PyFITSObject_create_image_hdu(struct PyFITSObject* self, PyObject* args, PyObjec goto create_image_hdu_cleanup; } - if (dither_seed >= 0) { + // zero means to use the default (system clock). + if (dither_seed != 0) { if (fits_set_dither_seed(self->fits, dither_seed, &status)) { goto create_image_hdu_cleanup; } diff --git a/fitsio/fitslib.py b/fitsio/fitslib.py index c1f69697..e6737d83 100644 --- a/fitsio/fitslib.py +++ b/fitsio/fitslib.py @@ -377,9 +377,16 @@ def write(filename, data, extname=None, extver=None, header=None, Preserves zeros Defaults to 'SUBTRACTIVE_DITHER_1' which follows the fpack defaults - dither_seed: int - Seed for the subtractive dither. Seeding makes the lossy - compression reproducible. + + dither_seed: int or None + Seed for the subtractive dither. Seeding makes the lossy compression + reproducible. Allowed values are + None or 0 or 'clock': + do not set the seed explicitly, use the system clock + -1 or 'checksum': + Set the seed based on the data checksum + 1-10_000: + use the input seed hcomp_scale: float Scale value for HCOMPRESS, 0.0 means lossless compression. Default is @@ -692,9 +699,15 @@ def write(self, data, units=None, extname=None, extver=None, Preserves zeros Defaults to 'SUBTRACTIVE_DITHER_1' which follows the fpack defaults - dither_seed: int + dither_seed: int or None Seed for the subtractive dither. Seeding makes the lossy - compression reproducible. + compression reproducible. Allowed values are + None or 0 or 'clock': + do not set the seed explicitly, use the system clock + -1 or 'checksum': + Set the seed based on the data checksum + 1-10_000: + use the input seed hcomp_scale: float Scale value for HCOMPRESS, 0.0 means lossless compression. Default @@ -803,9 +816,15 @@ def write_image(self, img, extname=None, extver=None, Preserves zeros Defaults to 'SUBTRACTIVE_DITHER_1' which follows the fpack defaults - dither_seed: int + dither_seed: int or None Seed for the subtractive dither. Seeding makes the lossy - compression reproducible. + compression reproducible. Allowed values are + None or 0 or 'clock': + do not set the seed explicitly, use the system clock + -1 or 'checksum': + Set the seed based on the data checksum + 1-10_000: + use the input seed hcomp_scale: float Scale value for HCOMPRESS, 0.0 means lossless compression. Default @@ -933,9 +952,15 @@ def create_image_hdu(self, Preserves zeros Defaults to 'SUBTRACTIVE_DITHER_1' which follows the fpack defaults - dither_seed: int + dither_seed: int or None Seed for the subtractive dither. Seeding makes the lossy - compression reproducible. + compression reproducible. Allowed values are + None or 0 or 'clock': + do not set the seed explicitly, use the system clock + -1 or 'checksum': + Set the seed based on the data checksum + 1-10_000: + use the input seed hcomp_scale: float Scale value for HCOMPRESS, 0.0 means lossless compression. Default is 0.0 following the fpack defaults. @@ -1821,12 +1846,23 @@ def get_qmethod(qmethod): def get_dither_seed(dither_seed): - if dither_seed is None: - # -1 means do not set the seed - return -1 + if dither_seed is None or dither_seed in ['clock', 'CLOCK']: + # 0 means do not set the seed, use the system clock. This is the + # default + seed_out = 0 + elif dither_seed in ['checksum', 'CHECKSUM']: + seed_out = -1 else: # must fit in an int - return numpy.int32(dither_seed) + seed_out = numpy.int32(dither_seed) + + if seed_out < -1 or seed_out > 10_000: + raise ValueError( + f'Got dither_seed {seed_out}, expected -1, 0 or ' + 'a seed in range [1, 10000]' + ) + + return seed_out def check_comptype_img(comptype, dtype_str): diff --git a/fitsio/tests/test_image_compression.py b/fitsio/tests/test_image_compression.py index 23c8eac4..73180ea7 100644 --- a/fitsio/tests/test_image_compression.py +++ b/fitsio/tests/test_image_compression.py @@ -230,8 +230,8 @@ def test_compress_preserve_zeros(): ] ) @pytest.mark.parametrize( - 'match_seed', - [False, True], + 'seed_type', + ['matched', 'unmatched', 'checksum', 'checksum_int'], ) @pytest.mark.parametrize( 'use_fits_object', @@ -241,7 +241,7 @@ def test_compress_preserve_zeros(): 'dtype', ['f4', 'f8'], ) -def test_compressed_seed(compress, match_seed, use_fits_object, dtype): +def test_compressed_seed(compress, seed_type, use_fits_object, dtype): """ Test writing and reading a rice compressed image """ @@ -253,14 +253,20 @@ def test_compressed_seed(compress, match_seed, use_fits_object, dtype): seed = 1919 rng = np.random.RandomState(seed) - if match_seed: + if seed_type == 'matched': # dither_seed = 9881 dither_seed1 = 9881 dither_seed2 = 9881 - else: + elif seed_type == 'unmatched': # dither_seed = None dither_seed1 = 3 dither_seed2 = 4 + elif seed_type == 'checksum': + dither_seed1 = 'checksum' + dither_seed2 = 'checksum' + elif seed_type == 'checksum_int': + dither_seed1 = -1 + dither_seed2 = -1 with tempfile.TemporaryDirectory() as tmpdir: fname1 = os.path.join(tmpdir, 'test1.fits') @@ -304,12 +310,43 @@ def test_compressed_seed(compress, match_seed, use_fits_object, dtype): mess = "%s compressed images ('%s')" % (compress, dtype) - if match_seed: + if seed_type in ['checksum', 'checksum_int', 'matched']: assert np.all(rdata1 == rdata2), mess else: assert np.all(rdata1 != rdata2), mess +@pytest.mark.parametrize( + 'dither_seed', + [-3, 10_001], +) +def test_compressed_seed_bad(dither_seed): + """ + Test writing and reading a rice compressed image + """ + compress = 'rice' + dtype = 'f4' + nrows = 5 + ncols = 20 + + qlevel = 16 + + seed = 1919 + rng = np.random.RandomState(seed) + + with tempfile.TemporaryDirectory() as tmpdir: + fname = os.path.join(tmpdir, 'test.fits') + + data = rng.normal(size=(nrows, ncols)) + data = data.astype(dtype) + + with pytest.raises(ValueError): + write( + fname, data, compress=compress, qlevel=qlevel, + dither_seed=dither_seed, + ) + + if __name__ == '__main__': test_compressed_seed( compress='rice', From 35ab99c777d2ba1d9cb47d02e6ef6a74d6b6b6ea Mon Sep 17 00:00:00 2001 From: Erin Sheldon Date: Tue, 20 Aug 2024 13:59:30 -0400 Subject: [PATCH 08/12] correct seed options case insensitive, allow all negative --- fitsio/fitslib.py | 41 +++++++++++++++++++------- fitsio/tests/test_image_compression.py | 2 +- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/fitsio/fitslib.py b/fitsio/fitslib.py index e6737d83..35c1a422 100644 --- a/fitsio/fitslib.py +++ b/fitsio/fitslib.py @@ -1846,21 +1846,40 @@ def get_qmethod(qmethod): def get_dither_seed(dither_seed): - if dither_seed is None or dither_seed in ['clock', 'CLOCK']: - # 0 means do not set the seed, use the system clock. This is the - # default + """ + Convert a seed value or indicator to the approprate integer value for + cfitsio + + Parameters + ---------- + dither_seed: number or string + Seed for the subtractive dither. Seeding makes the lossy compression + reproducible. Allowed values are + None or 0 or 'clock': + Return 0, do not set the seed explicitly, use the system clock + negative or 'checksum': + Return -1, means Set the seed based on the data checksum + 1-10_000: + use the input seed + """ + if isinstance(dither_seed, (str, bytes)): + dlow = dither_seed.lower() + if dlow == 'clock': + seed_out = 0 + elif dlow == 'checksum': + seed_out = -1 + else: + raise ValueError(f'Bad dither_seed {dither_seed}') + elif dither_seed is None: seed_out = 0 - elif dither_seed in ['checksum', 'CHECKSUM']: - seed_out = -1 else: # must fit in an int - seed_out = numpy.int32(dither_seed) + seed_out = numpy.int32(dither_seed).clip(min=-1) - if seed_out < -1 or seed_out > 10_000: - raise ValueError( - f'Got dither_seed {seed_out}, expected -1, 0 or ' - 'a seed in range [1, 10000]' - ) + if seed_out > 10_000: + raise ValueError( + f'Got dither_seed {seed_out}, expected avalue <= 10_000' + ) return seed_out diff --git a/fitsio/tests/test_image_compression.py b/fitsio/tests/test_image_compression.py index 73180ea7..26aef3d4 100644 --- a/fitsio/tests/test_image_compression.py +++ b/fitsio/tests/test_image_compression.py @@ -318,7 +318,7 @@ def test_compressed_seed(compress, seed_type, use_fits_object, dtype): @pytest.mark.parametrize( 'dither_seed', - [-3, 10_001], + ['blah', 10_001], ) def test_compressed_seed_bad(dither_seed): """ From c7f68e3ed2761ca897f8ae4f10aa27fcc0631e5f Mon Sep 17 00:00:00 2001 From: Erin Sheldon Date: Tue, 20 Aug 2024 14:04:10 -0400 Subject: [PATCH 09/12] docs --- fitsio/fitslib.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fitsio/fitslib.py b/fitsio/fitslib.py index 35c1a422..01ae48b7 100644 --- a/fitsio/fitslib.py +++ b/fitsio/fitslib.py @@ -383,7 +383,7 @@ def write(filename, data, extname=None, extver=None, header=None, reproducible. Allowed values are None or 0 or 'clock': do not set the seed explicitly, use the system clock - -1 or 'checksum': + negative or 'checksum': Set the seed based on the data checksum 1-10_000: use the input seed @@ -704,7 +704,7 @@ def write(self, data, units=None, extname=None, extver=None, compression reproducible. Allowed values are None or 0 or 'clock': do not set the seed explicitly, use the system clock - -1 or 'checksum': + negative or 'checksum': Set the seed based on the data checksum 1-10_000: use the input seed @@ -821,7 +821,7 @@ def write_image(self, img, extname=None, extver=None, compression reproducible. Allowed values are None or 0 or 'clock': do not set the seed explicitly, use the system clock - -1 or 'checksum': + negative or 'checksum': Set the seed based on the data checksum 1-10_000: use the input seed @@ -957,7 +957,7 @@ def create_image_hdu(self, compression reproducible. Allowed values are None or 0 or 'clock': do not set the seed explicitly, use the system clock - -1 or 'checksum': + negative or 'checksum': Set the seed based on the data checksum 1-10_000: use the input seed From 3e937f9df65c26da666a72e38b7568f1ef6a763e Mon Sep 17 00:00:00 2001 From: Erin Sheldon Date: Tue, 20 Aug 2024 14:08:46 -0400 Subject: [PATCH 10/12] remove clip --- fitsio/fitslib.py | 2 +- fitsio/tests/test_image_compression.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/fitsio/fitslib.py b/fitsio/fitslib.py index 01ae48b7..5e6dc88c 100644 --- a/fitsio/fitslib.py +++ b/fitsio/fitslib.py @@ -1874,7 +1874,7 @@ def get_dither_seed(dither_seed): seed_out = 0 else: # must fit in an int - seed_out = numpy.int32(dither_seed).clip(min=-1) + seed_out = numpy.int32(dither_seed) if seed_out > 10_000: raise ValueError( diff --git a/fitsio/tests/test_image_compression.py b/fitsio/tests/test_image_compression.py index 26aef3d4..b355c53e 100644 --- a/fitsio/tests/test_image_compression.py +++ b/fitsio/tests/test_image_compression.py @@ -266,7 +266,8 @@ def test_compressed_seed(compress, seed_type, use_fits_object, dtype): dither_seed2 = 'checksum' elif seed_type == 'checksum_int': dither_seed1 = -1 - dither_seed2 = -1 + # any negative means use checksum + dither_seed2 = -3 with tempfile.TemporaryDirectory() as tmpdir: fname1 = os.path.join(tmpdir, 'test1.fits') From 8e523815db6929513eccbe7fbde491198ad96d26 Mon Sep 17 00:00:00 2001 From: Erin Sheldon Date: Tue, 20 Aug 2024 14:10:57 -0400 Subject: [PATCH 11/12] force bytes to str --- fitsio/fitslib.py | 5 ++++- fitsio/tests/test_image_compression.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/fitsio/fitslib.py b/fitsio/fitslib.py index 5e6dc88c..2dc72f9f 100644 --- a/fitsio/fitslib.py +++ b/fitsio/fitslib.py @@ -1862,7 +1862,10 @@ def get_dither_seed(dither_seed): 1-10_000: use the input seed """ - if isinstance(dither_seed, (str, bytes)): + if isinstance(dither_seed, bytes): + dither_seed = str(dither_seed, 'utf-8') + + if isinstance(dither_seed, str): dlow = dither_seed.lower() if dlow == 'clock': seed_out = 0 diff --git a/fitsio/tests/test_image_compression.py b/fitsio/tests/test_image_compression.py index b355c53e..c12df4e0 100644 --- a/fitsio/tests/test_image_compression.py +++ b/fitsio/tests/test_image_compression.py @@ -263,7 +263,7 @@ def test_compressed_seed(compress, seed_type, use_fits_object, dtype): dither_seed2 = 4 elif seed_type == 'checksum': dither_seed1 = 'checksum' - dither_seed2 = 'checksum' + dither_seed2 = b'checksum' elif seed_type == 'checksum_int': dither_seed1 = -1 # any negative means use checksum From 76cf33e8ba436d56cbf83e0d24edf2f5a5820c1c Mon Sep 17 00:00:00 2001 From: Erin Sheldon Date: Tue, 20 Aug 2024 14:42:52 -0400 Subject: [PATCH 12/12] reduce test matrix --- .github/workflows/tests.yml | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9b0fe651..f1d18cb4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -16,25 +16,8 @@ jobs: fail-fast: false matrix: os: [macos-latest, ubuntu-latest] - pyver: ["3.8", "3.9", "3.10", "3.11", "3.12"] - npver: ["1.20", "1.21", "1.23", "1.26", "2.0"] - exclude: - - pyver: "3.11" - npver: "1.20" - - pyver: "3.11" - npver: "1.21" - - pyver: "3.10" - npver: "1.20" - - pyver: "3.12" - npver: "1.20" - - pyver: "3.12" - npver: "1.21" - - pyver: "3.12" - npver: "1.23" - - pyver: "3.8" - npver: "2.0" - - pyver: "3.8" - npver: "1.26" + pyver: ["3.12", "3.11"] + npver: ["1.26", "2.0"] runs-on: ${{ matrix.os }}