From a80ecc300831c97e4e5687c1089a9cd466ecb46f Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Thu, 14 Nov 2019 09:50:44 -0500 Subject: [PATCH 1/4] Pickle dark_date always and use it if existing for selecting dark map --- proseco/catalog.py | 1 - proseco/core.py | 20 +++++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/proseco/catalog.py b/proseco/catalog.py index f46a51e5..d77ef696 100644 --- a/proseco/catalog.py +++ b/proseco/catalog.py @@ -180,7 +180,6 @@ class ACATable(ACACatalogTable): optimize = MetaAttribute(default=True) call_args = MetaAttribute(default={}) version = MetaAttribute() - dark_date = MetaAttribute() # For validation with get_aca_catalog(obsid), store the starcheck # catalog in the ACATable meta. diff --git a/proseco/core.py b/proseco/core.py index ba474d42..a39641d7 100644 --- a/proseco/core.py +++ b/proseco/core.py @@ -540,6 +540,7 @@ class ACACatalogTable(BaseCatalogTable): t_ccd_acq = MetaAttribute() t_ccd_guide = MetaAttribute() date = MetaAttribute() + dark_date = MetaAttribute() dither_acq = MetaAttribute() dither_guide = MetaAttribute() detector = MetaAttribute() @@ -561,15 +562,28 @@ def dark(self): if hasattr(self, '_dark'): return self._dark + # If self.dark_date is already set (most likely after unpickling an existing + # catalog), then use that instead of the observation date for getting the dark map. + if self.dark_date: + dark_date = self.dark_date + select = 'nearest' + else: + dark_date = self.date + select = 'before' + # Dark current map handling. If asking for `dark` attribute without having set # it then auto-fetch from mica. Note that get_dark_cal_image caches returned values # using LRU cache on all params, so this is efficient for the different # catalog tables. - self.log(f'Getting dark cal image at date={self.date} t_ccd={self.t_ccd:.1f}') - self.dark = get_dark_cal_image(date=self.date, select='before', + self.log(f'Getting dark cal image at date={dark_date} t_ccd={self.t_ccd:.1f} ' + f'select={select}') + self.dark = get_dark_cal_image(date=dark_date, select=select, t_ccd_ref=self.t_ccd, aca_image=False) - dark_id = get_dark_cal_id(date=self.date, select='before') + + # In all cases set self.dark_date to be the actual date of the dark cal + # we are using, not what might have been set on input. + dark_id = get_dark_cal_id(date=dark_date, select=select) self.dark_date = dark_id[:4] + ':' + dark_id[4:] return self._dark From 500db87cfbbca9b5129f2c0602a08c0f98a14bfd Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Thu, 14 Nov 2019 09:59:10 -0500 Subject: [PATCH 2/4] Add warning for dark cal vs. dark_date mismatch --- proseco/core.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/proseco/core.py b/proseco/core.py index a39641d7..8d0ebe9f 100644 --- a/proseco/core.py +++ b/proseco/core.py @@ -4,6 +4,7 @@ import pickle import inspect import time +import warnings from copy import copy from pathlib import Path @@ -582,9 +583,16 @@ def dark(self): aca_image=False) # In all cases set self.dark_date to be the actual date of the dark cal - # we are using, not what might have been set on input. + # we are using, not what might have been set on input. But issue a warning + # if there is a mismatch. dark_id = get_dark_cal_id(date=dark_date, select=select) - self.dark_date = dark_id[:4] + ':' + dark_id[4:] + new_dark_date = dark_id[:4] + ':' + dark_id[4:] + + if self.dark_date and self.dark_date != new_dark_date: + warnings.warn(f'Mismatch between pickled dark_date {self.dark_date} and nearest ' + f'available dark cal in the archive at {new_dark_date}') + + self.dark_date = new_dark_date return self._dark From 9655c30fd38fbaac06dce6f333866ccafc508a6e Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Mon, 18 Nov 2019 14:51:16 -0500 Subject: [PATCH 3/4] Plan B for implementation and warning --- proseco/core.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/proseco/core.py b/proseco/core.py index 8d0ebe9f..166431af 100644 --- a/proseco/core.py +++ b/proseco/core.py @@ -568,31 +568,33 @@ def dark(self): if self.dark_date: dark_date = self.dark_date select = 'nearest' + dark_id = get_dark_cal_id(date=dark_date, select=select) + + # Warn if the pickled dark_date is not the same as we would have gotten based + # on the observation date. + dark_id_for_date = get_dark_cal_id(date=self.date, select='before') + if dark_id != dark_id_for_date: + warnings.warn(f'Unexpected dark_date: ' + f'dark_id nearest dark_date={dark_date} is {dark_id} ' + f'but dark_id before obs date={self.date} is {dark_id_for_date}') else: dark_date = self.date select = 'before' + dark_id = get_dark_cal_id(date=dark_date, select=select) # Dark current map handling. If asking for `dark` attribute without having set # it then auto-fetch from mica. Note that get_dark_cal_image caches returned values # using LRU cache on all params, so this is efficient for the different # catalog tables. - self.log(f'Getting dark cal image at date={dark_date} t_ccd={self.t_ccd:.1f} ' + self.log(f'Getting dark cal image {dark_id} at date={dark_date} t_ccd={self.t_ccd:.1f} ' f'select={select}') self.dark = get_dark_cal_image(date=dark_date, select=select, t_ccd_ref=self.t_ccd, aca_image=False) # In all cases set self.dark_date to be the actual date of the dark cal - # we are using, not what might have been set on input. But issue a warning - # if there is a mismatch. - dark_id = get_dark_cal_id(date=dark_date, select=select) - new_dark_date = dark_id[:4] + ':' + dark_id[4:] - - if self.dark_date and self.dark_date != new_dark_date: - warnings.warn(f'Mismatch between pickled dark_date {self.dark_date} and nearest ' - f'available dark cal in the archive at {new_dark_date}') - - self.dark_date = new_dark_date + # we are using, not what might have been set on input. + self.dark_date = dark_id[:4] + ':' + dark_id[4:] return self._dark From 168cc36ad6f79c7d48d726d0b694427e50516436 Mon Sep 17 00:00:00 2001 From: Tom Aldcroft Date: Mon, 18 Nov 2019 20:53:25 -0500 Subject: [PATCH 4/4] Plan C for implementation plus a test --- proseco/core.py | 25 +++++++++++-------------- proseco/tests/test_catalog.py | 14 ++++++++++++++ 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/proseco/core.py b/proseco/core.py index 166431af..1934197d 100644 --- a/proseco/core.py +++ b/proseco/core.py @@ -566,36 +566,33 @@ def dark(self): # If self.dark_date is already set (most likely after unpickling an existing # catalog), then use that instead of the observation date for getting the dark map. if self.dark_date: - dark_date = self.dark_date - select = 'nearest' - dark_id = get_dark_cal_id(date=dark_date, select=select) + dark_id = get_dark_cal_id(date=self.dark_date, select='nearest') + self.log(f'Found dark cal image {dark_id} nearest self.dark_date={self.dark_date}') # Warn if the pickled dark_date is not the same as we would have gotten based # on the observation date. dark_id_for_date = get_dark_cal_id(date=self.date, select='before') if dark_id != dark_id_for_date: warnings.warn(f'Unexpected dark_date: ' - f'dark_id nearest dark_date={dark_date} is {dark_id} ' + f'dark_id nearest dark_date={self.dark_date} is {dark_id} ' f'but dark_id before obs date={self.date} is {dark_id_for_date}') else: - dark_date = self.date - select = 'before' - dark_id = get_dark_cal_id(date=dark_date, select=select) + dark_id = get_dark_cal_id(date=self.date, select='before') + self.log(f'Found dark cal image {dark_id} before self.date={self.date}') + + # In all cases set self.dark_date to be the actual date of the dark cal + # we are using, not what might have been set on input. + self.dark_date = dark_id[:4] + ':' + dark_id[4:] # Dark current map handling. If asking for `dark` attribute without having set # it then auto-fetch from mica. Note that get_dark_cal_image caches returned values # using LRU cache on all params, so this is efficient for the different # catalog tables. - self.log(f'Getting dark cal image {dark_id} at date={dark_date} t_ccd={self.t_ccd:.1f} ' - f'select={select}') - self.dark = get_dark_cal_image(date=dark_date, select=select, + self.log(f'Getting dark cal image at date={self.dark_date} t_ccd={self.t_ccd:.1f} ') + self.dark = get_dark_cal_image(date=self.dark_date, select='nearest', t_ccd_ref=self.t_ccd, aca_image=False) - # In all cases set self.dark_date to be the actual date of the dark cal - # we are using, not what might have been set on input. - self.dark_date = dark_id[:4] + ':' + dark_id[4:] - return self._dark @dark.setter diff --git a/proseco/tests/test_catalog.py b/proseco/tests/test_catalog.py index c3ebce42..f688326d 100644 --- a/proseco/tests/test_catalog.py +++ b/proseco/tests/test_catalog.py @@ -760,3 +760,17 @@ def test_includes_for_obsid(): out = includes_for_obsid(8008) assert out == exp + + +def test_dark_date_warning(): + aca = get_aca_catalog(**STD_INFO) + acap = pickle.loads(pickle.dumps(aca)) + assert acap.dark_date == '2017:272' + + # Fudge date forward, after the 2018:002 dark cal + acap.date = '2018:010' + with pytest.warns(None) as warns: + acap.dark # Accessing the `dark` property triggers code to read it (and warn) + + assert len(warns) == 1 + assert 'Unexpected dark_date: dark_id nearest dark_date' in str(warns[0].message)