Merge pull request #75 from gbrammer/global-sky

Add global sky files
gbrammer · May 24, 2024 · 25edcb0 · 25edcb0
2 parents ae59f2b + 2194de7
commit 25edcb0
Show file tree

Hide file tree

Showing 93 changed files with 107,216 additions and 10 deletions.
diff --git a/msaexp/data/msa_sky/jw01180025001_sky.csv b/msaexp/data/msa_sky/jw01180025001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01180026001_sky.csv b/msaexp/data/msa_sky/jw01180026001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01180027001_sky.csv b/msaexp/data/msa_sky/jw01180027001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01180029001_sky.csv b/msaexp/data/msa_sky/jw01180029001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01180134001_sky.csv b/msaexp/data/msa_sky/jw01180134001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01180135001_sky.csv b/msaexp/data/msa_sky/jw01180135001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01181004001_sky.csv b/msaexp/data/msa_sky/jw01181004001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01181005001_sky.csv b/msaexp/data/msa_sky/jw01181005001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01181006001_sky.csv b/msaexp/data/msa_sky/jw01181006001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01181007001_sky.csv b/msaexp/data/msa_sky/jw01181007001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01181009001_sky.csv b/msaexp/data/msa_sky/jw01181009001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01181010001_sky.csv b/msaexp/data/msa_sky/jw01181010001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01181011001_sky.csv b/msaexp/data/msa_sky/jw01181011001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01181098001_sky.csv b/msaexp/data/msa_sky/jw01181098001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01208046001_sky.csv b/msaexp/data/msa_sky/jw01208046001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01208047001_sky.csv b/msaexp/data/msa_sky/jw01208047001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01208048001_sky.csv b/msaexp/data/msa_sky/jw01208048001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01208049001_sky.csv b/msaexp/data/msa_sky/jw01208049001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01208050001_sky.csv b/msaexp/data/msa_sky/jw01208050001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01208064001_sky.csv b/msaexp/data/msa_sky/jw01208064001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01210001001_sky.csv b/msaexp/data/msa_sky/jw01210001001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01210001002_sky.csv b/msaexp/data/msa_sky/jw01210001002_sky.csv
diff --git a/msaexp/data/msa_sky/jw01210001003_sky.csv b/msaexp/data/msa_sky/jw01210001003_sky.csv
diff --git a/msaexp/data/msa_sky/jw01211013001_sky.csv b/msaexp/data/msa_sky/jw01211013001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01211014001_sky.csv b/msaexp/data/msa_sky/jw01211014001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01211015001_sky.csv b/msaexp/data/msa_sky/jw01211015001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01211017001_sky.csv b/msaexp/data/msa_sky/jw01211017001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01211018001_sky.csv b/msaexp/data/msa_sky/jw01211018001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01211019001_sky.csv b/msaexp/data/msa_sky/jw01211019001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01211020001_sky.csv b/msaexp/data/msa_sky/jw01211020001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01211021001_sky.csv b/msaexp/data/msa_sky/jw01211021001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01213001001_sky.csv b/msaexp/data/msa_sky/jw01213001001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01213002001_sky.csv b/msaexp/data/msa_sky/jw01213002001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01213003001_sky.csv b/msaexp/data/msa_sky/jw01213003001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01213004001_sky.csv b/msaexp/data/msa_sky/jw01213004001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01286001001_sky.csv b/msaexp/data/msa_sky/jw01286001001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01345062001_sky.csv b/msaexp/data/msa_sky/jw01345062001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01345063001_sky.csv b/msaexp/data/msa_sky/jw01345063001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01345065001_sky.csv b/msaexp/data/msa_sky/jw01345065001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01345067001_sky.csv b/msaexp/data/msa_sky/jw01345067001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01345100001_sky.csv b/msaexp/data/msa_sky/jw01345100001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01345102001_sky.csv b/msaexp/data/msa_sky/jw01345102001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01433021001_sky.csv b/msaexp/data/msa_sky/jw01433021001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01433023001_sky.csv b/msaexp/data/msa_sky/jw01433023001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01747004001_sky.csv b/msaexp/data/msa_sky/jw01747004001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01747007001_sky.csv b/msaexp/data/msa_sky/jw01747007001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01747012001_sky.csv b/msaexp/data/msa_sky/jw01747012001_sky.csv
diff --git a/msaexp/data/msa_sky/jw01747014001_sky.csv b/msaexp/data/msa_sky/jw01747014001_sky.csv
diff --git a/msaexp/data/msa_sky/jw02073006001_sky.csv b/msaexp/data/msa_sky/jw02073006001_sky.csv
diff --git a/msaexp/data/msa_sky/jw02073008001_sky.csv b/msaexp/data/msa_sky/jw02073008001_sky.csv
diff --git a/msaexp/data/msa_sky/jw02198003001_sky.csv b/msaexp/data/msa_sky/jw02198003001_sky.csv
diff --git a/msaexp/data/msa_sky/jw02198004001_sky.csv b/msaexp/data/msa_sky/jw02198004001_sky.csv
diff --git a/msaexp/data/msa_sky/jw02282021001_sky.csv b/msaexp/data/msa_sky/jw02282021001_sky.csv
diff --git a/msaexp/data/msa_sky/jw02282022001_sky.csv b/msaexp/data/msa_sky/jw02282022001_sky.csv
diff --git a/msaexp/data/msa_sky/jw02282023001_sky.csv b/msaexp/data/msa_sky/jw02282023001_sky.csv
diff --git a/msaexp/data/msa_sky/jw02561002002_sky.csv b/msaexp/data/msa_sky/jw02561002002_sky.csv
diff --git a/msaexp/data/msa_sky/jw02561002003_sky.csv b/msaexp/data/msa_sky/jw02561002003_sky.csv
diff --git a/msaexp/data/msa_sky/jw02561002004_sky.csv b/msaexp/data/msa_sky/jw02561002004_sky.csv
diff --git a/msaexp/data/msa_sky/jw02561002005_sky.csv b/msaexp/data/msa_sky/jw02561002005_sky.csv
diff --git a/msaexp/data/msa_sky/jw02565002001_sky.csv b/msaexp/data/msa_sky/jw02565002001_sky.csv
diff --git a/msaexp/data/msa_sky/jw02565006001_sky.csv b/msaexp/data/msa_sky/jw02565006001_sky.csv
diff --git a/msaexp/data/msa_sky/jw02565007001_sky.csv b/msaexp/data/msa_sky/jw02565007001_sky.csv
diff --git a/msaexp/data/msa_sky/jw02565100001_sky.csv b/msaexp/data/msa_sky/jw02565100001_sky.csv
diff --git a/msaexp/data/msa_sky/jw02565200001_sky.csv b/msaexp/data/msa_sky/jw02565200001_sky.csv
diff --git a/msaexp/data/msa_sky/jw02565300001_sky.csv b/msaexp/data/msa_sky/jw02565300001_sky.csv
diff --git a/msaexp/data/msa_sky/jw02750002001_sky.csv b/msaexp/data/msa_sky/jw02750002001_sky.csv
diff --git a/msaexp/data/msa_sky/jw02756001001_sky.csv b/msaexp/data/msa_sky/jw02756001001_sky.csv
diff --git a/msaexp/data/msa_sky/jw02767005001_sky.csv b/msaexp/data/msa_sky/jw02767005001_sky.csv
diff --git a/msaexp/data/msa_sky/jw03215001001_sky.csv b/msaexp/data/msa_sky/jw03215001001_sky.csv
diff --git a/msaexp/data/msa_sky/jw03215001002_sky.csv b/msaexp/data/msa_sky/jw03215001002_sky.csv
diff --git a/msaexp/data/msa_sky/jw03215001003_sky.csv b/msaexp/data/msa_sky/jw03215001003_sky.csv
diff --git a/msaexp/data/msa_sky/jw03215001004_sky.csv b/msaexp/data/msa_sky/jw03215001004_sky.csv
diff --git a/msaexp/data/msa_sky/jw03215001005_sky.csv b/msaexp/data/msa_sky/jw03215001005_sky.csv
diff --git a/msaexp/data/msa_sky/jw04106006001_sky.csv b/msaexp/data/msa_sky/jw04106006001_sky.csv
diff --git a/msaexp/data/msa_sky/jw04233001001_sky.csv b/msaexp/data/msa_sky/jw04233001001_sky.csv
diff --git a/msaexp/data/msa_sky/jw04233001002_sky.csv b/msaexp/data/msa_sky/jw04233001002_sky.csv
diff --git a/msaexp/data/msa_sky/jw04233001003_sky.csv b/msaexp/data/msa_sky/jw04233001003_sky.csv
diff --git a/msaexp/data/msa_sky/jw04233005001_sky.csv b/msaexp/data/msa_sky/jw04233005001_sky.csv
diff --git a/msaexp/data/msa_sky/jw04233005002_sky.csv b/msaexp/data/msa_sky/jw04233005002_sky.csv
diff --git a/msaexp/data/msa_sky/jw04233005003_sky.csv b/msaexp/data/msa_sky/jw04233005003_sky.csv
diff --git a/msaexp/data/msa_sky/jw04233006001_sky.csv b/msaexp/data/msa_sky/jw04233006001_sky.csv
diff --git a/msaexp/data/msa_sky/jw04233006002_sky.csv b/msaexp/data/msa_sky/jw04233006002_sky.csv
diff --git a/msaexp/data/msa_sky/jw04233006003_sky.csv b/msaexp/data/msa_sky/jw04233006003_sky.csv
diff --git a/msaexp/data/msa_sky/jw04318100001_sky.csv b/msaexp/data/msa_sky/jw04318100001_sky.csv
diff --git a/msaexp/data/msa_sky/jw04446001001_sky.csv b/msaexp/data/msa_sky/jw04446001001_sky.csv
diff --git a/msaexp/data/msa_sky/jw04557025001_sky.csv b/msaexp/data/msa_sky/jw04557025001_sky.csv
diff --git a/msaexp/data/msa_sky/jw06541001001_sky.csv b/msaexp/data/msa_sky/jw06541001001_sky.csv
diff --git a/msaexp/data/msa_sky/jw06585004001_sky.csv b/msaexp/data/msa_sky/jw06585004001_sky.csv
diff --git a/msaexp/data/msa_sky/nirspec-global-sky.ipynb b/msaexp/data/msa_sky/nirspec-global-sky.ipynb
diff --git a/msaexp/data/msa_sky/nirspec_prism_background.png b/msaexp/data/msa_sky/nirspec_prism_background.png
diff --git a/msaexp/slit_combine.py b/msaexp/slit_combine.py
@@ -524,6 +524,8 @@ def __init__(
         fix_prism_norm=True,
         slit_hotpix_kwargs={},
         sky_arrays=None,
+        sky_file="read",
+        global_sky_df=7,
         estimate_sky_kwargs=None,
         flag_profile_kwargs=None,
         flag_trace_kwargs={},
@@ -596,6 +598,16 @@ def __init__(
         sky_arrays : array-like
             Optional sky data (in progress)
 
+        sky_file : str, None
+            - Filename of a tabulated global sky
+            - `"read"`: try to find a file provided with `msaexp` in
+              ``msaexp/data/sky_data``
+            - None: ignore
+
+        global_sky_df : int
+            If a ``sky_file`` is available and ``estimate_sky_kwargs`` are specified,
+            use this as the degrees of freedom in ``estimate_sky_kwargs``
+
         estimate_sky_kwargs : None, dict
             Arguments to pass to `~msaexp.slit_combine.SlitGroup.estimate_sky` to
             estimate sky directly from the slit data
@@ -745,6 +757,8 @@ def __init__(
             "has_sky_arrays": (sky_arrays is not None),
             "weight_type": weight_type,
             "percentile_outliers": 0,
+            "sky_file": "N/A",
+            "global_sky_df": global_sky_df,
         }
 
         # Comments on meta for header keywords
@@ -773,6 +787,8 @@ def __init__(
             "has_sky_arrays": "sky arrays specified",
             "weight_type": "Weighting scheme for 2D combination",
             "percentile_outliers": "Masked pixels from flag_percentile_outliers",
+            "sky_file": "Filename of a global sky background table",
+            "global_sky_df": "Degrees of freedom of fit with global sky",
         }
 
         self.shapes = []
@@ -795,6 +811,9 @@ def __init__(
 
         self.parse_data()
 
+        if sky_file is not None:
+            self.get_global_sky(sky_file=sky_file)
+
         if self.grating.startswith("G"):
             self.meta["diffs"] |= self.meta["grating_diffs"]
             msg = " Disperser {grating} is a grating.  diffs={diffs}"
@@ -1155,6 +1174,27 @@ def parse_metadata(self):
 
         return info
 
+    def get_global_sky(self, sky_file=None):
+        """
+        Try to read a global sky file from ``msaexp/data/msa_sky``
+        """
+        if sky_file in [None, "read"]:
+            visit = os.path.basename(self.files[0]).split("_")[0]
+            sky_file = os.path.join(
+                os.path.dirname(__file__),
+                "data",
+                "msa_sky",
+                f"{visit}_sky.csv",
+            )
+
+        if os.path.exists(sky_file):
+            sky_data = utils.read_catalog(sky_file)
+            msg = f" {'get_global_sky':<28}: {os.path.basename(sky_file)} to sky_arrays"
+            utils.log_comment(utils.LOGFILE, msg, verbose=VERBOSE_LOG)
+            self.sky_arrays = (sky_data["wave"], sky_data["flux"])
+            self.meta["sky_file"] = os.path.basename(sky_file)
+            self.meta["has_sky_arrays"] = True
+
     def calculate_slices(self):
         """
         Calculate slices to handle unequal cutout sizes
@@ -1730,12 +1770,17 @@ def estimate_sky(
         nbin = sky_wave.shape[0]
         xbin = np.interp(self.wave, sky_wave, np.arange(nbin) / nbin)
 
-        if df <= 0:  # | (self.sky_arrays is not None):
+        if self.meta["sky_file"] not in [None, "N/A"]:
+            df_use = self.meta["global_sky_df"]
+        else:
+            df_use = df
+
+        if df_use <= 0:  # | (self.sky_arrays is not None):
             spl_full = np.ones(xbin.size)[:, None]
         else:
             spl_full = utils.bspline_templates(
                 xbin.flatten(),
-                df=df,
+                df=df_use,
                 minmax=(0, 1),
                 get_matrix=True,
             )
@@ -1762,7 +1807,7 @@ def estimate_sky(
 
         AxT = (spl_full[ok_skyf, :].T / np.sqrt(self.var_rnoise[ok_sky])).T
         yx = (self.sci / np.sqrt(self.var_rnoise))[ok_sky]
-        if df < 0:  # | (self.sky_arrays is not None):
+        if df_use < 0:  # | (self.sky_arrays is not None):
             sky_coeffs = np.array([1.0])
         else:
             sky_coeffs = np.linalg.lstsq(AxT, yx, rcond=None)[0]
@@ -1775,12 +1820,12 @@ def estimate_sky(
             ok_sky &= ~sky_bad
             ok_skyf = ok_sky.flatten()
 
-            if df <= 0:
+            if df_use <= 0:
                 spl_full = np.ones(xbin.size)[:, None]
             else:
                 spl_full = utils.bspline_templates(
                     xbin.flatten(),
-                    df=df,
+                    df=df_use,
                     minmax=(0, 1),
                     get_matrix=True,
                 )
@@ -1793,20 +1838,20 @@ def estimate_sky(
 
             AxT = (spl_full[ok_skyf, :].T / np.sqrt(self.var_rnoise[ok_sky])).T
             yx = (self.sci / np.sqrt(self.var_rnoise))[ok_sky]
-            if df < 0:
+            if df_use < 0:
                 sky_coeffs = np.array([1.0])
             else:
                 sky_coeffs = np.linalg.lstsq(AxT, yx, rcond=None)[0]
 
-        if df <= 0:
+        if df_use <= 0:
             spl_array = np.ones(nbin)[:, None]
             sky_covar = np.array([1.0])
         else:
             sky_covar = utils.safe_invert(np.dot(AxT.T, AxT))
 
             spl_array = utils.bspline_templates(
                 np.arange(nbin) / nbin,
-                df=df,
+                df=df_use,
                 minmax=(0, 1),
                 get_matrix=True,
             )
@@ -1828,7 +1873,7 @@ def estimate_sky(
             "sky_coeffs": sky_coeffs,
             "sky_covar": sky_covar,
             "spl_trim": spl_trim,
-            "df": df,
+            "df": df_use,
             "sky2d": sky2d,
             "use": use,
             "npix": ok_sky.sum(),

diff --git a/msaexp/tests/test_combine.py b/msaexp/tests/test_combine.py
@@ -12,6 +12,7 @@
 
 TARGETS = ["1345_933"]
 
+
 def data_path():
     return os.path.join(os.path.dirname(__file__), "data")
 
@@ -113,6 +114,7 @@ def test_slit_group():
         bar_corr_mode="wave",
         fix_prism_norm=True,
         sky_arrays=None,
+        sky_file=None,
         estimate_sky_kwargs=None,
         flag_profile_kwargs={},
         flag_percentile_kwargs={},
@@ -234,6 +236,7 @@ def test_slit_group():
 
     assert np.nanmin(obj.bar[obj.mask]) > default_kwargs["min_bar"]
 
+    # These are the same if there is no sky model
     assert np.allclose(obj.data[obj.mask], obj.sci[obj.mask])
 
     assert np.allclose(np.nanmin(obj.fixed_yshutter), -1.13236, rtol=1.0e4)
@@ -308,6 +311,10 @@ def test_slit_group():
         make_plot=False,
     )
 
+    # Read global sky
+    obj.get_global_sky(sky_file="read")
+    assert obj.meta["sky_file"] == "jw01345062001_sky.csv"
+
     obj.apply_normalization_correction()
 
     obj.meta["bar_corr_mode"] = "slit"

diff --git a/setup.cfg b/setup.cfg
@@ -53,4 +53,5 @@ docs =
 
 [options.package_data]
 msaexp.data = 
-    *
+    *
+    msa_sky/*