From d889a7854ef3475021a74ed1acf0efdde5cd8def Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Fri, 28 Oct 2022 19:08:48 +0200 Subject: [PATCH 01/15] refactoring functions but buggy --- config/auxtel.ini | 6 +- runFitter.py | 2 +- spectractor/extractor/chromaticpsf.py | 12 ++ spectractor/extractor/extractor.py | 67 ++++++----- spectractor/extractor/images.py | 1 + spectractor/extractor/spectrum.py | 87 ++++++++++++++ spectractor/fit/fit_spectrogram.py | 45 +++++--- spectractor/fit/fit_spectrum.py | 6 +- spectractor/simulation/simulator.py | 156 +++++--------------------- 9 files changed, 196 insertions(+), 186 deletions(-) diff --git a/config/auxtel.ini b/config/auxtel.ini index 92929d229..6e9dc2b15 100644 --- a/config/auxtel.ini +++ b/config/auxtel.ini @@ -32,11 +32,11 @@ OBS_TRANSMISSION_SYSTEMATICS = 0.005 # observed object to choose between STAR, HG-AR, MONOCHROMATOR OBS_OBJECT_TYPE = STAR # telescope transmission file -OBS_TELESCOPE_TRANSMISSION = calexp_2020031500162-EMPTY_ronchi90lpmm-det000_auxtel_transmission.txt +OBS_TELESCOPE_TRANSMISSION = multispectra_holo4_003_HD142331_AuxTel_throughput.txt # full instrument transmission file -OBS_FULL_INSTRUMENT_TRANSMISSON = calexp_2020031500162-EMPTY_ronchi90lpmm-det000_auxtel_transmission.txt +OBS_FULL_INSTRUMENT_TRANSMISSON = multispectra_holo4_003_HD142331_AuxTel_throughput.txt # quantum efficiency of the detector file -OBS_QUANTUM_EFFICIENCY = calexp_2020031500162-EMPTY_ronchi90lpmm-det000_auxtel_transmission.txt +OBS_QUANTUM_EFFICIENCY = multispectra_holo4_003_HD142331_AuxTel_throughput.txt # Camera (x,y) rotation angle with respect to (north-up, east-left) system OBS_CAMERA_ROTATION = 0 # Camera (x,y) flip signs with respect to (north-up, east-left) system diff --git a/runFitter.py b/runFitter.py index c6fb1706b..7dc8d73a6 100644 --- a/runFitter.py +++ b/runFitter.py @@ -35,7 +35,7 @@ atmgrid_filename = file_name.replace('sim', 'reduc').replace('spectrum', 'atmsim') w = SpectrumFitWorkspace(file_name, atmgrid_file_name=atmgrid_filename, nsteps=1000, burnin=200, nbins=10, verbose=1, plot=True, live_fit=False) - run_spectrum_minimisation(w, method="newton") + #run_spectrum_minimisation(w, method="newton") w = SpectrogramFitWorkspace(file_name, atmgrid_file_name=atmgrid_filename, nsteps=2000, burnin=1000, nbins=10, verbose=1, plot=True, live_fit=False) run_spectrogram_minimisation(w, method="newton") diff --git a/spectractor/extractor/chromaticpsf.py b/spectractor/extractor/chromaticpsf.py index 425dc4b44..c25db41f7 100644 --- a/spectractor/extractor/chromaticpsf.py +++ b/spectractor/extractor/chromaticpsf.py @@ -768,6 +768,18 @@ def get_algebraic_distance_along_dispersion_axis(self, shift_x=0, shift_y=0): return np.asarray(np.sign(self.table['Dx']) * np.sqrt((self.table['Dx'] - shift_x) ** 2 + (self.table['Dy_disp_axis'] - shift_y) ** 2)) + def update(self, psf_poly_params, x0, y0, angle, plot=False): + profile_params = self.from_poly_params_to_profile_params(psf_poly_params, apply_bounds=True) + self.fill_table_with_profile_params(profile_params) + Dx = np.arange(self.Nx) - x0 #self.spectrogram_x0 - shift_x # distance in (x,y) spectrogram frame for column x + self.table["Dx"] = Dx + self.table['Dy_disp_axis'] = np.tan(angle * np.pi / 180) * self.table['Dx'] + self.table['Dy'] = np.copy(self.table['y_c']) - y0 + # self.profile_params = self.from_table_to_profile_params() + if plot: + self.plot_summary() + return profile_params + def plot_summary(self, truth=None): fig, ax = plt.subplots(2, 1, sharex='all', figsize=(12, 6)) PSF_models = [] diff --git a/spectractor/extractor/extractor.py b/spectractor/extractor/extractor.py index c2ef1c2aa..7e0c757d2 100644 --- a/spectractor/extractor/extractor.py +++ b/spectractor/extractor/extractor.py @@ -126,7 +126,6 @@ def __init__(self, spectrum, amplitude_priors_method="noprior", nwalkers=18, nst self.bounds = np.concatenate([np.array([(0, 2 / parameters.GRATING_ORDER_2OVER1), bounds_D, (-parameters.PIXSHIFT_PRIOR, parameters.PIXSHIFT_PRIOR), (-10 * parameters.PIXSHIFT_PRIOR, 10 * parameters.PIXSHIFT_PRIOR), - # (-20 , 20 ), (-90, 90), (0.2, 5), (-360, 360), (300, 1100), (-100, 100), (1.001, 3)]), list(psf_poly_params_bounds) * 2]) self.fixed = [False] * self.p.size @@ -399,6 +398,9 @@ def simulate(self, *params): A2, D2CCD, dx0, dy0, angle, B, rot, pressure, temperature, airmass, *poly_params = params poly_params_order1 = poly_params[:len(poly_params)//2] poly_params_order2 = poly_params[len(poly_params)//2:] + self.spectrum.adr_params[2] = temperature + self.spectrum.adr_params[3] = pressure + self.spectrum.adr_params[-1] = airmass # recompute angle and dy0 if fixed while y_c parameters are free # if self.fixed[3] and self.fixed[4] and not np.any([self.fixed[k] for k, par in enumerate(self.input_labels) if "y_c" in par]): # pval_leg = [self.p[k] for k, par in enumerate(self.input_labels) if "y_c" in par] @@ -417,12 +419,10 @@ def simulate(self, *params): profile_params[:, 1] = np.arange(self.Nx) self.spectrum.chromatic_psf.fill_table_with_profile_params(profile_params) - # Distance in x and y with respect to the true order 0 position at lambda_ref + # # Distance in x and y with respect to the true order 0 position at lambda_ref Dx = np.arange(self.Nx) - self.spectrum.spectrogram_x0 - dx0 # distance in (x,y) spectrogram frame for column x Dy_disp_axis = np.tan(angle * np.pi / 180) * Dx # disp axis height in spectrogram frame for x distance = np.sign(Dx) * np.sqrt(Dx * Dx + Dy_disp_axis * Dy_disp_axis) # algebraic distance along dispersion axis - self.spectrum.chromatic_psf.table["Dy_disp_axis"] = Dy_disp_axis - self.spectrum.chromatic_psf.table["Dx"] = Dx # First guess of wavelengths self.spectrum.disperser.D = np.copy(D2CCD) @@ -434,9 +434,6 @@ def simulate(self, *params): order=self.spectrum.order+np.sign(self.spectrum.order)) # Evaluate ADR - self.spectrum.adr_params[2] = temperature - self.spectrum.adr_params[3] = pressure - self.spectrum.adr_params[-1] = airmass adr_x = np.zeros_like(Dx) adr_y = np.zeros_like(Dy_disp_axis) for k in range(3): @@ -458,24 +455,34 @@ def simulate(self, *params): lambdas_order2 = self.spectrum.disperser.grating_pixel_to_lambda(distance - adr_u_2, self.spectrum.x0 + np.asarray([dx0, dy0]), order=self.spectrum.order+np.sign(self.spectrum.order)) + print("bef",self.lambdas[-5:], lambdas_order2[-5:]) + self.lambdas, lambdas_order2, dispersion_law, dispersion_law_order2 = self.spectrum.compute_dispersion_in_spectrogram(D2CCD, dx0, dy0, angle, with_adr=True, niter=3) + print("aft",self.lambdas[-5:], lambdas_order2[-5:]) + # Fill spectrogram trace as a function of the pixel column x - profile_params[:, 1] = Dx + self.spectrum.spectrogram_x0 + adr_x + dx0 - profile_params[:, 2] += Dy_disp_axis + (0*self.spectrum.spectrogram_y0 + adr_y + dy0) - self.bgd_width + # self.spectrum.chromatic_psf.table["Dy_disp_axis"] = Dy_disp_axis + # self.spectrum.chromatic_psf.table["Dx"] = Dx + befx = Dx + self.spectrum.spectrogram_x0 + adr_x + dx0 + befy = Dy_disp_axis - self.bgd_width + adr_y + dy0 + print("dispx bef", befx[-5:]) + print("dispy bef", befy[-5:]) + profile_params[:, 1] = dispersion_law.real + self.spectrum.spectrogram_x0 #+ adr_x + dx0 + profile_params[:, 2] += dispersion_law.imag - self.bgd_width # Dy_disp_axis + adr_y + dy0 + print("dispx aft", dispersion_law.real[-5:] + self.spectrum.spectrogram_x0) + print("dispy aft", dispersion_law.imag[-5:] - self.bgd_width) # Prepare order 2 profile params indexed by the wavelength associated to x profile_params_order2 = self.spectrum.chromatic_psf.from_poly_params_to_profile_params(poly_params_order2, apply_bounds=True) # Second diffraction order amplitude is the first order amplitude multiplied by ratio 2/1 # Ratio 2/1 is in flam/flam but no need to convert in ADU/ADU because lambda*dlambda is the same for both orders profile_params_order2[:, 0] = self.spectrum.disperser.ratio_order_2over1(self.lambdas) - # profile_params_order2[:, 1] = Dx + self.spectrum.spectrogram_x0 + adr_x_2 + dx0 - # profile_params_order2[:, 2] = Dy_disp_axis + (self.spectrum.spectrogram_y0 + adr_y_2 + dy0) - self.bgd_width # For each A(lambda)=A_x, affect an order 2 PSF with correct position and # same PSF as for the order 1 but at the same position distance_order2 = self.spectrum.disperser.grating_lambda_to_pixel(self.lambdas, - self.spectrum.x0 + np.asarray([dx0, dy0]), - order=self.spectrum.order+np.sign(self.spectrum.order)) + self.spectrum.x0 + np.asarray([dx0, dy0]), + order=self.spectrum.order+np.sign(self.spectrum.order)) # for k in range(3, profile_params.shape[1]): # # profile_params_order2[:, k] = interp1d(self.lambdas_order2, profile_params_order2[:, k], # # kind="cubic", fill_value="extrapolate")(self.lambdas) @@ -483,8 +490,14 @@ def simulate(self, *params): # kind="cubic", fill_value="extrapolate")(distance_order2) Dx_order2 = distance_order2 * np.cos(angle * np.pi / 180) Dy_disp_axis_order2 = distance_order2 * np.sin(angle * np.pi / 180) - profile_params_order2[:, 1] = Dx_order2 + self.spectrum.spectrogram_x0 + adr_x + dx0 - profile_params_order2[:, 2] += Dy_disp_axis_order2 + (0*self.spectrum.spectrogram_y0 + adr_y + dy0) - self.bgd_width + befx = Dx_order2 + self.spectrum.spectrogram_x0 + adr_x + dx0 + befy = Dy_disp_axis_order2 - self.bgd_width + adr_y + dy0 + print("dispx bef", befx[-5:]) + print("dispy bef", befy[-5:]) + profile_params_order2[:, 1] = dispersion_law_order2.real + self.spectrum.spectrogram_x0 # Dx_order2 + adr_x + dx0 + profile_params_order2[:, 2] += dispersion_law_order2.imag - self.bgd_width # Dy_disp_axis_order2 + (0*self.spectrum.spectrogram_y0 + adr_y + dy0) - self.bgd_width + print("dispx aft", dispersion_law_order2.real[-5:] + self.spectrum.spectrogram_x0) + print("dispy aft", dispersion_law_order2.imag[-5:] - self.bgd_width) if parameters.DEBUG and False: plt.figure(figsize=(18,4)) @@ -494,7 +507,7 @@ def simulate(self, *params): plt.scatter(profile_params_order2[:, 1], profile_params_order2[:, 2], label="order 2", cmap=from_lambda_to_colormap(self.lambdas), c=self.lambdas) plt.plot(profile_params[:, 1], profile_params[:, 2], label="profile") - plt.plot(profile_params[:, 1], Dy_disp_axis + self.spectrum.spectrogram_y0 + dy0 - self.bgd_width, 'k-', + plt.plot(profile_params[:, 1], self.spectrum.chromatic_psf["Dy_disp_axis"] + self.spectrum.spectrogram_y0 + dy0 - self.bgd_width, 'k-', label="disp_axis") plt.plot(self.spectrum.chromatic_psf.table['Dx'] + self.spectrum.spectrogram_x0 + dx0, self.spectrum.chromatic_psf.table['Dy'] + self.spectrum.spectrogram_y0 + dy0 - self.bgd_width, @@ -511,7 +524,6 @@ def simulate(self, *params): # save for plotting self.profile_params = profile_params self.profile_params_order2 = profile_params_order2 - self.Dy_disp_axis = Dy_disp_axis self.dx0 = dx0 self.dy0 = dy0 @@ -805,7 +817,7 @@ def plot_fitted_parameters(self,title="output from simulate"): plt.scatter(self.profile_params_order2[:, 1], self.profile_params_order2[:, 2], label="order 2", cmap=from_lambda_to_colormap(self.lambdas), s=30, c=self.lambdas) plt.plot(self.profile_params[:, 1], self.profile_params[:, 2], label="profile",lw=3) - plt.plot(self.profile_params[:, 1], self.Dy_disp_axis + self.spectrum.spectrogram_y0 + self.dy0 - self.bgd_width, 'k-', + plt.plot(self.profile_params[:, 1], self.spectrum.chromatic_psf.table['Dy_disp_axis'] + self.spectrum.spectrogram_y0 + self.dy0 - self.bgd_width, 'k-', label="disp_axis",lw=3) plt.plot(self.spectrum.chromatic_psf.table['Dx'] + self.spectrum.spectrogram_x0 + self.dx0, self.spectrum.chromatic_psf.table['Dy'] + self.spectrum.spectrogram_y0 + self.dy0 - self.bgd_width, @@ -860,9 +872,6 @@ def run_ffm_minimisation(w, method="newton", niter=2): my_logger.info(f"\n --- Start FFM with adjust_spectrogram_position_parameters --- ") w.adjust_spectrogram_position_parameters() - if parameters.DEBUG and parameters.DISPLAY: - w.plot_fitted_parameters(title="output of adjust_spectrogram_parameters") - if method != "newton": run_minimisation(w, method=method) else: @@ -887,15 +896,6 @@ def run_ffm_minimisation(w, method="newton", niter=2): if parameters.DEBUG and parameters.DISPLAY: w.plot_fit() - w.plot_fitted_parameters(title="output of run_minimisation") - - # don't want to fix parameters that should not be fixed - #my_logger.info(f"\n --- Start intermediate FFM with adjust_spectrogram_position_parameters (added to see if it help)") - #w.adjust_spectrogram_position_parameters() - - - - my_logger.info("\n --- Start regularization parameter only ---") # Optimize the regularisation parameter only if it was not done before @@ -922,8 +922,6 @@ def run_ffm_minimisation(w, method="newton", niter=2): if parameters.DEBUG and parameters.DISPLAY: w.plot_fit() - w.plot_fitted_parameters(title="output of regularization parameter") - my_logger.info("\n --- Start run_minimisation_sigma_clipping ---") for i in range(niter): @@ -939,8 +937,6 @@ def run_ffm_minimisation(w, method="newton", niter=2): if parameters.DEBUG and parameters.DISPLAY: w.plot_fit() - w.plot_fitted_parameters(title="run_minimisation_sigma_clipping") - w.spectrum.lambdas = np.copy(w.lambdas) w.spectrum.data = np.copy(w.amplitude_params) @@ -1357,7 +1353,6 @@ def extract_spectrum_from_image(image, spectrum, signal_width=10, ws=(20, 30), r my_logger.info(f'\n\tExtract spectrogram: crop raw image [{xmin}:{xmax},{ymin}:{ymax}] (size ({Nx}, {Ny}))') # Extract the non rotated background - my_logger.info('\n\t ======================= Extract the non rotated background =============================') bgd_model_func, bgd_res, bgd_rms = extract_spectrogram_background_sextractor(data, err, ws=ws) @@ -1392,6 +1387,8 @@ def extract_spectrum_from_image(image, spectrum, signal_width=10, ws=(20, 30), r spectrum.spectrogram_xmax = xmax spectrum.spectrogram_ymin = ymin spectrum.spectrogram_ymax = ymax + spectrum.spectrogram_Nx = Nx + spectrum.spectrogram_Ny = Ny spectrum.spectrogram_deg = spectrum.chromatic_psf.deg spectrum.spectrogram_saturation = spectrum.chromatic_psf.saturation diff --git a/spectractor/extractor/images.py b/spectractor/extractor/images.py index 22628dbbf..a7856251b 100644 --- a/spectractor/extractor/images.py +++ b/spectractor/extractor/images.py @@ -742,6 +742,7 @@ def load_AUXTEL_image(image): # pragma: no cover parameters.OBS_CAMERA_ROTATION -= 360 if parameters.OBS_CAMERA_ROTATION < -360: parameters.OBS_CAMERA_ROTATION += 360 + image.header["CAM_ROT"] = parameters.OBS_CAMERA_ROTATION if "CD2_1" in hdu_list[1].header: rotation_wcs = 180 / np.pi * np.arctan2(hdu_list[1].header["CD2_1"], hdu_list[1].header["CD1_1"]) + 90 if not np.isclose(rotation_wcs % 360, parameters.OBS_CAMERA_ROTATION % 360, atol=2): diff --git a/spectractor/extractor/spectrum.py b/spectractor/extractor/spectrum.py index af6227cf5..ef8b1dc20 100644 --- a/spectractor/extractor/spectrum.py +++ b/spectractor/extractor/spectrum.py @@ -621,6 +621,16 @@ def load_spectrum(self, input_file_name, spectrogram_file_name_override=None, # print(f'set {attribute} to {item}') else: print(f'Failed to set spectrum attribute {attribute} using header {header_key}') + if "CAM_ROT" in self.header: + parameters.OBS_CAMERA_ROTATION = float(self.header["CAM_ROT"]) + elif "ROTPA" in self.header: + parameters.OBS_CAMERA_ROTATION = 270 - float(self.header["ROTPA"]) + if parameters.OBS_CAMERA_ROTATION > 360: + parameters.OBS_CAMERA_ROTATION -= 360 + if parameters.OBS_CAMERA_ROTATION < -360: + parameters.OBS_CAMERA_ROTATION += 360 + else: + self.my_logger.warning("No information about camera rotation in Spectrum header.") # set the more complex items by hand here if target := self.header.get('TARGET'): @@ -746,6 +756,83 @@ def load_chromatic_psf(self, input_file_name): else: self.my_logger.warning(f'\n\tSpectrogram file {input_file_name} not found') + def compute_dispersion_in_spectrogram(self, D, shift_x, shift_y, angle, niter=3, with_adr=True): + """Compute the dispersion relation in a spectrogram, using grating dispersion model and ADR. + Origin is the order 0 centroid. + + Parameters + ---------- + D: float + The distance between the CCD and the disperser in mm. + shift_x: float + Shift in the x axis direction for order 0 position in pixel. + shift_y: float + Shift in the y axis direction for order 0 position in pixel. + angle: float + Main dispersion axis angle in degrees. + niter: int, optional + Number of iterations to compute ADR (default: 3). + with_adr: bool, optional + If True, add ADR effect to grating dispersion model (default: True). + + Returns + ------- + lambdas: array_like + Wavelength array for parameters.SPECTRUM_ORDER diffraction. + lambdas_order2: array_like + Wavelength array for parameters.SPECTRUM_ORDER+1 diffraction. + dispersion_law: array_like + Complex array coding the 2D dispersion relation in the spectrogram for parameters.SPECTRUM_ORDER diffraction. + dispersion_law_order2: array_like + Complex array coding the 2D dispersion relation in the spectrogram for parameters.SPECTRUM_ORDER+1 diffraction. + + """ + # Distance in x and y with respect to the true order 0 position at lambda_ref + Dx = np.arange(self.spectrogram_Nx) - self.spectrogram_x0 - shift_x # distance in (x,y) spectrogram frame for column x + Dy_disp_axis = np.tan(angle * np.pi / 180) * Dx # disp axis height in spectrogram frame for x + distance = np.sign(Dx) * np.sqrt(Dx * Dx + Dy_disp_axis * Dy_disp_axis) # algebraic distance along dispersion axis + + # Wavelengths using the order 0 shifts (ADR has no impact as it shifts order 0 and order p equally) + new_x0 = [self.x0[0] + shift_x, self.x0[1] + shift_y] + # First guess of wavelengths + self.disperser.D = np.copy(D) + lambdas = self.disperser.grating_pixel_to_lambda(distance, new_x0, order=self.order) + lambdas_order2 = self.disperser.grating_pixel_to_lambda(distance, new_x0, order=self.order+np.sign(self.order)) + + # Evaluate ADR + adr_x = np.zeros_like(Dx) + adr_y = np.zeros_like(Dy_disp_axis) + for k in range(niter): + adr_ra, adr_dec = adr_calib(lambdas, self.adr_params, parameters.OBS_LATITUDE, + lambda_ref=self.lambda_ref) + adr_x, adr_y = flip_and_rotate_adr_to_image_xy_coordinates(adr_ra, adr_dec, dispersion_axis_angle=0) + adr_u, adr_v = flip_and_rotate_adr_to_image_xy_coordinates(adr_ra, adr_dec, dispersion_axis_angle=angle) + + # Evaluate ADR for order 2 + adr_ra, adr_dec = adr_calib(lambdas_order2, self.adr_params, parameters.OBS_LATITUDE, + lambda_ref=self.lambda_ref) + # adr_x_2, adr_y_2 = flip_and_rotate_adr_to_image_xy_coordinates(adr_ra, adr_dec, dispersion_axis_angle=0) + adr_u_2, adr_v_2 = flip_and_rotate_adr_to_image_xy_coordinates(adr_ra, adr_dec, dispersion_axis_angle=angle) + + # Compute lambdas at pixel column x + lambdas = self.disperser.grating_pixel_to_lambda(distance - adr_u, new_x0, order=self.order) + lambdas_order2 = self.disperser.grating_pixel_to_lambda(distance - adr_u_2, new_x0, order=self.order+np.sign(self.order)) + + # Compute lambdas at pixel column x + # lambdas = self.disperser.grating_pixel_to_lambda(distance - 0*adr_u, new_x0, order=1) + # Position (not distance) in pixel of wavelength lambda order 1 centroid in the (x,y) spectrogram frame + dispersion_law = (Dx + shift_x + with_adr * adr_x) + 1j * (Dy_disp_axis + with_adr * adr_y + shift_y) + + # Compute lambdas at pixel column x + # lambdas_order2 = self.disperser.grating_pixel_to_lambda(distance - 0*adr_u, new_x0, order=2) + # Position (not distance) in pixel of wavelength lambda order 2 centroid in the (x,y) spectrogram frame + distance_order2 = self.disperser.grating_lambda_to_pixel(lambdas, x0=new_x0, order=self.order+np.sign(self.order)) + Dx_order2 = distance_order2 * np.cos(angle * np.pi / 180) + Dy_disp_axis_order2 = distance_order2 * np.sin(angle * np.pi / 180) + dispersion_law_order2 = (Dx_order2 + shift_x + with_adr * adr_x) + \ + 1j * (Dy_disp_axis_order2 + with_adr * adr_y + shift_y) + return lambdas, lambdas_order2, dispersion_law, dispersion_law_order2 + def detect_lines(lines, lambdas, spec, spec_err=None, cov_matrix=None, fwhm_func=None, snr_minlevel=3, ax=None, calibration_lines_only=False, diff --git a/spectractor/fit/fit_spectrogram.py b/spectractor/fit/fit_spectrogram.py index 2e5b26db3..bb1d3a40b 100644 --- a/spectractor/fit/fit_spectrogram.py +++ b/spectractor/fit/fit_spectrogram.py @@ -107,26 +107,31 @@ def __init__(self, file_name, atmgrid_file_name="", nwalkers=18, nsteps=1000, bu self.fixed_psf_params = np.array([0, 1, 2, 3, 4, 9]) self.atm_params_indices = np.array([2, 3, 4]) self.psf_params_start_index = self.p.size - self.p = np.concatenate([self.p, self.psf_poly_params]) + self.p = np.concatenate([self.p, self.psf_poly_params, np.copy(self.psf_poly_params)]) self.input_labels = ["A1", "A2", "ozone [db]", "PWV [mm]", "VAOD", r"D_CCD [mm]", - r"shift_x [pix]", r"shift_y [pix]", r"angle [deg]", "B"] + list(self.psf_poly_params_labels) + r"shift_x [pix]", r"shift_y [pix]", r"angle [deg]", "B"] + list(self.psf_poly_params_labels) * 2 self.axis_names = ["$A_1$", "$A_2$", "ozone [db]", "PWV [mm]", "VAOD", r"$D_{CCD}$ [mm]", r"$\Delta_{\mathrm{x}}$ [pix]", r"$\Delta_{\mathrm{y}}$ [pix]", - r"$\theta$ [deg]", "$B$"] + list(self.psf_poly_params_names) + r"$\theta$ [deg]", "$B$"] + list(self.psf_poly_params_names) * 2 bounds_D = (self.D - 5 * parameters.DISTANCE2CCD_ERR, self.D + 5 * parameters.DISTANCE2CCD_ERR) self.bounds = np.concatenate([np.array([(0, 2), (0, 2/parameters.GRATING_ORDER_2OVER1), (100, 700), (0, 10), (0, 0.1), bounds_D, (-2, 2), (-10, 10), (-90, 90), (0.8, 1.2)]), - psf_poly_params_bounds]) + list(psf_poly_params_bounds) * 2]) self.fixed = [False] * self.p.size for k, par in enumerate(self.input_labels): - if "x_c" in par or "saturation" in par or "y_c" in par: + if "x_c" in par or "saturation" in par: # or "y_c" in par: self.fixed[k] = True + for k, par in enumerate(self.input_labels): + if "y_c" in par: + self.fixed[k] = False + self.p[k] = 0 # A2 is free only if spectrogram is a simulation or if the order 2/1 ratio is not known and flat self.fixed[1] = "A2_T" not in self.spectrum.header # not self.spectrum.disperser.flat_ratio_order_2over1 # self.fixed[5:7] = [True, True] # DCCD, x0 + self.fixed[1] = False self.fixed[6] = True # Delta x - # self.fixed[7] = True # Delta y - # self.fixed[8] = True # angle + self.fixed[7] = True # Delta y + self.fixed[8] = True # angle self.fixed[9] = True # B if atmgrid_file_name != "": self.bounds[2] = (min(self.atmosphere.OZ_Points), max(self.atmosphere.OZ_Points)) @@ -139,6 +144,16 @@ def __init__(self, file_name, atmgrid_file_name="", nwalkers=18, nsteps=1000, bu self.amplitude_truth = None self.get_spectrogram_truth() + # PSF cube computation + self.psf_cube_masked = None + self.psf_cube = None + self.psf_cube_order2 = None + self.fix_psf_cube = False + self.fix_psf_cube_order2 = False + self.psf_params_index = np.arange(0, self.psf_params_start_index+len(self.psf_poly_params)) + self.psf_params_index_order2 = np.concatenate([np.arange(0, self.psf_params_start_index), np.arange(np.max(self.psf_params_index)+1, len(self.p))]) + self.psf_params_start_index_order2 = np.max(self.psf_params_index)+1 + def crop_spectrogram(self): """Crop the spectrogram in the middle, keeping a vertical width of 2*parameters.PIXWIDTH_SIGNAL around the signal region. @@ -480,11 +495,15 @@ def run_spectrogram_minimisation(fit_workspace, method="newton"): # fit_workspace.simulation.fast_sim = True # fit_workspace.simulation.fix_psf_cube = False # fit_workspace.fixed = np.copy(fixed) - # guess = fit_workspace.p - # params_table, costs = run_gradient_descent(fit_workspace, guess, epsilon, params_table, costs, - # fix=fit_workspace.fixed, xtol=1e-5, ftol=1e-3, niter=10) - - fit_workspace.simulation.fast_sim = False + # for ip, label in enumerate(fit_workspace.input_labels): + # if "y_c_0" in label: + # fit_workspace.fixed[ip] = False + # else: + # fit_workspace.fixed[ip] = True + # run_minimisation(fit_workspace, method="newton", epsilon=epsilon, fix=fit_workspace.fixed, + # xtol=1e-2, ftol=10 / fit_workspace.data.size, verbose=False) + + fit_workspace.simulation.fast_sim = True fit_workspace.simulation.fix_psf_cube = False fit_workspace.fixed = np.copy(fixed) # guess = fit_workspace.p @@ -492,7 +511,7 @@ def run_spectrogram_minimisation(fit_workspace, method="newton"): # fix=fit_workspace.fixed, xtol=1e-6, ftol=1 / fit_workspace.data.size, # niter=40) run_minimisation_sigma_clipping(fit_workspace, method="newton", epsilon=epsilon, fix=fit_workspace.fixed, - xtol=1e-6, ftol=1 / fit_workspace.data.size, sigma_clip=20, niter_clip=3, + xtol=1e-6, ftol=1 / fit_workspace.data.size, sigma_clip=100, niter_clip=3, verbose=False) my_logger.info(f"\n\tNewton: total computation time: {time.time() - start}s") if fit_workspace.filename != "": diff --git a/spectractor/fit/fit_spectrum.py b/spectractor/fit/fit_spectrum.py index 70007a2eb..8e980a887 100644 --- a/spectractor/fit/fit_spectrum.py +++ b/spectractor/fit/fit_spectrum.py @@ -81,8 +81,8 @@ def __init__(self, file_name, atmgrid_file_name="", nwalkers=18, nsteps=1000, bu self.A2 = 0 self.ozone = 400. self.pwv = 3 - self.aerosols = 0.05 - self.reso = -1 + self.aerosols = 0.01 + self.reso = 1 self.D = self.spectrum.header['D2CCD'] self.shift_x = self.spectrum.header['PIXSHIFT'] self.B = 0 @@ -91,7 +91,7 @@ def __init__(self, file_name, atmgrid_file_name="", nwalkers=18, nsteps=1000, bu self.fixed = [False] * self.p.size # self.fixed[0] = True self.fixed[1] = "A2_T" not in self.spectrum.header # fit A2 only on sims to evaluate extraction biases - self.fixed[5] = True + self.fixed[5] = False # self.fixed[6:8] = [True, True] self.fixed[7] = True self.fixed[8] = True diff --git a/spectractor/simulation/simulator.py b/spectractor/simulation/simulator.py index 639ad7c73..42f9a0e11 100644 --- a/spectractor/simulation/simulator.py +++ b/spectractor/simulation/simulator.py @@ -336,94 +336,6 @@ def integrand(lbda): # spectrum_err[idx] = 1e6 * np.max(spectrum_err) return spectrum, spectrum_err - def simulate_psf(self, psf_poly_params): - profile_params = self.chromatic_psf.from_poly_params_to_profile_params(psf_poly_params, apply_bounds=True) - self.chromatic_psf.fill_table_with_profile_params(profile_params) - self.chromatic_psf.table['Dy_disp_axis'] = \ - np.tan(self.rotation_angle * np.pi / 180) * self.chromatic_psf.table['Dx'] - self.chromatic_psf.table['Dy'] = np.copy(self.chromatic_psf.table['y_c']) - self.r0.imag - self.chromatic_psf.profile_params = self.chromatic_psf.from_table_to_profile_params() - if False: - self.chromatic_psf.plot_summary() - return self.chromatic_psf.profile_params - - def simulate_dispersion(self, D, shift_x, shift_y): - # Distance in x and y with respect to the true order 0 position at lambda_ref - Dx = np.arange(self.spectrogram_Nx) - self.spectrogram_x0 - shift_x # distance in (x,y) spectrogram frame for column x - Dy_disp_axis = np.tan(self.rotation_angle * np.pi / 180) * Dx # disp axis height in spectrogram frame for x - distance = np.sqrt(Dx**2 + Dy_disp_axis**2) - - # Wavelengths using the order 0 shifts (ADR has no impact as it shifts order 0 and order p equally) - new_x0 = [self.x0[0] + shift_x, self.x0[1] + shift_y] - self.disperser.D = D - self.lambdas = self.disperser.grating_pixel_to_lambda(distance, x0=new_x0, order=1) - distance_order2 = self.disperser.grating_lambda_to_pixel(self.lambdas, x0=new_x0, order=2) - distance_order2 = distance_order2[distance_order2 + self.r0.real < self.Nx + 100] # little margin of 100 pixels - Dx_order2 = distance_order2 * np.cos(self.rotation_angle * np.pi / 180) - Dy_disp_axis_order2 = distance_order2 * np.sin(self.rotation_angle * np.pi / 180) - self.lambdas_order2 = self.disperser.grating_pixel_to_lambda(distance_order2, x0=new_x0, order=2) - self.lambdas_binwidths = np.gradient(self.lambdas) - - # ADR for order 1 - adr_ra, adr_dec = adr_calib(self.lambdas, self.adr_params, parameters.OBS_LATITUDE, lambda_ref=self.lambda_ref) - adr_x, adr_y = flip_and_rotate_adr_to_image_xy_coordinates(adr_ra, adr_dec, dispersion_axis_angle=0) - # adr_u, adr_v = flip_and_rotate_adr_to_image_xy_coordinates(adr_ra, adr_dec, - # dispersion_axis_angle=self.rotation_angle) - - # Compute lambdas at pixel column x - # lambdas = self.disperser.grating_pixel_to_lambda(distance - 0*adr_u, new_x0, order=1) - # Position (not distance) in pixel of wavelength lambda order 1 centroid in the (x,y) spectrogram frame - dispersion_law = self.r0 + (Dx + shift_x + self.with_adr * adr_x) + \ - 1j * (Dy_disp_axis + self.with_adr * adr_y + shift_y) - - # ADR for order 2 - adr_ra, adr_dec = adr_calib(self.lambdas_order2, self.adr_params, parameters.OBS_LATITUDE, - lambda_ref=self.lambda_ref) - adr_x, adr_y = flip_and_rotate_adr_to_image_xy_coordinates(adr_ra, adr_dec, dispersion_axis_angle=0) - # adr_u, adr_v = flip_and_rotate_adr_to_image_xy_coordinates(adr_ra, adr_dec, - # dispersion_axis_angle=self.rotation_angle) - - # Compute lambdas at pixel column x - # lambdas_order2 = self.disperser.grating_pixel_to_lambda(distance - 0*adr_u, new_x0, order=2) - # Position (not distance) in pixel of wavelength lambda order 2 centroid in the (x,y) spectrogram frame - dispersion_law_order2 = self.r0 + (Dx_order2 + shift_x + self.with_adr * adr_x) + \ - 1j * (Dy_disp_axis_order2 + self.with_adr * adr_y + shift_y) - - # Dx_func = interp1d(lambdas / 2, self.chromatic_psf.table['Dx'], bounds_error=False, fill_value=(0, 0)) - # Dy_mean_func=interp1d(lambdas/2,self.chromatic_psf.table['Dy_disp_axis'],bounds_error=False,fill_value=(0,0)) - # dy_func = interp1d(lambdas / 2, self.chromatic_psf.table['Dy'] - self.chromatic_psf.table['Dy_disp_axis'], - # bounds_error=False, fill_value=(0, 0)) - # dispersion_law = r0 + (self.chromatic_psf.table['Dx'] - shift_x) + 1j * ( - # self.chromatic_psf.table['Dy'] - shift_y) - # dispersion_law_order2 = r0 + (Dx_func(lambdas_order2) - shift_x) + 1j * ( - # Dy_mean_func(lambdas_order2) + dy_func(lambdas_order2) - shift_y) - # Dx_func = interp1d(lambdas, self.chromatic_psf.table['Dx'], bounds_error=False, fill_value=(0, 0)) - # Dy_mean_func=interp1d(lambdas,self.chromatic_psf.table['Dy_dips_axis'],bounds_error=False, fill_value=(0, 0)) - - # dispersion laws from the PSF table - # dy_func = interp1d(lambdas, self.chromatic_psf.table['Dy'] - self.chromatic_psf.table['Dy_disp_axis'], - # bounds_error=False, fill_value="extrapolate") - # dispersion_law = r0 + (self.chromatic_psf.table['Dx'] - shift_x) \ - # + 1j * (self.chromatic_psf.table['Dy'] - shift_y) - # dispersion_law_order2 = dispersion_law + 1j * (dy_func(lambdas_order2) - self.chromatic_psf.table['Dy'] - # + self.chromatic_psf.table['Dy_disp_axis']) - - if False: - from spectractor.tools import from_lambda_to_colormap - plt.plot(self.chromatic_psf.table['Dx'], self.chromatic_psf.table['Dy_disp_axis'], 'k-', label="mean") - plt.scatter(dispersion_law.real-self.r0.real, -self.r0.imag + dispersion_law.imag, label="dispersion_law", - cmap=from_lambda_to_colormap(self.lambdas), c=self.lambdas) - plt.scatter(dispersion_law_order2.real-self.r0.real, -self.r0.imag + dispersion_law_order2.imag, - label="dispersion_law_order2", - cmap=from_lambda_to_colormap(self.lambdas), c=self.lambdas) - plt.title(f"x0={new_x0}") - plt.legend() - plt.show() - if parameters.PdfPages: - parameters.PdfPages.savefig() - - return self.lambdas, self.lambdas_order2, dispersion_law, dispersion_law_order2 - # @profile def simulate(self, A1=1.0, A2=0., ozone=300, pwv=5, aerosols=0.05, D=parameters.DISTANCE2CCD, shift_x=0., shift_y=0., angle=0., B=1., psf_poly_params=None): @@ -482,18 +394,24 @@ def simulate(self, A1=1.0, A2=0., ozone=300, pwv=5, aerosols=0.05, D=parameters. import time start = time.time() self.rotation_angle = angle + self.lambdas, lambdas_order2, dispersion_law, dispersion_law_order2 = self.compute_dispersion_in_spectrogram(D, shift_x, shift_y, angle, with_adr=True) + self.lambdas_binwidths = np.gradient(self.lambdas) + self.my_logger.debug(f'\n\tAfter dispersion: {time.time() - start}') + start = time.time() + psf_poly_params_order1 = psf_poly_params[:len(psf_poly_params)//2] + psf_poly_params_order2 = psf_poly_params[len(psf_poly_params)//2:] if self.profile_params is None or not self.fix_psf_cube: - self.profile_params = self.simulate_psf(psf_poly_params) + self.profile_params = self.chromatic_psf.update(psf_poly_params_order1, x0=self.r0.real + shift_x, + y0=self.r0.imag + shift_y, angle=angle, plot=False) + self.profile_params[:, 1] = dispersion_law.real + self.r0.real + self.profile_params[:, 2] += dispersion_law.imag + self.chromatic_psf.table["Dx"] = self.profile_params[:, 1] - self.r0.real + self.chromatic_psf.table["Dy"] = self.profile_params[:, 2] - self.r0.imag self.my_logger.debug(f'\n\tAfter psf params: {time.time() - start}') start = time.time() - lambdas, lambdas_order2, dispersion_law, dispersion_law_order2 = self.simulate_dispersion(D, shift_x, shift_y) - self.chromatic_psf.table["Dx"] = dispersion_law.real - self.r0.real - self.chromatic_psf.table["Dy"] = dispersion_law.imag - self.r0.imag - self.my_logger.debug(f'\n\tAfter dispersion: {time.time() - start}') - start = time.time() if self.atmosphere_sim is None or not self.fix_atm_sim: self.atmosphere_sim = self.atmosphere.simulate(ozone, pwv, aerosols) - spectrum, spectrum_err = self.simulate_spectrum(lambdas, self.atmosphere_sim) + spectrum, spectrum_err = self.simulate_spectrum(self.lambdas, self.atmosphere_sim) self.my_logger.debug(f'\n\tAfter spectrum: {time.time() - start}') # Fill the order 1 cube nlbda = dispersion_law.size @@ -501,8 +419,7 @@ def simulate(self, A1=1.0, A2=0., ozone=300, pwv=5, aerosols=0.05, D=parameters. start = time.time() self.psf_cube = np.zeros((nlbda, self.Ny, self.Nx)) for i in range(0, nlbda, 1): - p = np.array([1, dispersion_law[i].real, dispersion_law[i].imag] + list(self.profile_params[i, 3:])) - self.psf_cube[i] = self.psf.evaluate(self.pixels, p=p) + self.psf_cube[i] = self.psf.evaluate(self.pixels, p=self.profile_params[i, :]) self.my_logger.debug(f'\n\tAfter psf cube: {time.time() - start}') start = time.time() ima1 = np.zeros((self.Ny, self.Nx)) @@ -515,8 +432,8 @@ def simulate(self, A1=1.0, A2=0., ozone=300, pwv=5, aerosols=0.05, D=parameters. # Add order 2 if A2 > 0.: - spectrum_order2, spectrum_order2_err = self.disperser.ratio_order_2over1(lambdas_order2) * \ - self.simulate_spectrum(lambdas_order2, self.atmosphere_sim) + spectrum_order2, spectrum_order2_err = self.disperser.ratio_order_2over1(self.lambdas) * \ + self.simulate_spectrum(self.lambdas, self.atmosphere_sim) if np.any(np.isnan(spectrum_order2)): spectrum_order2[np.isnan(spectrum_order2)] = 0. nlbda2 = dispersion_law_order2.size @@ -525,15 +442,16 @@ def simulate(self, A1=1.0, A2=0., ozone=300, pwv=5, aerosols=0.05, D=parameters. self.psf_cube_order2 = np.zeros((nlbda2, self.Ny, self.Nx)) # For each A(lambda)=A_x, affect an order 2 PSF with correct position and # same PSF as for the order 1 but at the same position - profile_params_order2 = np.copy(self.profile_params) + profile_params_order2 = self.chromatic_psf.from_poly_params_to_profile_params(psf_poly_params_order2, + apply_bounds=True) profile_params_order2[:, 0] = 1 - profile_params_order2[:nlbda2, 1] = dispersion_law_order2.real - profile_params_order2[:nlbda2, 2] = dispersion_law_order2.imag - distance = np.abs(dispersion_law) - distance_order2 = np.abs(dispersion_law_order2) - for k in range(3, self.profile_params.shape[1]): - profile_params_order2[:nlbda2, k] = interp1d(distance, profile_params_order2[:, k], kind="cubic", - fill_value="extrapolate")(distance_order2) + profile_params_order2[:nlbda2, 1] = dispersion_law_order2.real + self.r0.real + profile_params_order2[:nlbda2, 2] += dispersion_law_order2.imag + # distance = np.abs(dispersion_law) + # distance_order2 = np.abs(dispersion_law_order2) + # for k in range(3, self.profile_params.shape[1]): + # profile_params_order2[:nlbda2, k] = interp1d(distance, profile_params_order2[:, k], kind="cubic", + # fill_value="extrapolate")(distance_order2) for i in range(0, nlbda2, 1): self.psf_cube_order2[i] = self.psf.evaluate(self.pixels, p=profile_params_order2[i, :]) self.my_logger.debug(f'\n\tAfter psf cube order 2: {time.time() - start}') @@ -555,30 +473,6 @@ def simulate(self, A1=1.0, A2=0., ozone=300, pwv=5, aerosols=0.05, D=parameters. if self.with_background: self.data += B * self.spectrogram_bgd self.my_logger.debug(f'\n\tAfter bgd: {time.time() - start}') - if False: - fig, ax = plt.subplots(2, 1, sharex="all", figsize=(12, 9)) - im = ax[0].imshow(self.data, origin='lower') - plt.colorbar(im, ax=ax[0], label=self.units) - ax[0].set_title('Model') - im = ax[1].imshow(self.err, origin='lower') - plt.colorbar(im, ax=ax[1], label=self.units) - ax[1].set_title('Err') - ax[1].set_xlabel('X [pixels]') - ax[0].set_ylabel('Y [pixels]') - ax[1].set_ylabel('Y [pixels]') - ax[0].grid() - ax[1].grid() - # if self.with_background: - # ax[2].plot(np.sum(self.data, axis=0), label="model") - # else: - # ax[2].plot(np.sum(self.data + self.spectrogram_bgd, axis=0), label="model") - # ax[2].plot(np.sum(self.spectrogram, axis=0), label="data") - # ax[2].grid() - # ax[2].legend() - fig.tight_layout() - plt.show() - if parameters.PdfPages: - parameters.PdfPages.savefig() return self.lambdas, self.data, self.err From 63f3cc476dc0dcf62ed774cc327e535095405a65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Wed, 2 Nov 2022 17:33:28 +0100 Subject: [PATCH 02/15] update pressure, airmass and temperature in atmospheregrid() with spectrum file name given --- spectractor/simulation/atmosphere.py | 32 ++++++++++++++++++---------- spectractor/simulation/simulator.py | 4 ++-- tests/test_simulator.py | 2 +- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/spectractor/simulation/atmosphere.py b/spectractor/simulation/atmosphere.py index 504ad9ae5..50d841f7f 100644 --- a/spectractor/simulation/atmosphere.py +++ b/spectractor/simulation/atmosphere.py @@ -143,7 +143,8 @@ def plot_transmission(self): # ---------------------------------------------------------------------------------- class AtmosphereGrid(Atmosphere): - def __init__(self, image_filename="", filename="", airmass=1., pressure=800., temperature=10., + def __init__(self, image_filename="", spectrum_filename="", atmgrid_filename="", + airmass=1., pressure=800., temperature=10., pwv_grid=[0, 10, 10], ozone_grid=[100, 700, 7], aerosol_grid=[0, 0.1, 10]): """Class to load and interpolate grids of atmospheric transmission computed with Libradtran. @@ -151,14 +152,16 @@ def __init__(self, image_filename="", filename="", airmass=1., pressure=800., te ---------- image_filename: str, optional The original image fits file name from which the grid was computed or has to be computed (default: ""). - filename: str, optional + spectrum_filename: str, optional + The file name of the spectrum fits file name from which the grid was computed or has to be computed (default: ""). + atmgrid_filename: str, optional The file name of the atmospheric grid if it exists (default: ""). airmass: float, optional - Airmass of the source object (default: 1). + Airmass of the source object (default: 1). Overwritten if spectrum_filename is given. pressure: float, optional - Pressure of the atmosphere in hPa (default: 800). + Pressure of the atmosphere in hPa (default: 800). Overwritten if spectrum_filename is given. temperature: float, optional - Temperature of the atmosphere in Celsius degrees (default: 10). + Temperature of the atmosphere in Celsius degrees (default: 10). Overwritten if spectrum_filename is given. pwv_grid: list List of 3 numbers for the PWV quantity: min, max, number of simulations (default: [0, 10, 10]). ozone_grid: list @@ -168,14 +171,16 @@ def __init__(self, image_filename="", filename="", airmass=1., pressure=800., te Examples -------- - >>> a = AtmosphereGrid(filename='./tests/data/reduc_20170530_134_atmsim.fits') + >>> a = AtmosphereGrid(atmgrid_filename='./tests/data/reduc_20170530_134_atmsim.fits') >>> a.image_filename.split('/')[-1] 'reduc_20170530_134_spectrum.fits' """ Atmosphere.__init__(self, airmass, pressure, temperature) self.my_logger = set_logger(self.__class__.__name__) self.image_filename = image_filename - self.filename = filename + if spectrum_filename != "": + self.image_filename = spectrum_filename + self.filename = atmgrid_filename # ------------------------------------------------------------------------ # Definition of data format for the atmospheric grid # ----------------------------------------------------------------------------- @@ -206,8 +211,13 @@ def __init__(self, image_filename="", filename="", airmass=1., pressure=800., te self.model = None self.header = fits.Header() - if filename != "": - self.load_file(filename) + if atmgrid_filename != "": + self.load_file(atmgrid_filename) + if spectrum_filename != "": + hdr = fits.getheader(spectrum_filename) + self.pressure = hdr["OUTPRESS"] + self.temperature = hdr["OUTTEMP"] + self.airmass = hdr["AIRMASS"] def set_grid(self, pwv_grid=[0, 10, 10], ozone_grid=[100, 700, 7], aerosol_grid=[0, 0.1, 10]): """Set the size of the simulation grid self.atmgrid before compute it. @@ -347,7 +357,7 @@ def plot_transmission_image(self): Examples -------- - >>> a = AtmosphereGrid(filename='tests/data/reduc_20170530_134_atmsim.fits') + >>> a = AtmosphereGrid(atmgrid_filename='tests/data/reduc_20170530_134_atmsim.fits') >>> a.plot_transmission_image() .. plot:: @@ -550,7 +560,7 @@ def simulate(self, ozone, pwv, aerosols): >>> from spectractor import parameters >>> import numpy as np >>> import matplotlib.pyplot as plt - >>> a = AtmosphereGrid(filename='tests/data/reduc_20170530_134_atmsim.fits') + >>> a = AtmosphereGrid(atmgrid_filename='tests/data/reduc_20170530_134_atmsim.fits') >>> lambdas = np.arange(200, 1200) >>> fig = plt.figure() >>> for pwv in np.arange(5): diff --git a/spectractor/simulation/simulator.py b/spectractor/simulation/simulator.py index 42f9a0e11..d861b4e9f 100644 --- a/spectractor/simulation/simulator.py +++ b/spectractor/simulation/simulator.py @@ -131,7 +131,7 @@ def simulate(self, A1=1.0, A2=0., ozone=300, pwv=5, aerosols=0.05, reso=0., Examples -------- >>> spectrum, telescope, disperser, target = SimulatorInit("./tests/data/reduc_20170530_134_spectrum.fits") - >>> atmosphere = AtmosphereGrid(filename="./tests/data/reduc_20170530_134_atmsim.fits") + >>> atmosphere = AtmosphereGrid(atmgrid_filename="./tests/data/reduc_20170530_134_atmsim.fits") >>> sim = SpectrumSimulation(spectrum, atmosphere, telescope, disperser, fast_sim=True) >>> lambdas, model, model_err = sim.simulate(A1=1, A2=1, ozone=300, pwv=5, aerosols=0.05, reso=0., ... D=parameters.DISTANCE2CCD, shift_x=0., B=0.) @@ -379,7 +379,7 @@ def simulate(self, A1=1.0, A2=0., ozone=300, pwv=5, aerosols=0.05, D=parameters. ------- >>> spectrum, telescope, disperser, target = SimulatorInit('./tests/data/reduc_20170530_134_spectrum.fits') - >>> atmosphere = AtmosphereGrid(filename="./tests/data/reduc_20170530_134_atmsim.fits") + >>> atmosphere = AtmosphereGrid(atmgrid_filename="./tests/data/reduc_20170530_134_atmsim.fits") >>> psf_poly_params = spectrum.chromatic_psf.from_table_to_poly_params() >>> sim = SpectrogramModel(spectrum, atmosphere, telescope, disperser, with_background=True, fast_sim=True) >>> lambdas, model, model_err = sim.simulate(A2=1, angle=-1.5, psf_poly_params=psf_poly_params) diff --git a/tests/test_simulator.py b/tests/test_simulator.py index 051ab0ecb..12df18d2b 100644 --- a/tests/test_simulator.py +++ b/tests/test_simulator.py @@ -42,7 +42,7 @@ def test_atmosphere(): a.plot_transmission() a.plot_transmission_image() - a = AtmosphereGrid(filename='tests/data/reduc_20170530_134_atmsim.fits') + a = AtmosphereGrid(atmgrid_filename='tests/data/reduc_20170530_134_atmsim.fits') lambdas = np.arange(200, 1200) transmission = a.simulate(ozone=400, pwv=5, aerosols=0.05) assert np.max(transmission(lambdas)) < 1 and np.min(transmission(lambdas)) >= 0 From f9c568c3b3aef85e5b0e1b6c13b46a2de5d50783 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Wed, 2 Nov 2022 17:33:51 +0100 Subject: [PATCH 03/15] add filter label in output spectrum header --- spectractor/extractor/spectrum.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spectractor/extractor/spectrum.py b/spectractor/extractor/spectrum.py index ef8b1dc20..6e7f10cde 100644 --- a/spectractor/extractor/spectrum.py +++ b/spectractor/extractor/spectrum.py @@ -34,6 +34,7 @@ 'humidity': 'OUTHUM', 'lambda_ref': 'LBDA_REF', 'parallactic_angle': 'PARANGLE', + 'filter_label': 'FILTER' } From 9d0767e2bff094127b048b9b6c8b10fcb891b583 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Wed, 2 Nov 2022 19:02:45 +0100 Subject: [PATCH 04/15] new arguments for atmosphere grid --- spectractor/fit/fit_spectrogram.py | 2 +- spectractor/fit/fit_spectrum.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/spectractor/fit/fit_spectrogram.py b/spectractor/fit/fit_spectrogram.py index bb1d3a40b..214b5e282 100644 --- a/spectractor/fit/fit_spectrogram.py +++ b/spectractor/fit/fit_spectrogram.py @@ -75,7 +75,7 @@ def __init__(self, file_name, atmgrid_file_name="", nwalkers=18, nsteps=1000, bu self.atmosphere = Atmosphere(self.airmass, self.pressure, self.temperature) else: self.use_grid = True - self.atmosphere = AtmosphereGrid(file_name, atmgrid_file_name) + self.atmosphere = AtmosphereGrid(spectrum_filename=file_name, atmgrid_filename=atmgrid_file_name) if parameters.VERBOSE: self.my_logger.info(f'\n\tUse atmospheric grid models from file {atmgrid_file_name}. ') self.crop_spectrogram() diff --git a/spectractor/fit/fit_spectrum.py b/spectractor/fit/fit_spectrum.py index 8e980a887..5adceb254 100644 --- a/spectractor/fit/fit_spectrum.py +++ b/spectractor/fit/fit_spectrum.py @@ -70,7 +70,7 @@ def __init__(self, file_name, atmgrid_file_name="", nwalkers=18, nsteps=1000, bu self.atmosphere = Atmosphere(self.airmass, self.pressure, self.temperature) else: self.use_grid = True - self.atmosphere = AtmosphereGrid(file_name, atmgrid_file_name) + self.atmosphere = AtmosphereGrid(spectrum_filename=file_name, atmgrid_filename=atmgrid_file_name) if parameters.VERBOSE: self.my_logger.info(f'\n\tUse atmospheric grid models from file {atmgrid_file_name}. ') self.lambdas = self.spectrum.lambdas From fd3d0b0460981a494af8f9ece72f60e0a974592c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Thu, 3 Nov 2022 09:10:59 +0100 Subject: [PATCH 05/15] PSF_REG default value closer to what is observed in auxtel data --- config/auxtel.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/auxtel.ini b/config/auxtel.ini index 6e9dc2b15..42433164a 100644 --- a/config/auxtel.ini +++ b/config/auxtel.ini @@ -110,7 +110,7 @@ PSF_TYPE = MoffatGauss # the order of the polynomials to model wavelength dependence of the PSF shape parameters PSF_POLY_ORDER = 2 # regularisation parameter for the chisq minimisation to extract the spectrum -PSF_FIT_REG_PARAM = 1 +PSF_FIT_REG_PARAM = 0.1 # step size in pixels for the first transverse PSF1D fit PSF_PIXEL_STEP_TRANSVERSE_FIT = 50 # PSF is not evaluated outside a region larger than max(PIXWIDTH_SIGNAL, PSF_FWHM_CLIP*fwhm) pixels From dfccb8d8983d2a7934011f7a2dd39cdfdb3c66e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Thu, 3 Nov 2022 09:23:07 +0100 Subject: [PATCH 06/15] cleaning --- runFitter.py | 2 +- spectractor/extractor/chromaticpsf.py | 3 +- spectractor/extractor/extractor.py | 158 ++++---------------------- 3 files changed, 26 insertions(+), 137 deletions(-) diff --git a/runFitter.py b/runFitter.py index 7dc8d73a6..c6fb1706b 100644 --- a/runFitter.py +++ b/runFitter.py @@ -35,7 +35,7 @@ atmgrid_filename = file_name.replace('sim', 'reduc').replace('spectrum', 'atmsim') w = SpectrumFitWorkspace(file_name, atmgrid_file_name=atmgrid_filename, nsteps=1000, burnin=200, nbins=10, verbose=1, plot=True, live_fit=False) - #run_spectrum_minimisation(w, method="newton") + run_spectrum_minimisation(w, method="newton") w = SpectrogramFitWorkspace(file_name, atmgrid_file_name=atmgrid_filename, nsteps=2000, burnin=1000, nbins=10, verbose=1, plot=True, live_fit=False) run_spectrogram_minimisation(w, method="newton") diff --git a/spectractor/extractor/chromaticpsf.py b/spectractor/extractor/chromaticpsf.py index c25db41f7..3ea2e2221 100644 --- a/spectractor/extractor/chromaticpsf.py +++ b/spectractor/extractor/chromaticpsf.py @@ -771,11 +771,10 @@ def get_algebraic_distance_along_dispersion_axis(self, shift_x=0, shift_y=0): def update(self, psf_poly_params, x0, y0, angle, plot=False): profile_params = self.from_poly_params_to_profile_params(psf_poly_params, apply_bounds=True) self.fill_table_with_profile_params(profile_params) - Dx = np.arange(self.Nx) - x0 #self.spectrogram_x0 - shift_x # distance in (x,y) spectrogram frame for column x + Dx = np.arange(self.Nx) - x0 # distance in (x,y) spectrogram frame for column x self.table["Dx"] = Dx self.table['Dy_disp_axis'] = np.tan(angle * np.pi / 180) * self.table['Dx'] self.table['Dy'] = np.copy(self.table['y_c']) - y0 - # self.profile_params = self.from_table_to_profile_params() if plot: self.plot_summary() return profile_params diff --git a/spectractor/extractor/extractor.py b/spectractor/extractor/extractor.py index 7e0c757d2..fb9271df8 100644 --- a/spectractor/extractor/extractor.py +++ b/spectractor/extractor/extractor.py @@ -3,7 +3,6 @@ import numpy as np import matplotlib.pyplot as plt from matplotlib import cm -from scipy.interpolate import interp1d from scipy import sparse import time @@ -117,11 +116,12 @@ def __init__(self, spectrum, amplitude_priors_method="noprior", nwalkers=18, nst self.psf_params_start_index = self.p.size self.p = np.concatenate([self.p, self.psf_poly_params, self.psf_poly_params]) self.input_labels = ["A2", r"D_CCD [mm]", r"shift_x [pix]", r"shift_y [pix]", - r"angle [deg]", "B", "R", "P [hPa]", "T [Celsius]", "z"] + list(self.psf_poly_params_labels) * 2 + r"angle [deg]", "B", "R", "P [hPa]", "T [Celsius]", "z"] + \ + list(self.psf_poly_params_labels) + [label+"_2" for label in self.psf_poly_params_labels] self.axis_names = ["$A_2$", r"$D_{CCD}$ [mm]", r"$\delta_{\mathrm{x}}^(\mathrm{fit})$ [pix]", r"$\delta_{\mathrm{y}}^(\mathrm{fit})$ [pix]", - r"$\alpha$ [deg]", "$B$", "R", r"$P_{\mathrm{atm}}$ [hPa]", r"$T_{\mathrm{atm}}$ [Celcius]", "$z$"]\ - + list(self.psf_poly_params_names) * 2 + r"$\alpha$ [deg]", "$B$", "R", r"$P_{\mathrm{atm}}$ [hPa]", r"$T_{\mathrm{atm}}$ [Celcius]", + "$z$"] + list(self.psf_poly_params_names) + [label+"_2" for label in self.psf_poly_params_names] bounds_D = (self.D - 3 * parameters.DISTANCE2CCD_ERR, self.D + 3 * parameters.DISTANCE2CCD_ERR) self.bounds = np.concatenate([np.array([(0, 2 / parameters.GRATING_ORDER_2OVER1), bounds_D, (-parameters.PIXSHIFT_PRIOR, parameters.PIXSHIFT_PRIOR), @@ -229,12 +229,12 @@ def __init__(self, spectrum, amplitude_priors_method="noprior", nwalkers=18, nst self.Q_dot_A0 = self.Q @ self.amplitude_priors # profile params saved to be plotted - self.profile_params = None - self.profile_params_order2 = None - self.D2CCD = parameters.DISTANCE2CCD - self.Dy_disp_axis = None - self.dx0 = 0 - self.dy0 = 0 + # self.profile_params = None + # self.profile_params_order2 = None + # self.D2CCD = parameters.DISTANCE2CCD + # self.Dy_disp_axis = None + # self.dx0 = 0 + # self.dy0 = 0 def set_y_c(self): for k, par in enumerate(self.input_labels): @@ -414,118 +414,32 @@ def simulate(self, *params): parameters.OBS_CAMERA_ROTATION = rot W_dot_data = self.W * (self.data + (1 - B) * self.bgd_flat) - profile_params = self.spectrum.chromatic_psf.from_poly_params_to_profile_params(poly_params_order1, apply_bounds=True) - profile_params[:, 0] = 1 - profile_params[:, 1] = np.arange(self.Nx) - self.spectrum.chromatic_psf.fill_table_with_profile_params(profile_params) - # # Distance in x and y with respect to the true order 0 position at lambda_ref - Dx = np.arange(self.Nx) - self.spectrum.spectrogram_x0 - dx0 # distance in (x,y) spectrogram frame for column x - Dy_disp_axis = np.tan(angle * np.pi / 180) * Dx # disp axis height in spectrogram frame for x - distance = np.sign(Dx) * np.sqrt(Dx * Dx + Dy_disp_axis * Dy_disp_axis) # algebraic distance along dispersion axis - - # First guess of wavelengths - self.spectrum.disperser.D = np.copy(D2CCD) - self.lambdas = self.spectrum.disperser.grating_pixel_to_lambda(distance, - self.spectrum.x0 + np.asarray([dx0, dy0]), - order=self.spectrum.order) - lambdas_order2 = self.spectrum.disperser.grating_pixel_to_lambda(distance, - self.spectrum.x0 + np.asarray([dx0, dy0]), - order=self.spectrum.order+np.sign(self.spectrum.order)) - - # Evaluate ADR - adr_x = np.zeros_like(Dx) - adr_y = np.zeros_like(Dy_disp_axis) - for k in range(3): - adr_ra, adr_dec = adr_calib(self.lambdas, self.spectrum.adr_params, parameters.OBS_LATITUDE, - lambda_ref=self.spectrum.lambda_ref) - adr_x, adr_y = flip_and_rotate_adr_to_image_xy_coordinates(adr_ra, adr_dec, dispersion_axis_angle=0) - adr_u, adr_v = flip_and_rotate_adr_to_image_xy_coordinates(adr_ra, adr_dec, dispersion_axis_angle=angle) - - # Evaluate ADR for order 2 - adr_ra, adr_dec = adr_calib(lambdas_order2, self.spectrum.adr_params, parameters.OBS_LATITUDE, - lambda_ref=self.spectrum.lambda_ref) - # adr_x_2, adr_y_2 = flip_and_rotate_adr_to_image_xy_coordinates(adr_ra, adr_dec, dispersion_axis_angle=0) - adr_u_2, adr_v_2 = flip_and_rotate_adr_to_image_xy_coordinates(adr_ra, adr_dec, dispersion_axis_angle=angle) - - # Compute lambdas at pixel column x - self.lambdas = self.spectrum.disperser.grating_pixel_to_lambda(distance - adr_u, - self.spectrum.x0 + np.asarray([dx0, dy0]), - order=self.spectrum.order) - lambdas_order2 = self.spectrum.disperser.grating_pixel_to_lambda(distance - adr_u_2, - self.spectrum.x0 + np.asarray([dx0, dy0]), - order=self.spectrum.order+np.sign(self.spectrum.order)) - print("bef",self.lambdas[-5:], lambdas_order2[-5:]) - self.lambdas, lambdas_order2, dispersion_law, dispersion_law_order2 = self.spectrum.compute_dispersion_in_spectrogram(D2CCD, dx0, dy0, angle, with_adr=True, niter=3) - print("aft",self.lambdas[-5:], lambdas_order2[-5:]) + # Evaluate PSF profile + profile_params = self.spectrum.chromatic_psf.update(poly_params_order1, self.spectrum.spectrogram_x0 + dx0, + self.spectrum.spectrogram_y0 + dy0, angle, plot=False) + # Evaluate ADR and compute wavelength arrays + self.lambdas, lambdas_order2, dispersion_law, dispersion_law_order2 = self.spectrum.compute_dispersion_in_spectrogram(D2CCD, dx0, dy0, angle, with_adr=True, niter=5) # Fill spectrogram trace as a function of the pixel column x - # self.spectrum.chromatic_psf.table["Dy_disp_axis"] = Dy_disp_axis - # self.spectrum.chromatic_psf.table["Dx"] = Dx - befx = Dx + self.spectrum.spectrogram_x0 + adr_x + dx0 - befy = Dy_disp_axis - self.bgd_width + adr_y + dy0 - print("dispx bef", befx[-5:]) - print("dispy bef", befy[-5:]) - profile_params[:, 1] = dispersion_law.real + self.spectrum.spectrogram_x0 #+ adr_x + dx0 - profile_params[:, 2] += dispersion_law.imag - self.bgd_width # Dy_disp_axis + adr_y + dy0 - print("dispx aft", dispersion_law.real[-5:] + self.spectrum.spectrogram_x0) - print("dispy aft", dispersion_law.imag[-5:] - self.bgd_width) + profile_params[:, 0] = 1 + profile_params[:, 1] = dispersion_law.real + self.spectrum.spectrogram_x0 + profile_params[:, 2] += dispersion_law.imag - self.bgd_width # Prepare order 2 profile params indexed by the wavelength associated to x profile_params_order2 = self.spectrum.chromatic_psf.from_poly_params_to_profile_params(poly_params_order2, apply_bounds=True) # Second diffraction order amplitude is the first order amplitude multiplied by ratio 2/1 # Ratio 2/1 is in flam/flam but no need to convert in ADU/ADU because lambda*dlambda is the same for both orders profile_params_order2[:, 0] = self.spectrum.disperser.ratio_order_2over1(self.lambdas) - - # For each A(lambda)=A_x, affect an order 2 PSF with correct position and - # same PSF as for the order 1 but at the same position - distance_order2 = self.spectrum.disperser.grating_lambda_to_pixel(self.lambdas, - self.spectrum.x0 + np.asarray([dx0, dy0]), - order=self.spectrum.order+np.sign(self.spectrum.order)) - # for k in range(3, profile_params.shape[1]): - # # profile_params_order2[:, k] = interp1d(self.lambdas_order2, profile_params_order2[:, k], - # # kind="cubic", fill_value="extrapolate")(self.lambdas) - # profile_params_order2[:, k] = interp1d(distance, profile_params_order2[:, k], - # kind="cubic", fill_value="extrapolate")(distance_order2) - Dx_order2 = distance_order2 * np.cos(angle * np.pi / 180) - Dy_disp_axis_order2 = distance_order2 * np.sin(angle * np.pi / 180) - befx = Dx_order2 + self.spectrum.spectrogram_x0 + adr_x + dx0 - befy = Dy_disp_axis_order2 - self.bgd_width + adr_y + dy0 - print("dispx bef", befx[-5:]) - print("dispy bef", befy[-5:]) - profile_params_order2[:, 1] = dispersion_law_order2.real + self.spectrum.spectrogram_x0 # Dx_order2 + adr_x + dx0 - profile_params_order2[:, 2] += dispersion_law_order2.imag - self.bgd_width # Dy_disp_axis_order2 + (0*self.spectrum.spectrogram_y0 + adr_y + dy0) - self.bgd_width - print("dispx aft", dispersion_law_order2.real[-5:] + self.spectrum.spectrogram_x0) - print("dispy aft", dispersion_law_order2.imag[-5:] - self.bgd_width) - - if parameters.DEBUG and False: - plt.figure(figsize=(18,4)) - plt.imshow(self.data.reshape((self.Ny, self.Nx)), origin="lower") - plt.scatter(profile_params[:, 1], profile_params[:, 2], label="profile", - cmap=from_lambda_to_colormap(self.lambdas), c=self.lambdas) - plt.scatter(profile_params_order2[:, 1], profile_params_order2[:, 2], label="order 2", - cmap=from_lambda_to_colormap(self.lambdas), c=self.lambdas) - plt.plot(profile_params[:, 1], profile_params[:, 2], label="profile") - plt.plot(profile_params[:, 1], self.spectrum.chromatic_psf["Dy_disp_axis"] + self.spectrum.spectrogram_y0 + dy0 - self.bgd_width, 'k-', - label="disp_axis") - plt.plot(self.spectrum.chromatic_psf.table['Dx'] + self.spectrum.spectrogram_x0 + dx0, - self.spectrum.chromatic_psf.table['Dy'] + self.spectrum.spectrogram_y0 + dy0 - self.bgd_width, - label="y_c") - plt.legend() - plt.title(f"D_CCD={D2CCD:.2f}, dx0={dx0:.2g}, dy0={dy0:.2g}") - plt.xlim((0, self.Nx)) - plt.ylim((0, self.Ny)) - plt.grid() - plt.gca().set_aspect("auto") - plt.suptitle("simulate") - plt.show() + profile_params_order2[:, 1] = dispersion_law_order2.real + self.spectrum.spectrogram_x0 + profile_params_order2[:, 2] += dispersion_law_order2.imag - self.bgd_width # save for plotting - self.profile_params = profile_params - self.profile_params_order2 = profile_params_order2 - self.dx0 = dx0 - self.dy0 = dy0 + # self.profile_params = profile_params + # self.profile_params_order2 = profile_params_order2 + # self.dx0 = dx0 + # self.dy0 = dy0 # Matrix filling # if self.psf_cube is None or not self.fix_psf_cube: # slower @@ -808,30 +722,6 @@ def adjust_spectrogram_position_parameters(self): self.set_mask(params=self.p) # self.set_y_c() - def plot_fitted_parameters(self,title="output from simulate"): - - plt.figure(figsize=(18,6)) - plt.imshow(self.data.reshape((self.Ny, self.Nx)), origin="lower") - plt.scatter(self.profile_params[:, 1], self.profile_params[:, 2], label="profile", - cmap=from_lambda_to_colormap(self.lambdas), s=30, c=self.lambdas) - plt.scatter(self.profile_params_order2[:, 1], self.profile_params_order2[:, 2], label="order 2", - cmap=from_lambda_to_colormap(self.lambdas), s=30, c=self.lambdas) - plt.plot(self.profile_params[:, 1], self.profile_params[:, 2], label="profile",lw=3) - plt.plot(self.profile_params[:, 1], self.spectrum.chromatic_psf.table['Dy_disp_axis'] + self.spectrum.spectrogram_y0 + self.dy0 - self.bgd_width, 'k-', - label="disp_axis",lw=3) - plt.plot(self.spectrum.chromatic_psf.table['Dx'] + self.spectrum.spectrogram_x0 + self.dx0, - self.spectrum.chromatic_psf.table['Dy'] + self.spectrum.spectrogram_y0 + self.dy0 - self.bgd_width, - label="y_c",lw=3) - plt.legend() - plt.title(f"D_CCD={self.D2CCD:.2f}, dx0={self.dx0:.2g}, dy0={self.dy0:.2g}") - plt.xlim((0, self.Nx)) - plt.ylim((0, self.Ny)) - plt.grid() - plt.gca().set_aspect("auto") - plt.suptitle(title) - plt.tight_layout() - plt.show() - def run_ffm_minimisation(w, method="newton", niter=2): """Interface function to fit spectrogram simulation parameters to data. From 6619af22910f3c97225958fa49abd647c686a30b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Thu, 3 Nov 2022 09:25:07 +0100 Subject: [PATCH 07/15] cleaning --- spectractor/extractor/extractor.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/spectractor/extractor/extractor.py b/spectractor/extractor/extractor.py index fb9271df8..c90184fc4 100644 --- a/spectractor/extractor/extractor.py +++ b/spectractor/extractor/extractor.py @@ -435,12 +435,6 @@ def simulate(self, *params): profile_params_order2[:, 1] = dispersion_law_order2.real + self.spectrum.spectrogram_x0 profile_params_order2[:, 2] += dispersion_law_order2.imag - self.bgd_width - # save for plotting - # self.profile_params = profile_params - # self.profile_params_order2 = profile_params_order2 - # self.dx0 = dx0 - # self.dy0 = dy0 - # Matrix filling # if self.psf_cube is None or not self.fix_psf_cube: # slower psf_cube = self.spectrum.chromatic_psf.build_psf_cube(self.pixels, profile_params, From 2a5cbf444ad7a520c8589ddaf8e1c2e373715ba9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Thu, 3 Nov 2022 09:25:51 +0100 Subject: [PATCH 08/15] cleaning --- spectractor/extractor/extractor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spectractor/extractor/extractor.py b/spectractor/extractor/extractor.py index c90184fc4..e1f23f5eb 100644 --- a/spectractor/extractor/extractor.py +++ b/spectractor/extractor/extractor.py @@ -141,7 +141,7 @@ def __init__(self, spectrum, amplitude_priors_method="noprior", nwalkers=18, nst # with respect to the true spectrum injected in the simulation # A2 is free only if spectrogram is a simulation or if the order 2/1 ratio is not known and flat self.fixed[0] = (not self.spectrum.disperser.flat_ratio_order_2over1) and (not ("A2_T" in self.spectrum.header)) - self.fixed[1] = True # D2CCD: spectrogram can not tell something on this parameter: rely on calibrate_pectrum + self.fixed[1] = True # D2CCD: spectrogram can not tell something on this parameter: rely on calibrate_spectrum self.fixed[2] = True # delta x: if False, extracted spectrum is biased compared with truth self.fixed[3] = True # delta y self.fixed[4] = True # angle From 1311fc309b1537972ed7296d9d318e4146b7b5a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Thu, 3 Nov 2022 09:36:00 +0100 Subject: [PATCH 09/15] cleaning --- spectractor/extractor/images.py | 66 ++++++++------------------------- 1 file changed, 15 insertions(+), 51 deletions(-) diff --git a/spectractor/extractor/images.py b/spectractor/extractor/images.py index a7856251b..b77b46e9f 100644 --- a/spectractor/extractor/images.py +++ b/spectractor/extractor/images.py @@ -205,40 +205,9 @@ def rebin(self): >>> im.target_guess array([405., 295.]) """ - - old_data=np.copy(self.data) - old_shape=old_data.shape - new_shape = np.asarray(self.data.shape) // parameters.CCD_REBIN self.data = rebin(self.data, new_shape) self.stat_errors = np.sqrt(rebin(self.stat_errors ** 2, new_shape)) - new_shape = self.data.shape - - if parameters.DEBUG and parameters.DISPLAY: - fig, ax = plt.subplots(1, 3, figsize=(18, 5)) - - im0 = ax[0].imshow(old_data, origin='lower', norm=LogNorm()) - fig.colorbar(im0, ax=ax[0],orientation = 'horizontal') - ax[0].grid() - ax[0].set_title(f"original data : {old_shape}") - - - im1=ax[1].imshow(self.data,origin='lower',norm=LogNorm()) - fig.colorbar(im1, ax=ax[1],orientation = 'horizontal') - ax[1].grid() - ax[1].set_title(f"rebinned data : {new_shape}") - - im2=ax[2].imshow(self.stat_errors,origin='lower',norm=LogNorm()) - fig.colorbar(im2, ax=ax[2],orientation = 'horizontal') - ax[2].grid() - ax[2].set_title("rebinned stat errors") - - if parameters.LSST_SAVEFIGPATH: # pragma: no cover - plt.gcf().savefig(os.path.join(parameters.LSST_SAVEFIGPATH, 'rebinned_image.pdf'), transparent=True) - if parameters.DISPLAY: # pragma: no cover - plt.show() - - if self.target_guess is not None: self.target_guess = np.asarray(self.target_guess) / parameters.CCD_REBIN @@ -554,6 +523,7 @@ def plot_image(self, ax=None, scale="lin", title="", units="", plot_stats=False, if parameters.PdfPages: parameters.PdfPages.savefig() + def load_CTIO_image(image): """Specific routine to load CTIO fits files and load their data and properties for Spectractor. @@ -835,8 +805,6 @@ def find_target(image, guess=None, rotated=False, widths=[parameters.XWINDOW, pa else: my_logger.info(f"\n\tNo WCS {wcs_file_name} available, use 2D fit to find target pixel position.") - - if parameters.SPECTRACTOR_FIT_TARGET_CENTROID == "fit" or rotated: if target_pixcoords[0] == -1 and target_pixcoords[1] == -1: if guess is None: @@ -853,23 +821,19 @@ def find_target(image, guess=None, rotated=False, widths=[parameters.XWINDOW, pa widths=(Dx, Dy)) sub_image_x0, sub_image_y0 = x0, y0 - if parameters.DEBUG: - - if parameters.DISPLAY: - - fig, ax = plt.subplots(1, 2, figsize=(12, 5)) - im0 = ax[0].imshow(sub_image_subtracted,origin="lower",norm=LogNorm()) - fig.colorbar(im0, ax=ax[0]) - ax[0].set_title("sub_image_subtracted") - im1=ax[1].imshow(sub_errors, origin="lower", norm=LogNorm()) - ax[1].set_title("sub_image_errors") - fig.colorbar(im1, ax=ax[1]) - - if parameters.LSST_SAVEFIGPATH: # pragma: no cover - plt.gcf().savefig(os.path.join(parameters.LSST_SAVEFIGPATH, 'sub_image_subtracted.pdf'), - transparent=True) - if parameters.DISPLAY: # pragma: no cover - plt.show() + if parameters.DEBUG and parameters.DISPLAY: + fig, ax = plt.subplots(1, 2, figsize=(12, 5)) + im0 = ax[0].imshow(sub_image_subtracted, origin="lower", norm=LogNorm()) + fig.colorbar(im0, ax=ax[0]) + ax[0].set_title("sub_image_subtracted") + im1 = ax[1].imshow(sub_errors, origin="lower", norm=LogNorm()) + ax[1].set_title("sub_image_errors") + fig.colorbar(im1, ax=ax[1]) + if parameters.LSST_SAVEFIGPATH: # pragma: no cover + plt.gcf().savefig(os.path.join(parameters.LSST_SAVEFIGPATH, 'sub_image_subtracted.pdf'), + transparent=True) + if parameters.DISPLAY: # pragma: no cover + plt.show() for i in range(niter): # find the target @@ -991,7 +955,7 @@ def find_target_init(image, guess, rotated=False, widths=[parameters.XWINDOW, pa sub_image_subtracted = sub_image - bkgd_2D(X, Y) # SDC : very important clipping negative signal, avoiding crash later - sub_image_subtracted = np.where(sub_image_subtracted<0,0,sub_image_subtracted) + sub_image_subtracted = np.where(sub_image_subtracted < 0, 0, sub_image_subtracted) saturated_pixels = np.where(sub_image >= image.saturation) if len(saturated_pixels[0]) > 0: From 4197ae8ac7a16d9e153fd1d9f4051d6bf3ebfdc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Thu, 3 Nov 2022 09:37:10 +0100 Subject: [PATCH 10/15] Add camera_angle attribute to Spectrum --- spectractor/extractor/spectrum.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/spectractor/extractor/spectrum.py b/spectractor/extractor/spectrum.py index 6e7f10cde..d0e1e00d8 100644 --- a/spectractor/extractor/spectrum.py +++ b/spectractor/extractor/spectrum.py @@ -9,7 +9,7 @@ import astropy from spectractor import parameters -from spectractor.config import set_logger, load_config, apply_rebinning_to_parameters +from spectractor.config import set_logger, load_config from spectractor.extractor.dispersers import Hologram from spectractor.extractor.targets import load_target from spectractor.tools import (ensure_dir, load_fits, plot_image_simple, @@ -34,7 +34,8 @@ 'humidity': 'OUTHUM', 'lambda_ref': 'LBDA_REF', 'parallactic_angle': 'PARANGLE', - 'filter_label': 'FILTER' + 'filter_label': 'FILTER', + 'camera_angle': 'CAM_ROT' } @@ -87,6 +88,8 @@ class Spectrum: Dispersion axis angle in the image in degrees, positive if anticlockwise. parallactic_angle: float Parallactic angle in degrees. + camera_angle: float + The North-West axe angle with respect to the camera horizontal axis in degrees. lines: Lines Lines instance that contains data on the emission or absorption lines to be searched and fitted in the spectrum. header: Fits.Header @@ -202,6 +205,7 @@ def __init__(self, file_name="", image=None, order=1, target=None, config="", fa self.chromatic_psf = ChromaticPSF(self.psf, Nx=1, Ny=1, deg=1, saturation=1) self.rotation_angle = 0 self.parallactic_angle = None + self.camera_angle = 0 self.spectrogram = None self.spectrogram_bgd = None self.spectrogram_bgd_rms = None @@ -248,6 +252,7 @@ def __init__(self, file_name="", image=None, order=1, target=None, config="", fa self.units = image.units self.gain = image.gain self.rotation_angle = image.rotation_angle + self.camera_angle = parameters.OBS_CAMERA_ROTATION self.my_logger.info('\n\tSpectrum info copied from image') self.dec = image.dec self.hour_angle = image.hour_angle From a3277ebf31e39edaafd46f3e1465be042c6bb55f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Thu, 3 Nov 2022 09:43:37 +0100 Subject: [PATCH 11/15] cleaning --- spectractor/extractor/extractor.py | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/spectractor/extractor/extractor.py b/spectractor/extractor/extractor.py index e1f23f5eb..946058ffb 100644 --- a/spectractor/extractor/extractor.py +++ b/spectractor/extractor/extractor.py @@ -14,7 +14,6 @@ from spectractor.extractor.chromaticpsf import ChromaticPSF from spectractor.extractor.psf import load_PSF from spectractor.tools import ensure_dir, plot_image_simple, from_lambda_to_colormap, plot_spectrum_simple -from spectractor.simulation.adr import adr_calib, flip_and_rotate_adr_to_image_xy_coordinates from spectractor.fit.fitter import run_minimisation, run_minimisation_sigma_clipping, RegFitWorkspace, FitWorkspace @@ -24,7 +23,7 @@ def dumpParameters(): print(item, getattr(parameters, item)) -def dumpfitparameters(w,thelogguer): +def dumpfitparameters(w, thelogger): N1 = len(w.input_labels) N2 = len(w.p) assert N1 == N2 @@ -33,13 +32,13 @@ def dumpfitparameters(w,thelogguer): for idx in range(N1): tag = w.input_labels[idx] val = w.p[idx] - fixed=w.fixed[idx] - b1= w.bounds[idx][0] - b2= w.bounds[idx][1] - line = "- fit param #{} :: {} = {} \t fixed = {} \t bounds {:.3f} - {:.3f}".format(idx,tag,val,fixed,b1,b2) + fixed = w.fixed[idx] + b1 = w.bounds[idx][0] + b2 = w.bounds[idx][1] + line = "- fit param #{} :: {} = {} \t fixed = {} \t bounds {:.3f} - {:.3f}".format(idx, tag, val, fixed, b1, b2) list_of_strings.append(line) txt = "\n".join(list_of_strings) - thelogguer.info(txt) + thelogger.info(txt) class FullForwardModelFitWorkspace(FitWorkspace): @@ -259,7 +258,6 @@ def set_mask(self, params=None): Examples -------- >>> spec = Spectrum("./tests/data/reduc_20170530_134_spectrum.fits", config="./config/ctio.ini") - >>> spec = Spectrum("outputs/exposure_2022031600330_postisrccd_spectrum.fits", config="./config/auxtel.ini") >>> w = FullForwardModelFitWorkspace(spectrum=spec, amplitude_priors_method="fixed", verbose=True) >>> _ = w.simulate(*w.p) >>> w.plot_fit() @@ -769,14 +767,13 @@ def run_ffm_minimisation(w, method="newton", niter=2): if parameters.DEBUG: my_logger.info("\n --- before run_minimisation ---") - dumpfitparameters(w,my_logger) - + dumpfitparameters(w, my_logger) run_minimisation(w, method=method, fix=w.fixed, xtol=1e-4, ftol=100 / (w.data.size - len(w.mask))) if parameters.DEBUG: my_logger.info("\n --- after run_minimisation ---") - dumpfitparameters(w,my_logger) + dumpfitparameters(w, my_logger) if parameters.DEBUG and parameters.DISPLAY: w.plot_fit() @@ -817,7 +814,7 @@ def run_ffm_minimisation(w, method="newton", niter=2): if parameters.DEBUG: my_logger.info("\n --- after run_minimisation_sigma_clipping ---") - dumpfitparameters(w,my_logger) + dumpfitparameters(w, my_logger) if parameters.DEBUG and parameters.DISPLAY: w.plot_fit() @@ -909,7 +906,7 @@ def run_ffm_minimisation(w, method="newton", niter=2): def Spectractor(file_name, output_directory, target_label, guess=None, disperser_label="", config='./config/ctio.ini', - atmospheric_lines=True, line_detection=True): + atmospheric_lines=True): """ Spectractor Main function to extract a spectrum from an image @@ -982,14 +979,14 @@ def Spectractor(file_name, output_directory, target_label, guess=None, disperser my_logger.info(f"\n\tNo guess position of order 0 has been given. Assuming the spectrum to extract comes " f"from the brightest object, guess position is set as {image.target_guess}.") if parameters.DEBUG: - image.plot_image(scale='symlog', title="before rebinning",target_pixcoords=image.target_guess) + image.plot_image(scale='symlog', title="before rebinning", target_pixcoords=image.target_guess) # Use fast mode if parameters.CCD_REBIN > 1: my_logger.info('\n\t ======================= REBIN =============================') image.rebin() if parameters.DEBUG: - image.plot_image(scale='symlog', title="after rebinning ",target_pixcoords=image.target_guess) + image.plot_image(scale='symlog', title="after rebinning ", target_pixcoords=image.target_guess) # Set output path ensure_dir(output_directory) @@ -1096,8 +1093,6 @@ def extract_spectrum_from_image(image, spectrum, signal_width=10, ws=(20, 30), r ws = [signal_width + 20, signal_width + 30] my_logger.info('\n\t ======================= extract_spectrum_from_image =============================') - - my_logger.info( f'\n\tExtracting spectrum from image: spectrum with width 2*{signal_width:.0f} pixels ' f'and background from {ws[0]:.0f} to {ws[1]:.0f} pixels') From e92e35e1a382f43fa98dd02c3de0de4047eb1fa1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Thu, 3 Nov 2022 09:44:58 +0100 Subject: [PATCH 12/15] cleaning --- spectractor/extractor/extractor.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/spectractor/extractor/extractor.py b/spectractor/extractor/extractor.py index 946058ffb..090b7e0f0 100644 --- a/spectractor/extractor/extractor.py +++ b/spectractor/extractor/extractor.py @@ -227,14 +227,6 @@ def __init__(self, spectrum, amplitude_priors_method="noprior", nwalkers=18, nst # self.Q = L.T @ U.T @ U @ L self.Q_dot_A0 = self.Q @ self.amplitude_priors - # profile params saved to be plotted - # self.profile_params = None - # self.profile_params_order2 = None - # self.D2CCD = parameters.DISTANCE2CCD - # self.Dy_disp_axis = None - # self.dx0 = 0 - # self.dy0 = 0 - def set_y_c(self): for k, par in enumerate(self.input_labels): if "y_c" in par: From f5f747dcebd2ac7462ab089fe24233168e9cbbb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Thu, 3 Nov 2022 09:49:51 +0100 Subject: [PATCH 13/15] cleaning --- spectractor/extractor/spectrum.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/spectractor/extractor/spectrum.py b/spectractor/extractor/spectrum.py index d0e1e00d8..a1aecae16 100644 --- a/spectractor/extractor/spectrum.py +++ b/spectractor/extractor/spectrum.py @@ -624,17 +624,10 @@ def load_spectrum(self, input_file_name, spectrogram_file_name_override=None, for attribute, header_key in fits_mappings.items(): if (item := self.header.get(header_key)) is not None: setattr(self, attribute, item) - # print(f'set {attribute} to {item}') else: print(f'Failed to set spectrum attribute {attribute} using header {header_key}') if "CAM_ROT" in self.header: parameters.OBS_CAMERA_ROTATION = float(self.header["CAM_ROT"]) - elif "ROTPA" in self.header: - parameters.OBS_CAMERA_ROTATION = 270 - float(self.header["ROTPA"]) - if parameters.OBS_CAMERA_ROTATION > 360: - parameters.OBS_CAMERA_ROTATION -= 360 - if parameters.OBS_CAMERA_ROTATION < -360: - parameters.OBS_CAMERA_ROTATION += 360 else: self.my_logger.warning("No information about camera rotation in Spectrum header.") From 567974d911e3932674ea5f3d8b58b410b2036c52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Thu, 3 Nov 2022 09:54:01 +0100 Subject: [PATCH 14/15] cleaning --- spectractor/fit/fit_spectrogram.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/spectractor/fit/fit_spectrogram.py b/spectractor/fit/fit_spectrogram.py index 214b5e282..1d60c4704 100644 --- a/spectractor/fit/fit_spectrogram.py +++ b/spectractor/fit/fit_spectrogram.py @@ -6,11 +6,11 @@ import copy from spectractor import parameters -from spectractor.config import set_logger, load_config +from spectractor.config import set_logger from spectractor.tools import plot_image_simple, from_lambda_to_colormap from spectractor.simulation.simulator import SimulatorInit, SpectrogramModel from spectractor.simulation.atmosphere import Atmosphere, AtmosphereGrid -from spectractor.fit.fitter import FitWorkspace, run_minimisation, run_gradient_descent, run_minimisation_sigma_clipping +from spectractor.fit.fitter import FitWorkspace, run_minimisation, run_minimisation_sigma_clipping plot_counter = 0 @@ -109,10 +109,12 @@ def __init__(self, file_name, atmgrid_file_name="", nwalkers=18, nsteps=1000, bu self.psf_params_start_index = self.p.size self.p = np.concatenate([self.p, self.psf_poly_params, np.copy(self.psf_poly_params)]) self.input_labels = ["A1", "A2", "ozone [db]", "PWV [mm]", "VAOD", r"D_CCD [mm]", - r"shift_x [pix]", r"shift_y [pix]", r"angle [deg]", "B"] + list(self.psf_poly_params_labels) * 2 + r"shift_x [pix]", r"shift_y [pix]", r"angle [deg]", "B"] + \ + list(self.psf_poly_params_labels) + [label+"_2" for label in self.psf_poly_params_labels] self.axis_names = ["$A_1$", "$A_2$", "ozone [db]", "PWV [mm]", "VAOD", r"$D_{CCD}$ [mm]", r"$\Delta_{\mathrm{x}}$ [pix]", r"$\Delta_{\mathrm{y}}$ [pix]", - r"$\theta$ [deg]", "$B$"] + list(self.psf_poly_params_names) * 2 + r"$\theta$ [deg]", "$B$"] + \ + list(self.psf_poly_params_names) + [label+"_2" for label in self.psf_poly_params_names] bounds_D = (self.D - 5 * parameters.DISTANCE2CCD_ERR, self.D + 5 * parameters.DISTANCE2CCD_ERR) self.bounds = np.concatenate([np.array([(0, 2), (0, 2/parameters.GRATING_ORDER_2OVER1), (100, 700), (0, 10), (0, 0.1), bounds_D, (-2, 2), (-10, 10), (-90, 90), (0.8, 1.2)]), From ee7e11838bdb7142c7a2b9c69a59cbdc39c98ee9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Neveu?= Date: Thu, 3 Nov 2022 10:06:15 +0100 Subject: [PATCH 15/15] cleaning --- spectractor/simulation/simulator.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/spectractor/simulation/simulator.py b/spectractor/simulation/simulator.py index d861b4e9f..4418a1bb0 100644 --- a/spectractor/simulation/simulator.py +++ b/spectractor/simulation/simulator.py @@ -15,7 +15,6 @@ from spectractor.simulation.throughput import TelescopeTransmission from spectractor.simulation.atmosphere import Atmosphere, AtmosphereGrid import spectractor.parameters as parameters -from spectractor.simulation.adr import adr_calib, flip_and_rotate_adr_to_image_xy_coordinates class SpectrumSimulation(Spectrum): @@ -153,17 +152,6 @@ def simulate(self, A1=1.0, A2=0., ozone=300, pwv=5, aerosols=0.05, reso=0., distance = self.chromatic_psf.get_algebraic_distance_along_dispersion_axis(shift_x=shift_x) lambdas = self.disperser.grating_pixel_to_lambda(distance, x0=new_x0, order=1) lambdas_order2 = self.disperser.grating_pixel_to_lambda(distance, x0=new_x0, order=2) - # lambda_ref = self.lambda_ref - # adr_ra, adr_dec = adr_calib(lambdas, self.adr_params, parameters.OBS_LATITUDE, lambda_ref=lambda_ref) - # adr_u, _ = flip_and_rotate_adr_to_image_xy_coordinates(adr_ra, adr_dec, - # dispersion_axis_angle=self.rotation_angle) - # distance_order1 = distance - adr_u - # adr_ra, adr_dec = adr_calib(lambdas_order2, self.adr_params, parameters.OBS_LATITUDE, lambda_ref=lambda_ref) - # adr_u, _ = flip_and_rotate_adr_to_image_xy_coordinates(adr_ra, adr_dec, - # dispersion_axis_angle=self.rotation_angle) - # distance_order2 = distance - adr_u - # lambdas = self.disperser.grating_pixel_to_lambda(distance_order1, x0=new_x0, order=1) - # lambdas_order2 = self.disperser.grating_pixel_to_lambda(distance_order2, x0=new_x0, order=2) self.lambdas = lambdas self.lambdas_order2 = lambdas_order2 atmospheric_transmission = self.atmosphere.simulate(ozone, pwv, aerosols) @@ -440,18 +428,11 @@ def simulate(self, A1=1.0, A2=0., ozone=300, pwv=5, aerosols=0.05, D=parameters. if self.psf_cube_order2 is None or not self.fix_psf_cube: start = time.time() self.psf_cube_order2 = np.zeros((nlbda2, self.Ny, self.Nx)) - # For each A(lambda)=A_x, affect an order 2 PSF with correct position and - # same PSF as for the order 1 but at the same position profile_params_order2 = self.chromatic_psf.from_poly_params_to_profile_params(psf_poly_params_order2, apply_bounds=True) profile_params_order2[:, 0] = 1 profile_params_order2[:nlbda2, 1] = dispersion_law_order2.real + self.r0.real profile_params_order2[:nlbda2, 2] += dispersion_law_order2.imag - # distance = np.abs(dispersion_law) - # distance_order2 = np.abs(dispersion_law_order2) - # for k in range(3, self.profile_params.shape[1]): - # profile_params_order2[:nlbda2, k] = interp1d(distance, profile_params_order2[:, k], kind="cubic", - # fill_value="extrapolate")(distance_order2) for i in range(0, nlbda2, 1): self.psf_cube_order2[i] = self.psf.evaluate(self.pixels, p=profile_params_order2[i, :]) self.my_logger.debug(f'\n\tAfter psf cube order 2: {time.time() - start}')