Skip to content

Commit

Permalink
use proper calib version in provenance and pathlib
Browse files Browse the repository at this point in the history
  • Loading branch information
@morcuended
morcuended committed Sep 25, 2023
1 parent 7573a1f commit ff686d5
Showing 1 changed file with 112 additions and 104 deletions.
216 changes: 112 additions & 104 deletions osa/provenance/utils.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
"""Utility functions for OSA pipeline provenance."""


import os
from importlib.resources import files
from pathlib import Path

Expand All @@ -27,10 +26,11 @@ def parse_variables(class_instance):
# datasequence.py
# -c sequencer.cfg
# -d 2020_02_18
# --prod-id v0.4.3_v00
# --pedcal-file .../20200218/pro/calibration_filters_52.Run02006.0000.h5
# --drs4-pedestal-file .../20200218/pro/drs4_pedestal.Run02005.0000.h5
# --prod-id v0.4
# --pedcal-file .../20200218/v0.4.3/calibration_filters_52.Run02006.0000.h5
# --drs4-pedestal-file .../20200218/v0.4.3/drs4_pedestal.Run02005.0000.h5
# --time-calib-file .../20191124/pro/time_calibration.Run01625.0000.h5
# --systematic_correction_file .../20200725/pro/ffactor_systematics_20200725.h5
# --drive-file .../DrivePositioning/DrivePosition_20200218.txt
# --run-summary .../monitoring/RunSummary/RunSummary_20200101.ecsv
# 02006.0000
Expand All @@ -39,22 +39,20 @@ def parse_variables(class_instance):
flat_date = date_to_dir(options.date)
configfile_dl1b = cfg.get("lstchain", "dl1b_config")
configfile_dl2 = cfg.get("lstchain", "dl2_config")
raw_dir = cfg.get("LST1", "R0_DIR")
rf_models_directory = cfg.get("lstchain", "RF_MODELS")
dl1_dir = cfg.get("LST1", "DL1_DIR")
dl2_dir = cfg.get("LST1", "DL2_DIR")
calib_dir = cfg.get("LST1", "CALIB_DIR")
pedestal_dir = cfg.get("LST1", "PEDESTAL_DIR")
# summary_dir = cfg.get("LST1", "RUN_SUMMARY_DIR")
# calib_base_dir = cfg.get("LST1", "CALIB_BASE_DIR")
# sys_dir = calib_base_dir / "ffactor_systematics"
raw_dir = Path(cfg.get("LST1", "R0_DIR"))
rf_models_directory = Path(cfg.get("lstchain", "RF_MODELS"))
dl1_dir = Path(cfg.get("LST1", "DL1_DIR"))
dl2_dir = Path(cfg.get("LST1", "DL2_DIR"))
calib_dir = Path(cfg.get("LST1", "CALIB_DIR"))
pedestal_dir = Path(cfg.get("LST1", "PEDESTAL_DIR"))

class_instance.SoftwareVersion = get_lstchain_version()
class_instance.ProcessingConfigFile = options.configfile
class_instance.ProcessingConfigFile = str(options.configfile)
class_instance.ObservationDate = flat_date
if class_instance.__name__ in REDUCTION_TASKS:
muon_dir = Path(dl1_dir) / flat_date / options.prod_id
outdir_dl1 = Path(dl1_dir) / flat_date / options.prod_id / options.dl1_prod_id
outdir_dl2 = Path(dl2_dir) / flat_date / options.prod_id / options.dl2_prod_id
muon_dir = dl1_dir / flat_date / options.prod_id / "muons"
outdir_dl1 = dl1_dir / flat_date / options.prod_id / options.dl1_prod_id
outdir_dl2 = dl2_dir / flat_date / options.prod_id / options.dl2_prod_id

if class_instance.__name__ in ["drs4_pedestal", "calibrate_charge"]:
# drs4_pedestal_run_id [0] 1804
Expand All @@ -63,123 +61,133 @@ def parse_variables(class_instance):
class_instance.PedestalRun = f"{class_instance.args[0]:05d}"
class_instance.CalibrationRun = f"{class_instance.args[1]:05d}"

pro = "pro"
# TODO - massive reprocessing vs. next day processing
version = get_lstchain_version()

# according to code in onsite scripts in lstchain
class_instance.RawObservationFilePedestal = os.path.realpath(
f"{raw_dir}/{flat_date}/LST-1.1.Run{class_instance.args[0]:05d}.fits.fz"
)
class_instance.RawObservationFileCalibration = os.path.realpath(
f"{raw_dir}/{flat_date}/LST-1.1.Run{class_instance.args[1]:05d}.fits.fz"
)
class_instance.PedestalCheckPlot = os.path.realpath(
f"{pedestal_dir}/{flat_date}/{pro}/"
f"log/drs4_pedestal.Run{class_instance.args[0]:05d}.0000.pdf"
)
class_instance.CalibrationCheckPlot = os.path.realpath(
f"{calib_dir}/{flat_date}/{pro}/"
f"log/calibration_filters_52.Run{class_instance.args[1]:05d}.0000.pdf"
)
class_instance.RawObservationFilePedestal = str((
raw_dir / flat_date / f"LST-1.1.Run{class_instance.args[0]:05d}.fits.fz"
).resolve())
class_instance.RawObservationFileCalibration = str((
raw_dir / flat_date / f"LST-1.1.Run{class_instance.args[1]:05d}.fits.fz"
).resolve())
class_instance.PedestalCheckPlot = str((
pedestal_dir
/ flat_date
/ version
/ f"log/drs4_pedestal.Run{class_instance.args[0]:05d}.0000.pdf"
).resolve())
class_instance.CalibrationCheckPlot = str((
calib_dir
/ flat_date
/ version
/ f"log/calibration_filters_52.Run{class_instance.args[1]:05d}.0000.pdf"
).resolve())

# according to code in sequence_calibration_files from paths.py
class_instance.PedestalFile = os.path.realpath(
f"{pedestal_dir}/{flat_date}/{pro}/"
f"drs4_pedestal.Run{class_instance.args[0]:05d}.0000.h5"
)
class_instance.CoefficientsCalibrationFile = os.path.realpath(
f"{calib_dir}/{flat_date}/{pro}/"
f"calibration_filters_52.Run{class_instance.args[1]:05d}.0000.h5"
)
class_instance.PedestalFile = str((
pedestal_dir
/ flat_date
/ version
/ f"drs4_pedestal.Run{class_instance.args[0]:05d}.0000.h5"
).resolve())
class_instance.CoefficientsCalibrationFile = str((
calib_dir
/ flat_date
/ version
/ f"calibration_filters_52.Run{class_instance.args[1]:05d}.0000.h5"
).resolve())

if class_instance.__name__ == "r0_to_dl1":
# calibration_file [0] .../20200218/pro/calibration_filters_52.Run02006.0000.h5
# drs4_pedestal_file [1] .../20200218/pro/drs4_pedestal.Run02005.0000.h5
# time_calib_file [2] .../20191124/pro/time_calibration.Run01625.0000.h5
# systematic_corr [3] .../20200101/pro/no_sys_corrected_calib_20210514.0000.h5
# calibration_file [0] .../20200218/v0.4.3/calibration_filters_52.Run02006.0000.h5
# drs4_pedestal_file [1] .../20200218/v0.4.3/drs4_pedestal.Run02005.0000.h5
# time_calib_file [2] .../20191124/v0.4.3/time_calibration.Run01625.0000.h5
# systematic_corr [3] .../20200101/v0.4.3/no_sys_corrected_calib_20210514.0000.h5
# drive_file [4] .../DrivePositioning/DrivePosition_20200218.txt
# run_summary_file [5] .../RunSummary/RunSummary_20200101.ecsv
# pedestal_ids_file [6] .../path/to/interleaved/pedestal/events.h5
# run_str [7] 02006.0000

run_subrun_id = class_instance.args[7]
class_instance.ObservationRun = run_subrun_id.split(".")[0]
# use realpath to resolve symbolic links and return abspath
calibration_file = os.path.realpath(class_instance.args[0])
pedestal_file = os.path.realpath(class_instance.args[1])
timecalibration_file = os.path.realpath(class_instance.args[2])
systematic_correction_file = os.path.realpath(class_instance.args[3])
class_instance.R0SubrunDataset = os.path.realpath(
f"{raw_dir}/{flat_date}/LST-1.1.Run{run_subrun_id}.fits.fz"
)
class_instance.CoefficientsCalibrationFile = calibration_file
class_instance.PedestalFile = pedestal_file
class_instance.TimeCalibrationFile = timecalibration_file
class_instance.SystematicCorrectionFile = systematic_correction_file
class_instance.PointingFile = os.path.realpath(class_instance.args[4])
class_instance.RunSummaryFile = os.path.realpath(class_instance.args[5])
class_instance.DL1SubrunDataset = os.path.realpath(
f"{outdir_dl1}/dl1_LST-1.Run{run_subrun_id}.h5"
)
class_instance.MuonsSubrunDataset = os.path.realpath(
f"{muon_dir}/muons_LST-1.Run{run_subrun_id}.fits"
)
run_subrun = class_instance.args[7]
run = run_subrun.split(".")[0]
class_instance.ObservationRun = run

calibration_file = Path(class_instance.args[0]).resolve()
pedestal_file = Path(class_instance.args[1]).resolve()
timecalibration_file = Path(class_instance.args[2]).resolve()
systematic_correction_file = Path(class_instance.args[3]).resolve()
class_instance.R0SubrunDataset = str((
raw_dir / flat_date / f"LST-1.1.Run{run_subrun}.fits.fz"
).resolve())
class_instance.CoefficientsCalibrationFile = str(calibration_file)
class_instance.PedestalFile = str(pedestal_file)
class_instance.TimeCalibrationFile = str(timecalibration_file)
class_instance.SystematicCorrectionFile = str(systematic_correction_file)
class_instance.PointingFile = str(Path(class_instance.args[4]).resolve())
class_instance.RunSummaryFile = str(Path(class_instance.args[5]).resolve())
class_instance.DL1SubrunDataset = str(
(outdir_dl1 / f"dl1_LST-1.Run{run_subrun}.h5").resolve()
)
class_instance.MuonsSubrunDataset = str((
muon_dir / f"muons_LST-1.Run{run_subrun}.fits"
).resolve())
class_instance.InterleavedPedestalEventsFile = None
if class_instance.args[6] is not None:
class_instance.InterleavedPedestalEventsFile = os.path.realpath(class_instance.args[6])
class_instance.InterleavedPedestalEventsFile = str(Path(class_instance.args[6]))

if class_instance.__name__ == "dl1ab":
# run_str [0] 02006.0000

class_instance.Analysisconfigfile_dl1 = os.path.realpath(configfile_dl1b)
class_instance.Analysisconfigfile_dl1 = str(Path(configfile_dl1b))
class_instance.ObservationRun = class_instance.args[0].split(".")[0]
class_instance.StoreImage = cfg.getboolean("lstchain", "store_image_dl1ab")
class_instance.DL1SubrunDataset = os.path.realpath(
f"{outdir_dl1}/dl1_LST-1.Run{class_instance.args[0]}.h5"
)
class_instance.DL1SubrunDataset = str((
outdir_dl1 / f"dl1_LST-1.Run{class_instance.args[0]}.h5"
).resolve())

if class_instance.__name__ == "dl1_datacheck":
# run_str [0] 02006.0000

class_instance.ObservationRun = class_instance.args[0].split(".")[0]
class_instance.DL1SubrunDataset = os.path.realpath(
f"{outdir_dl1}/dl1_LST-1.Run{class_instance.args[0]}.h5"
)
class_instance.MuonsSubrunDataset = os.path.realpath(
f"{muon_dir}/muons_LST-1.Run{class_instance.args[0]}.fits"
)
class_instance.DL1CheckSubrunDataset = os.path.realpath(
f"{outdir_dl1}/datacheck_dl1_LST-1.Run{class_instance.args[0]}.h5"
)
class_instance.DL1CheckHDF5File = os.path.realpath(
f"{outdir_dl1}/datacheck_dl1_LST-1.Run{class_instance.ObservationRun}.h5"
)
class_instance.DL1CheckPDFFile = os.path.realpath(
f"{outdir_dl1}/datacheck_dl1_LST-1.Run{class_instance.ObservationRun}.pdf"
)
run_subrun = class_instance.args[0]
run = run_subrun.split(".")[0]

class_instance.ObservationRun = run
class_instance.DL1SubrunDataset = str(
(outdir_dl1 / f"dl1_LST-1.Run{run_subrun}.h5").resolve()
)
class_instance.MuonsSubrunDataset = str((
muon_dir / f"muons_LST-1.Run{run_subrun}.fits"
).resolve())
class_instance.DL1CheckSubrunDataset = str((
outdir_dl1 / f"datacheck_dl1_LST-1.Run{run_subrun}.h5"
).resolve())
class_instance.DL1CheckHDF5File = str((
outdir_dl1 / f"datacheck_dl1_LST-1.Run{run}.h5"
).resolve())
class_instance.DL1CheckPDFFile = str((
outdir_dl1 / f"datacheck_dl1_LST-1.Run{run}.pdf"
).resolve())

if class_instance.__name__ == "dl1_to_dl2":
# run_str [0] 02006.0000
run_subrun = class_instance.args[0]
run = run_subrun.split(".")[0]

class_instance.Analysisconfigfile_dl2 = configfile_dl2
class_instance.ObservationRun = class_instance.args[0].split(".")[0]
class_instance.RFModelEnergyFile = os.path.realpath(f"{rf_models_directory}/reg_energy.sav")
class_instance.RFModelDispNormFile = os.path.realpath(
f"{rf_models_directory}/reg_disp_norm.sav"
)
class_instance.RFModelDispSignFile = os.path.realpath(
f"{rf_models_directory}/reg_disp_sign.sav"
class_instance.ObservationRun = run
class_instance.RFModelEnergyFile = str((rf_models_directory / "reg_energy.sav").resolve())
class_instance.RFModelDispNormFile = str(
(rf_models_directory / "reg_disp_norm.sav").resolve()
)
class_instance.RFModelGammanessFile = os.path.realpath(f"{rf_models_directory}/cls_gh.sav")
class_instance.DL1SubrunDataset = os.path.realpath(
f"{outdir_dl1}/dl1_LST-1.Run{class_instance.args[0]}.h5"
class_instance.RFModelDispSignFile = str(
(rf_models_directory / "reg_disp_sign.sav").resolve()
)
class_instance.DL2SubrunDataset = os.path.realpath(
f"{outdir_dl2}/dl2_LST-1.Run{class_instance.args[0]}.h5"
class_instance.RFModelGammanessFile = str((rf_models_directory / "cls_gh.sav").resolve())
class_instance.DL1SubrunDataset = str(
(outdir_dl1 / f"dl1_LST-1.Run{run_subrun}.h5").resolve()
)
class_instance.DL2MergedFile = os.path.realpath(
f"{outdir_dl2}/dl2_LST-1.Run{class_instance.ObservationRun}.h5"
class_instance.DL2SubrunDataset = str(
(outdir_dl2 / f"dl2_LST-1.Run{run_subrun}.h5").resolve()
)
class_instance.DL2MergedFile = str((outdir_dl2 / f"dl2_LST-1.Run{run}.h5").resolve())

if class_instance.__name__ in REDUCTION_TASKS:
class_instance.session_name = class_instance.ObservationRun
Expand All @@ -189,5 +197,5 @@ def parse_variables(class_instance):

def get_log_config():
"""Get logging configuration from provenance logger config file."""
std_logger_file = files("osa.provenance").joinpath("config/logger.yaml")
std_logger_file = files("osa.provenance") / "config/logger.yaml"
return std_logger_file.read_text()

0 comments on commit ff686d5

Please sign in to comment.