diff --git a/environment.yml b/environment.yml index ba1880a8..fd675889 100644 --- a/environment.yml +++ b/environment.yml @@ -25,7 +25,7 @@ dependencies: - ctapipe_io_lst=0.22 - ctaplot~=0.6.4 - pyirf~=0.10 - - lstchain>=0.10.5 + - lstchain>=0.10.7 - tenacity # dev dependencies - pytest diff --git a/pyproject.toml b/pyproject.toml index d6d308bf..59bd222a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ classifiers = [ requires-python = ">=3.9" dependencies = [ "astropy~=5.0", - "lstchain>=0.10.5", + "lstchain>=0.10.7", "matplotlib", "numpy", "pandas", diff --git a/src/osa/scripts/closer.py b/src/osa/scripts/closer.py index 2f97bcec..03a36446 100644 --- a/src/osa/scripts/closer.py +++ b/src/osa/scripts/closer.py @@ -152,10 +152,12 @@ def post_process(seq_tuple): post_process_files(seq_list) # Merge DL1 datacheck files and produce PDFs. It also produces - # the daily datacheck report using the longterm script. + # the daily datacheck report using the longterm script, and updates + # the longterm DL1 datacheck file with the cherenkov_transparency script. if cfg.getboolean("lstchain", "merge_dl1_datacheck"): list_job_id = merge_dl1_datacheck(seq_list) - daily_datacheck(daily_longterm_cmd(list_job_id)) + longterm_job_id = daily_datacheck(daily_longterm_cmd(list_job_id)) + cherenkov_transparency(cherenkov_transparency_cmd(longterm_job_id)) # Extract the provenance info extract_provenance(seq_list) @@ -466,6 +468,7 @@ def daily_longterm_cmd(parent_job_ids: List[str]) -> List[str]: return [ "sbatch", + "--parsable", "-D", options.directory, "-o", @@ -484,6 +487,39 @@ def daily_datacheck(cmd: List[str]): log.info("Daily dl1 checks using longterm script.") log.debug(f"Executing {stringify(cmd)}") + if not options.simulate and not options.test and shutil.which("sbatch") is not None: + job = subprocess.run(cmd, check=True) + job_id = job.stdout.strip() + return job_id + else: + log.debug("Simulate launching scripts") + + +def cherenkov_transparency_cmd(longterm_job_id: str) -> List[str]: + """Build the cherenkov transparency command.""" + nightdir = date_to_dir(options.date) + datacheck_dir = destination_dir("DATACHECK", create_dir=False) + longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR")) / options.prod_id / nightdir + longterm_datacheck_file = longterm_dir / f"DL1_datacheck_{nightdir}.h5" + + return [ + "sbatch", + "-D", + options.directory, + "-o", + "log/cherenkov_transparency_%j.log", + f"--dependency=afterok:{longterm_job_id}", + "lstchain_cherenkov_transparency", + f"--update-datacheck-file={longterm_datacheck_file}", + f"--input-dir={datacheck_dir}", + ] + + +def cherenkov_transparency(cmd: List[str]): + """Update longterm dl1 check file with cherenkov transparency information.""" + log.info("Update longterm dl1 check file with cherenkov_transparency script.") + log.debug(f"Executing {stringify(cmd)}") + if not options.simulate and not options.test and shutil.which("sbatch") is not None: subprocess.run(cmd, check=True) else: diff --git a/src/osa/scripts/tests/test_osa_scripts.py b/src/osa/scripts/tests/test_osa_scripts.py index b4f0b4d2..7dcc7ee1 100644 --- a/src/osa/scripts/tests/test_osa_scripts.py +++ b/src/osa/scripts/tests/test_osa_scripts.py @@ -319,6 +319,7 @@ def test_daily_longterm_cmd(): expected_cmd = [ "sbatch", + "--parsable", "-D", options.directory, "-o",