From 604c4c56c3bdcf5834cdf9e7e4311d476de0af31 Mon Sep 17 00:00:00 2001 From: Maria Lainez <98marialainez@gmail.com> Date: Fri, 2 Feb 2024 08:36:36 +0100 Subject: [PATCH 1/6] Launch lstchain_cherenkov_transparency.py script as a new step of closer --- src/osa/scripts/closer.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/osa/scripts/closer.py b/src/osa/scripts/closer.py index 2f97bcec..f47cbfd4 100644 --- a/src/osa/scripts/closer.py +++ b/src/osa/scripts/closer.py @@ -8,6 +8,7 @@ import shutil import subprocess import sys +import time from datetime import datetime, timedelta from pathlib import Path from typing import Tuple, Iterable, List @@ -157,6 +158,12 @@ def post_process(seq_tuple): list_job_id = merge_dl1_datacheck(seq_list) daily_datacheck(daily_longterm_cmd(list_job_id)) + # Wait 5 minutes for the longterm DL1 datacheck file to be produced + time.sleep(300) + + # Update longterm DL1 datacheck file with cherenkov_transparency script + cherenkov_transparency() + # Extract the provenance info extract_provenance(seq_list) @@ -490,5 +497,31 @@ def daily_datacheck(cmd: List[str]): log.debug("Simulate launching scripts") +def cherenkov_transparency(): + """Update longterm dl1 check files with cherenkov transparency information.""" + nightdir = date_to_dir(options.date) + datacheck_dir = destination_dir("DATACHECK", create_dir=False) + longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR")) / options.prod_id / nightdir + longterm_datacheck_file = longterm_dir / f"DL1_datacheck_{nightdir}.h5" + + cmd = [ + "sbatch", + "-D", + options.directory, + "-o", + "log/cherenkov_transparency_%j.log", + "lstchain_cherenkov_transparency", + f"--update-datacheck-file={longterm_datacheck_file}", + f"--input-dir={datacheck_dir}", + ] + + log.debug(f"Executing {stringify(cmd)}") + + if not options.simulate and not options.test and shutil.which("sbatch") is not None: + subprocess.run(cmd, check=True) + else: + log.debug("Simulate launching scripts") + + if __name__ == "__main__": main() From ca2e7ece7ae3227bc66ee797e8f00d377b0d7cc3 Mon Sep 17 00:00:00 2001 From: Maria Lainez <98marialainez@gmail.com> Date: Fri, 2 Feb 2024 12:32:39 +0100 Subject: [PATCH 2/6] Update lstchain to v0.10.7 --- environment.yml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index ba1880a8..fd675889 100644 --- a/environment.yml +++ b/environment.yml @@ -25,7 +25,7 @@ dependencies: - ctapipe_io_lst=0.22 - ctaplot~=0.6.4 - pyirf~=0.10 - - lstchain>=0.10.5 + - lstchain>=0.10.7 - tenacity # dev dependencies - pytest diff --git a/pyproject.toml b/pyproject.toml index d6d308bf..59bd222a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ classifiers = [ requires-python = ">=3.9" dependencies = [ "astropy~=5.0", - "lstchain>=0.10.5", + "lstchain>=0.10.7", "matplotlib", "numpy", "pandas", From 48e82facb59c7fe64451becf96d0af461949231b Mon Sep 17 00:00:00 2001 From: Maria Lainez <98marialainez@gmail.com> Date: Fri, 2 Feb 2024 14:42:11 +0100 Subject: [PATCH 3/6] Add a slurm dependency between the longterm DL1 datacheck jobs --- src/osa/scripts/closer.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/osa/scripts/closer.py b/src/osa/scripts/closer.py index f47cbfd4..e98b9bc1 100644 --- a/src/osa/scripts/closer.py +++ b/src/osa/scripts/closer.py @@ -8,7 +8,6 @@ import shutil import subprocess import sys -import time from datetime import datetime, timedelta from pathlib import Path from typing import Tuple, Iterable, List @@ -153,16 +152,12 @@ def post_process(seq_tuple): post_process_files(seq_list) # Merge DL1 datacheck files and produce PDFs. It also produces - # the daily datacheck report using the longterm script. + # the daily datacheck report using the longterm script, and updates + # the longterm DL1 datacheck file with the cherenkov_transparency script. if cfg.getboolean("lstchain", "merge_dl1_datacheck"): list_job_id = merge_dl1_datacheck(seq_list) - daily_datacheck(daily_longterm_cmd(list_job_id)) - - # Wait 5 minutes for the longterm DL1 datacheck file to be produced - time.sleep(300) - - # Update longterm DL1 datacheck file with cherenkov_transparency script - cherenkov_transparency() + longterm_job_id = daily_datacheck(daily_longterm_cmd(list_job_id)) + cherenkov_transparency(cherenkov_transparency_cmd(longterm_job_id)) # Extract the provenance info extract_provenance(seq_list) @@ -492,29 +487,36 @@ def daily_datacheck(cmd: List[str]): log.debug(f"Executing {stringify(cmd)}") if not options.simulate and not options.test and shutil.which("sbatch") is not None: - subprocess.run(cmd, check=True) + job = subprocess.run(cmd, check=True) + job_id = job.stdout.strip() + return job_id else: log.debug("Simulate launching scripts") -def cherenkov_transparency(): - """Update longterm dl1 check files with cherenkov transparency information.""" +def cherenkov_transparency_cmd(longterm_job_id: str) -> List[str]: + """Build the cherenkov transparency command.""" nightdir = date_to_dir(options.date) datacheck_dir = destination_dir("DATACHECK", create_dir=False) longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR")) / options.prod_id / nightdir longterm_datacheck_file = longterm_dir / f"DL1_datacheck_{nightdir}.h5" - cmd = [ + return [ "sbatch", "-D", options.directory, "-o", "log/cherenkov_transparency_%j.log", + f"--dependency=afterok:{longterm_job_id}", "lstchain_cherenkov_transparency", f"--update-datacheck-file={longterm_datacheck_file}", f"--input-dir={datacheck_dir}", ] + +def cherenkov_transparency(cmd: List[str]): + """Update longterm dl1 check files with cherenkov transparency information.""" + log.info("Daily dl1 checks using longterm script.") log.debug(f"Executing {stringify(cmd)}") if not options.simulate and not options.test and shutil.which("sbatch") is not None: From 0b89de71cf8e92cb700f50dd7d262e3a5b586a21 Mon Sep 17 00:00:00 2001 From: Maria Lainez <98marialainez@gmail.com> Date: Fri, 2 Feb 2024 14:45:37 +0100 Subject: [PATCH 4/6] Correct log information --- src/osa/scripts/closer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/osa/scripts/closer.py b/src/osa/scripts/closer.py index e98b9bc1..0badd68e 100644 --- a/src/osa/scripts/closer.py +++ b/src/osa/scripts/closer.py @@ -515,8 +515,8 @@ def cherenkov_transparency_cmd(longterm_job_id: str) -> List[str]: def cherenkov_transparency(cmd: List[str]): - """Update longterm dl1 check files with cherenkov transparency information.""" - log.info("Daily dl1 checks using longterm script.") + """Update longterm dl1 check file with cherenkov transparency information.""" + log.info("Update longterm dl1 check file with cherenkov_transparency script.") log.debug(f"Executing {stringify(cmd)}") if not options.simulate and not options.test and shutil.which("sbatch") is not None: From b2444c11dd11ef2187408790a7ee6e3b053e95e1 Mon Sep 17 00:00:00 2001 From: Maria Lainez <98marialainez@gmail.com> Date: Mon, 5 Feb 2024 12:25:45 +0100 Subject: [PATCH 5/6] Add parsable option to daily_longterm_cmd --- src/osa/scripts/closer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/osa/scripts/closer.py b/src/osa/scripts/closer.py index 0badd68e..03a36446 100644 --- a/src/osa/scripts/closer.py +++ b/src/osa/scripts/closer.py @@ -468,6 +468,7 @@ def daily_longterm_cmd(parent_job_ids: List[str]) -> List[str]: return [ "sbatch", + "--parsable", "-D", options.directory, "-o", From 930f713e7c3533f31c39851d51cedf5faec45d4a Mon Sep 17 00:00:00 2001 From: Maria Lainez <98marialainez@gmail.com> Date: Mon, 5 Feb 2024 18:15:32 +0100 Subject: [PATCH 6/6] Correct expected daily_longterm command --- src/osa/scripts/tests/test_osa_scripts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/osa/scripts/tests/test_osa_scripts.py b/src/osa/scripts/tests/test_osa_scripts.py index b4f0b4d2..7dcc7ee1 100644 --- a/src/osa/scripts/tests/test_osa_scripts.py +++ b/src/osa/scripts/tests/test_osa_scripts.py @@ -319,6 +319,7 @@ def test_daily_longterm_cmd(): expected_cmd = [ "sbatch", + "--parsable", "-D", options.directory, "-o",