From 35e66d87fbb0b79a7042fb9f1959926bc40ea7e7 Mon Sep 17 00:00:00 2001 From: Maria Lainez <98marialainez@gmail.com> Date: Fri, 15 Dec 2023 12:40:46 +0100 Subject: [PATCH 1/5] Do not create the DL2 directory if --no-dl2 option is given --- src/osa/scripts/provprocess.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/src/osa/scripts/provprocess.py b/src/osa/scripts/provprocess.py index 1c5962ff..1ed9e95b 100644 --- a/src/osa/scripts/provprocess.py +++ b/src/osa/scripts/provprocess.py @@ -413,29 +413,31 @@ def produce_provenance(session_log_filename, base_filename): read_prov(filename=session_log_filename), str(paths_r0_dl1["out_path"]), ) - dl1_lines = plines_r0 + plines_ab[1:] - - # create r0_to_dl1 prov files only if filtering - if options.filter == "r0_to_dl1": - produce_provenance_files(plines_r0 + plines_ab[1:], paths_r0_dl1) - - if options.filter == "dl1_to_dl2" or not options.filter: - paths_dl1_dl2 = define_paths("dl1_to_dl2", PATH_DL2, options.dl2_prod_id, base_filename) plines_check = parse_lines_run( "dl1_datacheck", read_prov(filename=session_log_filename), str(paths_dl1_dl2["out_path"]), ) - plines_dl2 = parse_lines_run( - "dl1_to_dl2", - read_prov(filename=session_log_filename), - str(paths_dl1_dl2["out_path"]), - ) - dl1_dl2_lines = plines_check + plines_dl2[1:] + dl1_lines = plines_r0 + plines_ab[1:] + plines_check[1:] + + # create r0_to_dl1 prov files only if filtering + if options.filter == "r0_to_dl1": + produce_provenance_files(plines_r0 + plines_ab[1:] + plines_check[1:], paths_r0_dl1) + + if options.filter == "dl1_to_dl2" or not options.filter: + if not options.no_dl2: + paths_dl1_dl2 = define_paths("dl1_to_dl2", PATH_DL2, options.dl2_prod_id, base_filename) + plines_dl2 = parse_lines_run( + "dl1_to_dl2", + read_prov(filename=session_log_filename), + str(paths_dl1_dl2["out_path"]), + ) + dl1_dl2_lines = plines_dl2 # create dl1_to_dl2 prov files only if filtering if options.filter == "dl1_to_dl2": - produce_provenance_files(plines_check + plines_dl2[1:], paths_dl1_dl2) + if not options.no_dl2: + produce_provenance_files(plines_dl2, paths_dl1_dl2) # create calibration_to_dl1 and calibration_to_dl2 prov files if not options.filter: From 1b7adbe87f8de0eec96fdaa01a3d200600b68559 Mon Sep 17 00:00:00 2001 From: Maria Lainez <98marialainez@gmail.com> Date: Fri, 15 Dec 2023 12:45:32 +0100 Subject: [PATCH 2/5] change path of dl1_datacheck lines --- src/osa/scripts/provprocess.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/osa/scripts/provprocess.py b/src/osa/scripts/provprocess.py index 1ed9e95b..d2db95d7 100644 --- a/src/osa/scripts/provprocess.py +++ b/src/osa/scripts/provprocess.py @@ -416,7 +416,7 @@ def produce_provenance(session_log_filename, base_filename): plines_check = parse_lines_run( "dl1_datacheck", read_prov(filename=session_log_filename), - str(paths_dl1_dl2["out_path"]), + str(paths_r0_dl1["out_path"]), ) dl1_lines = plines_r0 + plines_ab[1:] + plines_check[1:] From ee59684b42b4e7976ff61ae01efaac70ff827c64 Mon Sep 17 00:00:00 2001 From: Maria Lainez <98marialainez@gmail.com> Date: Fri, 12 Jan 2024 11:12:47 +0100 Subject: [PATCH 3/5] Move dl1_datacheck to r0_to_dl1 key in parse_lines_log --- src/osa/scripts/provprocess.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/osa/scripts/provprocess.py b/src/osa/scripts/provprocess.py index d2db95d7..0b603a3f 100644 --- a/src/osa/scripts/provprocess.py +++ b/src/osa/scripts/provprocess.py @@ -81,8 +81,8 @@ def parse_lines_log(filter_cut, calib_runs, run_number): filter_cut = "all" cuts = { "calibration": ["drs4_pedestal", "calibrate_charge"], - "r0_to_dl1": ["r0_to_dl1", "dl1ab"], - "dl1_to_dl2": ["dl1_datacheck", "dl1_to_dl2"], + "r0_to_dl1": ["r0_to_dl1", "dl1ab", "dl1_datacheck"], + "dl1_to_dl2": ["dl1_to_dl2"], } cuts["all"] = cuts["calibration"] + cuts["r0_to_dl1"] + cuts["dl1_to_dl2"] From 2a02a871073f426408a213671056912df12b4c01 Mon Sep 17 00:00:00 2001 From: Maria Lainez <98marialainez@gmail.com> Date: Fri, 12 Jan 2024 11:35:31 +0100 Subject: [PATCH 4/5] Adapt tests --- src/osa/scripts/tests/test_osa_scripts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/osa/scripts/tests/test_osa_scripts.py b/src/osa/scripts/tests/test_osa_scripts.py index d2fceb48..5c2e80bd 100644 --- a/src/osa/scripts/tests/test_osa_scripts.py +++ b/src/osa/scripts/tests/test_osa_scripts.py @@ -99,7 +99,7 @@ def test_simulate_processing( with open(json_file_dl1) as file: dl1 = yaml.safe_load(file) - assert len(dl1["entity"]) == 16 + assert len(dl1["entity"]) == 19 assert len(dl1["activity"]) == 4 assert len(dl1["used"]) == 13 assert len(dl1["wasGeneratedBy"]) == 7 From 18c563704fe21d87fa56817ab0fc40db5d4a6cec Mon Sep 17 00:00:00 2001 From: Maria Lainez <98marialainez@gmail.com> Date: Fri, 12 Jan 2024 13:48:39 +0100 Subject: [PATCH 5/5] Adapt tests --- src/osa/scripts/tests/test_osa_scripts.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/osa/scripts/tests/test_osa_scripts.py b/src/osa/scripts/tests/test_osa_scripts.py index 5c2e80bd..b4f0b4d2 100644 --- a/src/osa/scripts/tests/test_osa_scripts.py +++ b/src/osa/scripts/tests/test_osa_scripts.py @@ -100,9 +100,9 @@ def test_simulate_processing( with open(json_file_dl1) as file: dl1 = yaml.safe_load(file) assert len(dl1["entity"]) == 19 - assert len(dl1["activity"]) == 4 - assert len(dl1["used"]) == 13 - assert len(dl1["wasGeneratedBy"]) == 7 + assert len(dl1["activity"]) == 5 + assert len(dl1["used"]) == 15 + assert len(dl1["wasGeneratedBy"]) == 10 with open(json_file_dl2) as file: dl2 = yaml.safe_load(file)