Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add gain selection to OSA workflow #230

Merged
merged 37 commits into from
Mar 12, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
3b415a1
Add the possibility to run gain selection for a single date or for a …
marialainez Jun 15, 2023
aba765d
Write a flag file if gain selection finishes successfully
marialainez Jun 16, 2023
e42d9e1
Check if the gain selection finished successfully before starting the…
marialainez Jun 16, 2023
a6ba55e
add necessary imports
marialainez Jun 16, 2023
97e9618
Mistake in the name of the gain selection file
marialainez Jun 20, 2023
0cc7643
Create the flag file using .touch()
marialainez Jun 20, 2023
ac3acf9
Remove unnecessary import
marialainez Jun 20, 2023
2407ae5
add fixture for the gain selection flag file
marialainez Jun 22, 2023
03f68a1
update config.py to the main branch
marialainez Jun 22, 2023
09cf58f
Merge branch main
marialainez Feb 20, 2024
80ed20b
Add start and end time arguments for apply_gain_selection
marialainez Feb 20, 2024
269043e
Add comment to try again later
marialainez Feb 20, 2024
2e82c50
Change name of the directory of the gain selection flag file
marialainez Feb 20, 2024
ed77a8d
Make sure the gain selection flag file exists for the tests
marialainez Feb 21, 2024
1e01368
Make sure the GainSel directory for the date exists for the tests
marialainez Feb 21, 2024
5db8c22
fix tests
marialainez Feb 21, 2024
2619970
fix tests
marialainez Feb 21, 2024
647c6ed
Check if any runs were taken before checking if gain selection finsih…
marialainez Feb 22, 2024
511c92d
Add --no-gainsel option to avoid checking if the gain selection flag …
marialainez Feb 22, 2024
0e464f5
import run_summary_table
marialainez Feb 22, 2024
2bf5f41
Add date as argument of GainSel_flag_file
marialainez Feb 27, 2024
ad33d55
Add date to GainSel_flag_file function in sequencer
marialainez Feb 27, 2024
5f3837b
Use argparser to add the arguments of the script, so that a specific …
marialainez Feb 27, 2024
7590ff3
Create directory before creating gain selection flag file
marialainez Feb 27, 2024
5cb8019
Use date (instead of args.date) in the for loop
marialainez Feb 27, 2024
4cfbe24
Remove unnecessary import
marialainez Feb 27, 2024
cfac411
Use new lstchain script for dates from 20231205
marialainez Feb 27, 2024
57fa22d
Fix mistake
marialainez Feb 27, 2024
eacd04b
Make gain_selection script more informative
marialainez Feb 27, 2024
f2a88c8
Fix undefined name
marialainez Feb 27, 2024
7115e77
Unnecessary f-string
marialainez Feb 27, 2024
e8dc1e2
By default use --no-flatfield-heuristic option
marialainez Feb 27, 2024
73002cc
Use both queues short and long to run the gain selection
marialainez Mar 5, 2024
73a7489
Add option to avoid checking the number of pending jobs in queue
marialainez Mar 5, 2024
f402ff1
Fix name
marialainez Mar 5, 2024
200aa18
Correction
marialainez Mar 5, 2024
e4e6d3d
Add --no-gainsel option also for autocloser
marialainez Mar 11, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/osa/configs/sequencer.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ HIGH_LEVEL_DIR: %(OSA_DIR)s/HighLevel
LONGTERM_DIR: %(OSA_DIR)s/DL1DataCheck_LongTerm
MERGED_SUMMARY: %(OSA_DIR)s/Catalog/merged_RunSummary.ecsv
SEQUENCER_WEB_DIR: %(OSA_DIR)s/SequencerWeb
GAIN_SELECTION_FLAG_DIR: %(OSA_DIR)s/GainSel

# To be set by the user. Using PROD-ID will overcome the automatic
# fetching of lstchain version. Otherwise leave it empty (and without the colon symbol).
Expand Down Expand Up @@ -138,6 +139,7 @@ WORKFLOWPREFIX: Workflow
GRAPHSUFFIX: .dot
SVGSUFFIX: .svg
end_of_activity: NightFinished.txt
gain_selection_check: GainSelFinished.txt

[OUTPUT]
# REPORTWIDTH is the width in characters of the heading frame for the output
Expand Down
25 changes: 22 additions & 3 deletions src/osa/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,13 @@ def monitoring_dir(base_test_dir):
return monitoring_dir


@pytest.fixture(scope="session")
def osa_dir(base_test_dir):
osa_dir = base_test_dir / "OSA"
osa_dir.mkdir(parents=True, exist_ok=True)
return osa_dir


@pytest.fixture(scope="session")
def run_summary_dir(monitoring_dir):
summary_dir = monitoring_dir / "RunSummary"
Expand Down Expand Up @@ -436,6 +443,8 @@ def sequence_file_list(
drs4_time_calibration_files,
systematic_correction_files,
r0_data,
gain_selection_flag_file,
merged_run_summary,
):
for r0_file in r0_data:
assert r0_file.exists()
Expand All @@ -448,6 +457,8 @@ def sequence_file_list(

assert run_summary_file.exists()
assert run_catalog.exists()
assert gain_selection_flag_file.exists()
assert merged_run_summary.exists()

run_program("sequencer", "-d", "2020-01-17", "--no-submit", "-t", "LST1")
# First sequence in the list corresponds to the calibration run 1809
Expand Down Expand Up @@ -548,11 +559,9 @@ def run_catalog(run_catalog_dir):


@pytest.fixture(scope="session")
def database(base_test_dir):
def database(osa_dir):
import sqlite3

osa_dir = base_test_dir / "OSA"
osa_dir.mkdir(parents=True, exist_ok=True)
db_file = osa_dir / "osa.db"
with sqlite3.connect(db_file) as connection:
cursor = connection.cursor()
Expand All @@ -562,3 +571,13 @@ def database(base_test_dir):
)
cursor.connection.commit()
yield cursor


@pytest.fixture(scope="session")
def gain_selection_flag_file(osa_dir):

GainSel_dir = osa_dir / "GainSel" / "20200117"
GainSel_dir.mkdir(parents=True, exist_ok=True)
file = GainSel_dir / "GainSelFinished.txt"
file.touch()
return file
10 changes: 7 additions & 3 deletions src/osa/scripts/autocloser.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
config_file: Path,
ignore_cronlock: bool = False,
test: bool = False,
no_gainsel: bool = False,
):
"""
Parameters
Expand Down Expand Up @@ -82,7 +83,7 @@
if not self.lock_automatic_sequencer() and not ignore_cronlock:
log.warning(f"{self.telescope} already locked! Ignoring {self.telescope}")
return
if not self.simulate_sequencer(date, config_file, test):
if not self.simulate_sequencer(date, config_file, test, no_gainsel):
log.warning(
f"Simulation of the sequencer failed "
f"for {self.telescope}! Ignoring {self.telescope}"
Expand Down Expand Up @@ -121,7 +122,7 @@
self.locked = True
return True

def simulate_sequencer(self, date: str, config_file: Path, test: bool):
def simulate_sequencer(self, date: str, config_file: Path, test: bool, no_gainsel: bool):
"""Launch the sequencer in simulation mode."""
if test:
self.read_file()
Expand All @@ -135,6 +136,9 @@
date,
self.telescope,
]
if no_gainsel:
sequencer_cmd.insert(1, "--no-gainsel")

Check warning on line 140 in src/osa/scripts/autocloser.py

View check run for this annotation

Codecov / codecov/patch

src/osa/scripts/autocloser.py#L140

Added line #L140 was not covered by tests

log.debug(f"Executing {' '.join(sequencer_cmd)}")
sequencer = subprocess.Popen(
sequencer_cmd,
Expand Down Expand Up @@ -445,7 +449,7 @@
# create telescope and sequence objects
log.info("Simulating sequencer...")

telescope = Telescope(args.tel_id, date, args.config)
telescope = Telescope(args.tel_id, date, args.config, no_gainsel=args.no_gainsel)

log.info(f"Processing {args.tel_id}...")

Expand Down
Loading
Loading