Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Easy] 5-minute timeout for smoke test tutorials #1707

Closed
wants to merge 2 commits into from
Closed

[Easy] 5-minute timeout for smoke test tutorials #1707

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ce10dbf
5-minute timeout for smoke test tutorials
esantorella Feb 27, 2023
886e86e
Apply suggestions from code review
esantorella Feb 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 12 additions & 5 deletions scripts/run_tutorials.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,9 @@ def parse_ipynb(file: Path) -> str:
return script


def run_script(script: str, env: Optional[Dict[str, str]] = None) -> None:
def run_script(
script: str, timeout_minutes: int, env: Optional[Dict[str, str]] = None
) -> None:
# need to keep the file around & close it so subprocess does not run into I/O issues
with tempfile.NamedTemporaryFile(delete=False) as tf:
tf_name = tf.name
Expand All @@ -87,7 +89,7 @@ def run_script(script: str, env: Optional[Dict[str, str]] = None) -> None:
capture_output=True,
text=True,
env=env,
timeout=1800, # Count runtime >30 minutes as a failure
timeout=timeout_minutes * 60,
)
os.remove(tf_name)
return run_out
Expand All @@ -100,16 +102,22 @@ def run_tutorial(
Runs the tutorial in a subprocess, catches any raised errors and returns
them as a string, and returns runtime and memory information as a dict.
"""
timeout_minutes = 5 if smoke_test else 30
script = parse_ipynb(tutorial)
tic = time.monotonic()
print(f"Running tutorial {tutorial.name}.")
env = {"SMOKE_TEST": "True"} if smoke_test else None
try:
mem_usage, run_out = memory_usage(
(run_script, (script,), {"env": env}), retval=True, include_children=True
(run_script, (script, timeout_minutes), {"env": env}),
retval=True,
include_children=True,
)
except subprocess.TimeoutExpired:
error = f"Tutorial {tutorial.name} exceeded the maximum runtime of 30 minutes."
error = (
f"Tutorial {tutorial.name} exceeded the maximum runtime of "
f"{timeout_minutes} minutes."
)
return error, {}

try:
Expand Down Expand Up @@ -204,7 +212,6 @@ def run_tutorials(
df.loc[tutorial.name, "ran_successfully"] = True
for k in ["runtime", "start_mem", "max_mem"]:
df.loc[tutorial.name, k] = performance_info[k]
print(df)

if num_errors > 0:
raise RuntimeError(
Expand Down