`inspect_failed_jobs.py` helper script for batch-diagnosing job failures #147

janosh · 2024-07-11T22:45:53Z

janosh
Jul 11, 2024

in case it helps anyone else, here's a Python script that fetches the last --lines of
std_err.txt, queue.err and custodian.json for the --njobs most recently failed jobs and renders them to the terminable or more searchable HTML (when passing --html). the intended use case is to get a quick idea of the prevalence of different kinds of failure modes.

has to be run on the cluster where the jobs were ran in order to read the above files.

Example invocation

python inspect_failed_jobs.py --project proj_name --html --njobs 50

Example output

Next Steps

not sure yet, having a GUI like fireworks would be nice but this very far from that and uses just static HTML (i.e. a very different approach though much more than this 1st step could already be done with that)

inspect_failed_jobs.py

import argparse
import json
import subprocess
from datetime import datetime
from pathlib import Path
from typing import Any

from jobflow_remote import JobController
from jobflow_remote.jobs.state import JobState

__author__ = "Janosh Riebesell"
__date__ = "2024-07-11"

# known caveats:
# - copy clipboard doesn't work in VS Code at least on macOS due to Live Preview extension permission issue
# - opening in Live Preview requires the Live Preview extension to be installed
# - links to run directories don't work with VS Code yet (as in revealing that folder in the VS Code file explorer)


def get_recent_failed_jobs(
    job_controller: JobController, n_jobs: int = 10
) -> list[dict[str, Any]]:
    """Get the n most recent failed jobs with end_time."""
    job_coll = job_controller.jobs
    query = {"state": JobState.FAILED.value, "end_time": {"$exists": True}}
    return list(job_coll.find(query).sort("end_time", -1).limit(n_jobs))


def get_file_content(file_path: Path, last_lines: int = -1) -> str:
    """Get the content of a file, optionally limited to the last few lines."""
    if file_path.exists():
        with open(file_path) as file:
            lines = file.readlines()
            content = "".join(lines) if last_lines == -1 else "".join(lines[-last_lines:])
            return content.strip() if content.strip() else f"[empty {file_path.name}]"
    else:
        return f"No {file_path.name} found"


def get_custodian_content(file_path: Path) -> str:
    """Get the content of the custodian.json file, removing the 'job' key."""
    if file_path.exists():
        with open(file_path) as file:
            try:
                data = json.load(file)[0]
                data.pop("job", None)  # Remove the 'job' key if it exists
                return json.dumps(data, indent=2)
            except json.JSONDecodeError:
                return "Error: Invalid JSON in custodian.json"
    else:
        return "No custodian.json found"


def generate_html(
    failed_jobs: list[dict[str, Any]], last_lines: int, proj_name: str
) -> str:
    """Generate an HTML report of failed jobs with collapsible code fences."""
    num_jobs = len(failed_jobs)
    current_date = datetime.now().astimezone().strftime("%Y-%m-%d %H:%M:%S")

    html_content = f"""
    <html>
    <head>
        <style>
            body {{
                font-family: Arial, sans-serif;
                transition: background-color 0.3s, color 0.3s;
                line-height: 1.6;
                background-color: #1e1e1e;
                color: #e0e0e0;
            }}
            body.light-mode {{
                background-color: #ffffff;
                color: #000000;
            }}
            .job {{ margin-bottom: 20px; border: 1px solid #444; padding: 15px; border-radius: 5px; }}
            .job-info {{ margin-bottom: 10px; }}
            .file-content {{
                background-color: #2d2d2d;
                border-radius: 3px;
                overflow: auto;
                margin: 10px 0;
                padding: 10px;
                position: relative;
            }}
            body.light-mode .file-content {{ background-color: #f8f8f8; }}
            details {{ margin-bottom: 10px; }}
            summary {{ cursor: pointer; padding: 5px; background-color: #333; border-radius: 3px; }}
            body.light-mode summary {{ background-color: #ddd; }}
            .info-copy-btn {{
                cursor: pointer;
                background: none;
                border: none;
                font-size: 14px;
                color: #aaa;
                padding: 2px 5px;
                transition: color 0.3s;
                margin-left: 5px;
            }}
            .info-copy-btn:hover {{ color: #fff; }}
            body.light-mode .info-copy-btn {{ color: #666; }}
            body.light-mode .info-copy-btn:hover {{ color: #000; }}
            #mode-toggle {{
                position: fixed;
                top: 10px;
                right: 10px;
                background: none;
                border: 1px solid #444;
                padding: 5px 10px;
                cursor: pointer;
                border-radius: 3px;
                color: #e0e0e0;
            }}
            body.light-mode #mode-toggle {{
                border-color: #ddd;
                color: #000000;
            }}
            a {{ color: #0077cc; text-decoration: none; }}
        </style>
        <script>
            function to_clipboard(text, button) {{
                navigator.clipboard.writeText(text).then(() => {{
                    const originalContent = button.textContent;
                    button.textContent = '✓';
                    button.disabled = true;
                    setTimeout(() => {{
                        button.textContent = originalContent;
                        button.disabled = false;
                    }}, 1000);
                }}).catch(err => {{
                    console.error('Failed to copy: ', err);
                }});
            }}

            function toggleMode() {{
                const body_classes = document.body.classList;
                body_classes.toggle('light-mode');
                const modeToggle = document.getElementById('mode-toggle');
                modeToggle.textContent = body_classes.contains('light-mode') ? '🌙' : '☀️';
            }}
        </script>
    </head>
    <body>
        <h1>{num_jobs} failed in {proj_name} as of {current_date}</h1>
        <button id="mode-toggle" onclick="toggleMode()">☀️</button>
    """

    for job in failed_jobs:
        run_dir = job.get("run_dir")
        end_time = (
            job["end_time"].astimezone().strftime("%Y-%m-%d %H:%M:%S")
            if job["end_time"]
            else "N/A"
        )
        db_id = job["db_id"]
        html_content += f"""
        <div class="job">
            <div class="job-info">
                <strong>Job ID:</strong> {db_id}
                <button class="info-copy-btn" onclick="to_clipboard('{db_id}', this)">📋</button>
                <button class="info-copy-btn" onclick="to_clipboard('jf -p {proj_name} job info {db_id}', this)">🔗</button><br>
                <strong>End Time:</strong> {end_time}
                <button class="info-copy-btn" onclick="to_clipboard('{end_time}', this)">📋</button><br>
                <strong>Run Directory:</strong> <a href="vscode://file/{run_dir}">{run_dir}</a>
                <button class="info-copy-btn" onclick="to_clipboard('{run_dir}', this)">📋</button>
            </div>
        """
        if run_dir:
            run_dir_path = Path(run_dir)
            std_err = get_file_content(
                run_dir_path / "std_err.txt", last_lines=last_lines
            )
            queue_err = get_file_content(run_dir_path / "queue.err")
            custodian_content = get_custodian_content(run_dir_path / "custodian.json")

            html_content += (
                f"""
            <details>
                <summary>std_err.txt (last {last_lines} lines)</summary>
                <div class="file-content">
                    <pre><code>{std_err}</code></pre>
                    <button class="copy-btn" onclick="to_clipboard(`{std_err}`, this)">📋</button>
                </div>
            </details>
            """
                if std_err != "[empty std_err.txt]"
                else ""
            )

            html_content += (
                f"""
            <details>
                <summary>queue.err</summary>
                <div class="file-content">
                    <pre><code>{queue_err}</code></pre>
                    <button class="copy-btn" onclick="to_clipboard(`{queue_err}`, this)">📋</button>
                </div>
            </details>
            """
                if queue_err != "[empty queue.err]"
                else ""
            )

            html_content += f"""
            <details>
                <summary>custodian.json</summary>
                <div class="file-content">
                    <pre><code>{custodian_content}</code></pre>
                    <button class="copy-btn" onclick="to_clipboard(`{custodian_content}`, this)">📋</button>
                </div>
            </details>
            """
        else:
            html_content += "<div>No run directory found for this job</div>"
        html_content += "</div>"

    html_content += "</body></html>"
    return html_content


# %%
def open_in_live_preview(html_content: str) -> None:
    """Create a temporary HTML file and open it in VS Code's Live Preview."""
    temp_file = Path.cwd() / "tmp-job-errors.html"
    with open(temp_file, mode="w") as file:
        file.write(html_content)

    # Use VS Code's command line interface to open the file in Live Preview
    subprocess.run(["code", "--goto", f"{temp_file}:1"], check=False)  # noqa: S603, S607


parser = argparse.ArgumentParser(description="Fetch recent failed job errors")
parser.add_argument(
    "-n", "--njobs", type=int, default=5, help="Number of recent failed jobs to check"
)
parser.add_argument(
    "--lines", type=int, default=10, help="Number of lines to print from std_err.txt"
)
parser.add_argument("-p", "--project", type=str, required=True, help="Project name")
parser.add_argument(
    "--html",
    action="store_true",
    help="Generate HTML output and open in Live Preview",
)
args = parser.parse_args()

job_controller = JobController.from_project_name(args.project)
failed_jobs = get_recent_failed_jobs(job_controller, args.njobs)

if args.html:
    html_content = generate_html(failed_jobs, args.lines, args.project)
    open_in_live_preview(html_content)
else:
    current_date = datetime.now().astimezone().strftime("%Y-%m-%d %H:%M:%S")
    print(f"{len(failed_jobs)} failed in {args.project} as of {current_date}\n")
    for job in failed_jobs:
        run_dir = job.get("run_dir")
        end_time = (
            job["end_time"].astimezone().strftime("%Y-%m-%d %H:%M:%S")
            if job["end_time"]
            else "N/A"
        )
        if run_dir:
            print(f"\nJob ID: {job['db_id']}")
            print(f"End Time: {end_time}")
            print(f"Run Directory: vscode://file/{run_dir}")
            print(f"\nLast {args.lines} lines of std_err.txt in {run_dir}:")
            print(get_file_content(Path(run_dir) / "std_err.txt", last_lines=args.lines))
            print(f"\nqueue.err in {run_dir}:")
            print(get_file_content(Path(run_dir) / "queue.err"))
            print(f"\ncustodian.json in {run_dir}:")
            print(get_custodian_content(Path(run_dir) / "custodian.json"))
        else:
            print(f"\nNo run directory found for job {job['db_id']}")

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

`inspect_failed_jobs.py` helper script for batch-diagnosing job failures #147

{{title}}

{{editor}}'s edit

{{editor}}'s edit

Replies: 0 comments

Select a reply

inspect_failed_jobs.py helper script for batch-diagnosing job failures #147

janosh Jul 11, 2024

Replies: 0 comments

`inspect_failed_jobs.py` helper script for batch-diagnosing job failures #147

janosh
Jul 11, 2024