📝 Update markdown includes format #1254

tiangolo · 2024-12-22T10:50:07Z

📝 Update markdown includes format

This would solve / related to: #1150

The changes were generated with a script that was developed iteratively, updating it and fixing it to account for all the cases. It's much better and easier to automatize the process and review the results consistently than to review changes made by hand, file by file.

Here's the script:

import re
from pathlib import Path

import typer
from pydantic import BaseModel


class InvalidFormatError(Exception):
    """
    Invalid format
    """


def get_hl_lines(old_first_line: str) -> list[tuple[int, int]] | None:
    match = re.search(r"hl_lines=\"(.*)\"", old_first_line)
    if match is None:
        return None
    old_hl = match.group(1)
    assert isinstance(old_hl, str)
    hl_lines = []
    for line in old_hl.split():
        if "-" in line:
            start, _, end = line.partition("-")
            hl_lines.append((int(start), int(end)))
        else:
            hl_lines.append((int(line), int(line)))
    return hl_lines


class FileReference(BaseModel):
    file: str
    lines: tuple[int, int] | None = None


def get_file_reference(old_reference_line: str) -> FileReference:
    file_match = re.search(r"./(.*).py", old_reference_line)
    if file_match is None:
        raise InvalidFormatError("Invalid file reference.")
    file_ref = file_match.group(0)
    line_match = re.search(r"\[ln:(.*)\]", old_reference_line)
    if line_match is None:
        return FileReference(file=file_ref)
    line_ref = line_match.group(1)
    if "-" not in line_ref:
        return FileReference(file=file_ref, lines=(int(line_ref), int(line_ref)))
    start, end = line_ref.split("-")
    return FileReference(file=file_ref, lines=(int(start), int(end)))


PYTHON_BLOCK = "```Python"
END_BLOCK = "```"
PYTHON_TAB = ("//// tab | Python", "//// tab | 🐍", "//// tab | 파이썬")
TAB_END = "////"
FULL_PREVIEW = "/// details | 👀 Full file preview"
END_PREVIEW = "///"


class FencedPython(BaseModel):
    first_line: str
    body: list[str]

    def __str__(self) -> str:
        formatted_body_lines = format_fenced_python(self)
        body = "\n".join(formatted_body_lines)
        return body

    def contains_includes(self) -> bool:
        return any(line.startswith("{!") for line in self.body)


class Details(BaseModel):
    first_line: str
    body: list[str]

    def __str__(self) -> str:
        for line in self.body:
            if line.startswith("{!"):
                ref = get_file_reference(line)
                return "{* " + ref.file + " ln[0] *}"
        return ""


class Tab(BaseModel):
    first_line: str
    body: list[str]

    def __str__(self) -> str:
        body = "\n".join(self.body).strip()
        return f"{self.first_line}\n\n{body}\n\n{TAB_END}"


class TabGroup(BaseModel):
    tabs: list[Tab]
    details: list[Details] = []

    def __str__(self) -> str:
        return format_tab_group(self)


def format_fenced_python(block: FencedPython) -> list[str]:
    if not block.contains_includes():
        return [block.first_line] + block.body + [END_BLOCK]

    old_hl_ines = get_hl_lines(block.first_line)
    refs: list[tuple[int, FileReference]] = []
    offset = 0
    for i, line in enumerate(block.body):
        if line.startswith("{!"):
            ref = get_file_reference(line)
            refs.append((offset + i, ref))
            if ref.lines is not None:
                offset += ref.lines[1] - ref.lines[0]
    first_ref = refs[0][1]
    assert all(ref[1].file == first_ref.file for ref in refs)
    new_hl_lines: list[tuple[int, int]] = []
    for hl_line in old_hl_ines or []:
        for o, ref in refs:
            assert ref.lines is not None
            ln_start, ln_end = ref.lines
            ln_start_render = o + 1
            ln_end_render = ln_end - ln_start + ln_start_render
            old_hl_start, old_hl_end = hl_line
            if old_hl_start >= ln_start_render and old_hl_end <= ln_end_render:
                new_hl_start = old_hl_start - o + ln_start - 1
                new_hl_end = old_hl_end - o + ln_start - 1
                new_hl_lines.append((new_hl_start, new_hl_end))
                break
    file_ref = first_ref.file
    lns = [ref[1].lines for ref in refs if ref[1].lines is not None]
    new_lns = []
    for ln in lns:
        if ln[0] == ln[1]:
            new_lns.append(str(ln[0]))
            continue
        new_lns.append(f"{ln[0]}:{ln[1]}")
    new_ln_str = ",".join(new_lns)
    new_hl = []
    for start, end in new_hl_lines:
        if start == end:
            new_hl.append(str(start))
            continue
        new_hl.append(f"{start}:{end}")
    new_hl_str = ",".join(new_hl)
    new_ln_instruction = f"ln[{new_ln_str}]" if new_ln_str else ""
    new_hl_instruction = f"hl[{new_hl_str}]" if new_hl_str else ""
    new_include = "{* " + f"{file_ref}"
    if new_ln_instruction:
        new_include += f" {new_ln_instruction}"
    if new_hl_instruction:
        new_include += f" {new_hl_instruction}"
    new_include += " *}"

    return [new_include]


def format_tab_group(group: TabGroup) -> str:
    if not group.tabs and group.details:
        return str(group.details[0])
    first_tab = group.tabs[0]
    fenced_block: FencedPython | None = None
    for line in first_tab.body:
        if fenced_block:
            if line == END_BLOCK:
                if fenced_block.contains_includes():
                    # Return the first one, omit the rest
                    return str(fenced_block)
                return "\n\n".join(str(tab) for tab in group.tabs)
            fenced_block.body.append(line)
            continue
        if line and not line.startswith(PYTHON_BLOCK):
            return "\n\n".join(str(tab) for tab in group.tabs)
        if line.startswith(PYTHON_BLOCK):
            fenced_block = FencedPython(first_line=line, body=[])
    raise RuntimeError("Invalid tab group")


def process_lines(lines: list[str]) -> list[str | FencedPython | Tab | Details]:
    new_sections: list[str | FencedPython | Tab | Details] = []
    fenced_block: FencedPython | None = None
    tab: Tab | None = None
    details: Details | None = None

    for line in lines:
        if not fenced_block and not tab and not details:
            if line.startswith(PYTHON_BLOCK):
                fenced_block = FencedPython(first_line=line, body=[])
                continue
            if line.startswith(PYTHON_TAB):
                tab = Tab(first_line=line, body=[])
                continue
            if line.startswith(FULL_PREVIEW):
                details = Details(first_line=line, body=[])
                continue
            new_sections.append(line)
            continue
        if fenced_block:
            if line == END_BLOCK:
                new_sections.append(fenced_block)
                fenced_block = None
                continue
            fenced_block.body.append(line)
            continue
        if tab:
            if line == TAB_END:
                new_sections.append(tab)
                tab = None
                continue
            tab.body.append(line)
            continue
        if details:
            if line == END_PREVIEW:
                new_sections.append(details)
                details = None
                continue
            details.body.append(line)
    return new_sections


def process_fragments(
    fragments: list[str | FencedPython | Tab | Details],
) -> list[str | FencedPython | TabGroup | Details]:
    new_fragments: list[str | FencedPython | TabGroup | Details] = []
    tab_group: TabGroup | None = None
    for fragment in fragments:
        if not tab_group:
            if isinstance(fragment, Tab):
                tab_group = TabGroup(tabs=[fragment])
                continue
            if isinstance(fragment, Details):
                tab_group = TabGroup(tabs=[], details=[fragment])
                continue
            new_fragments.append(fragment)
            continue
        if tab_group:
            if isinstance(fragment, Tab):
                tab_group.tabs.append(fragment)
                continue
            if isinstance(fragment, Details):
                tab_group.details.append(fragment)
                continue
            if fragment == "":
                continue
            new_fragments.append(tab_group)
            new_fragments.append("")
            tab_group = None
            new_fragments.append(fragment)
            continue
    if tab_group:
        new_fragments.append(tab_group)
    return new_fragments


def process_file(file_path: Path) -> None:
    lines = file_path.read_text().splitlines()
    sections = process_lines(lines)
    groups = process_fragments(sections)
    group_str = [str(group) for group in groups]
    file_path.write_text("\n".join(group_str).strip() + "\n")


skip_file_names = ["code-structure.md"]


def main(file_path: Path = None) -> None:
    if file_path:
        process_file(file_path)
        return
    files_with_errors = []
    for f in Path("docs/").glob("**/*.md"):
        if f.name in skip_file_names:
            continue
        try:
            process_file(f)
        except Exception as e:
            print(f"An error occurred in file {f}: {e}")
            files_with_errors.append(f)
    print("Files with errors:")
    for f in files_with_errors:
        print(f)


if __name__ == "__main__":
    typer.run(main)
    # file_path = Path("demo.md")
    # process_file(file_path)

Modified Pages to Review

github-actions · 2024-12-22T11:15:59Z

📝 Docs preview for commit e962393 at: https://885cf9ae.sqlmodel.pages.dev

Modified Pages

github-actions · 2024-12-22T13:20:04Z

📝 Docs preview for commit ed2a90c at: https://662a5e84.sqlmodel.pages.dev

Modified Pages

tiangolo added 3 commits December 22, 2024 10:34

📝 Update markdown includes format in first batch of docs

ff0e6e3

📝 Fix duplicate single lines

a174b24

📝 Update markdown include syntax for the rest of the docs

526e6db

github-actions bot added the docs Improvements or additions to documentation label Dec 22, 2024

⬆️ Upgrade docs dependency markdown-include-variants

e962393

📝 Update includes in corner cases

ed2a90c

tiangolo marked this pull request as ready for review December 22, 2024 14:24

tiangolo merged commit 5100200 into main Dec 22, 2024
26 checks passed

tiangolo deleted the includes branch December 22, 2024 14:30

tiangolo mentioned this pull request Dec 22, 2024

Update docs include syntax for source examples #1150

Closed

1 task

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

📝 Update markdown includes format #1254

📝 Update markdown includes format #1254

tiangolo commented Dec 22, 2024 •

edited

Loading

github-actions bot commented Dec 22, 2024

github-actions bot commented Dec 22, 2024

📝 Update markdown includes format #1254

📝 Update markdown includes format #1254

Conversation

tiangolo commented Dec 22, 2024 • edited Loading

Modified Pages to Review

github-actions bot commented Dec 22, 2024

Modified Pages

github-actions bot commented Dec 22, 2024

Modified Pages

tiangolo commented Dec 22, 2024 •

edited

Loading