From 42fabc39456d0c96faec9e73d77ca733896a258b Mon Sep 17 00:00:00 2001 From: John Sirois Date: Fri, 22 Mar 2024 10:29:27 -0700 Subject: [PATCH] Fix `science doc open`. (#60) This now uses a local HTTP server to serve docs which is much more straightforward than trying to hack around all the quirks trying to serve a site via file:// URLs brings. --- CHANGES.md | 5 + noxfile.py | 17 ++- science/__init__.py | 2 +- science/commands/doc.py | 229 ++++++++++++++++++++++++++++++++++++++++ science/exe.py | 59 +++++++++-- 5 files changed, 299 insertions(+), 13 deletions(-) create mode 100644 science/commands/doc.py diff --git a/CHANGES.md b/CHANGES.md index 24ddcc2..b4ecf62 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,10 @@ # Release Notes +## 0.3.3 + +Fix `science doc open` to use a local documentation server. This works around all the problems +you encounter trying to get a documentation site to function like the production via file:// URLs. + ## 0.3.2 Upgrade the science internal Python distribution to [PBS][PBS] CPython 3.12.2 and perform Mac diff --git a/noxfile.py b/noxfile.py index c4b97d1..d29d747 100644 --- a/noxfile.py +++ b/noxfile.py @@ -7,6 +7,7 @@ import json import os import shutil +import subprocess from functools import wraps from pathlib import Path from typing import Any, Callable, Collection, Iterable, TypeVar, cast @@ -37,7 +38,12 @@ def run_pex(session: Session, script, *args, silent=False, **env) -> Any | None: session.run("pex", PEX_REQUIREMENT, "--venv", "--sh-boot", "-o", str(pex_pex)) session.run("python", "-m", "pip", "uninstall", "-y", "pex") return session.run( - "python", str(pex_pex), *args, env={"PEX_SCRIPT": script, **env}, silent=silent + "python", + str(pex_pex), + *args, + env={"PEX_SCRIPT": script, **env}, + silent=silent, + stderr=subprocess.DEVNULL if silent else None, ) @@ -310,8 +316,8 @@ def _run_sphinx(session: Session, builder_name: str) -> Path: @python_session(include_project=True) -def doc(session: Session) -> None: - _run_sphinx(session, builder_name="html") +def doc(session: Session) -> Path: + return _run_sphinx(session, builder_name="html") @python_session(include_project=True, extra_reqs=["doc"]) @@ -322,7 +328,10 @@ def linkcheck(session: Session) -> None: @python_session() def run(session: Session) -> None: science_pyz = create_zipapp(session) - session.run("python", str(science_pyz), *session.posargs) + doc_path = doc(session) + session.run( + "python", str(science_pyz), *session.posargs, env={"SCIENCE_DOC_LOCAL": str(doc_path)} + ) def _package(session: Session, docsite: Path, *extra_lift_args: str) -> None: diff --git a/science/__init__.py b/science/__init__.py index 5e25073..a66e7b6 100644 --- a/science/__init__.py +++ b/science/__init__.py @@ -3,6 +3,6 @@ from packaging.version import Version -__version__ = "0.3.2" +__version__ = "0.3.3" VERSION = Version(__version__) diff --git a/science/commands/doc.py b/science/commands/doc.py new file mode 100644 index 0000000..d1f5c1c --- /dev/null +++ b/science/commands/doc.py @@ -0,0 +1,229 @@ +# Copyright 2024 Science project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +import json +import logging +import os +import re +import shlex +import subprocess +import sys +import time +from dataclasses import dataclass +from datetime import datetime +from functools import cache +from pathlib import Path, PurePath + +import psutil + +from science import __version__ +from science.cache import science_cache +from science.platform import Platform + +logger = logging.getLogger(__name__) + +# SCIE -> SKI (rotate right) -> ISK (substitute 1) -> 1SK (ascii decimal) -> 1 83 75 +SERVER_DEFAULT_PORT = 18375 + +SERVER_NAME = f"Science v{__version__} docs HTTP server" + + +def _server_dir(ensure: bool = False) -> Path: + server_dir = science_cache() / "docs" / "server" / __version__ + if ensure: + server_dir.mkdir(parents=True, exist_ok=True) + return server_dir + + +def _render_unix_time(unix_time: float) -> str: + return datetime.fromtimestamp(unix_time).strftime("%Y-%m-%d %H:%M:%S") + + +@dataclass(frozen=True) +class ServerInfo: + url: str + pid: int + create_time: float + + def __str__(self) -> str: + return f"{self.url} @ {self.pid} (started at {_render_unix_time(self.create_time)})" + + +@dataclass(frozen=True) +class Pidfile: + @classmethod + def _pidfile(cls, ensure: bool = False) -> Path: + return _server_dir(ensure) / "pidfile" + + @classmethod + def load(cls) -> Pidfile | None: + pidfile = cls._pidfile() + try: + with pidfile.open() as fp: + data = json.load(fp) + return cls( + ServerInfo(url=data["url"], pid=data["pid"], create_time=data["create_time"]) + ) + except (OSError, ValueError, KeyError) as e: + logger.debug(f"Failed to load {SERVER_NAME} pid file from {pidfile}: {e}") + return None + + @staticmethod + def _read_url(server_log: Path, timeout: float) -> str | None: + # N.B.: The simple http server module output is: + # Serving HTTP on 0.0.0.0 port 33539 (http://0.0.0.0:33539/) ... + # Or: + # Serving HTTP on :: port 33539 (http://[::]:33539/) ... + # Etc. + + start = time.time() + while time.time() - start < timeout: + with server_log.open() as fp: + for line in fp: + match = re.search(r"Serving HTTP on \S+ port (?P\d+) ", line) + if match: + port = match.group("port") + return "http://localhost:{port}".format(port=port) + return None + + @classmethod + def record(cls, server_log: Path, pid: int, timeout: float = 5.0) -> Pidfile | None: + url = cls._read_url(server_log, timeout) + if not url: + return None + + try: + create_time = psutil.Process(pid).create_time() + except psutil.Error: + return None + + with cls._pidfile(ensure=True).open("w") as fp: + json.dump(dict(url=url, pid=pid, create_time=create_time), fp, indent=2, sort_keys=True) + return cls(ServerInfo(url=url, pid=pid, create_time=create_time)) + + server_info: ServerInfo + + @property + @cache + def _process(self) -> psutil.Process | None: + try: + process = psutil.Process(self.server_info.pid) + except psutil.Error: + return None + else: + try: + create_time = process.create_time() + except psutil.Error: + return None + else: + if create_time != self.server_info.create_time: + try: + command = shlex.join(process.cmdline()) + except psutil.Error: + command = "" + logger.debug( + f"Pid has rolled over for {self.server_info} to {command} (started at " + f"{_render_unix_time(create_time)})" + ) + return None + return process + + def alive(self) -> bool: + if process := self._process: + try: + return process.is_running() + except psutil.Error: + pass + return False + + def kill(self) -> None: + if process := self._process: + process.terminate() + + +@dataclass(frozen=True) +class LaunchResult: + server_info: ServerInfo + already_running: bool + + +@dataclass(frozen=True) +class LaunchError(Exception): + """Indicates an error launching the doc server.""" + + log: PurePath + additional_msg: str | None = None + + def __str__(self) -> str: + lines = ["Error launching docs server."] + if self.additional_msg: + lines.append(self.additional_msg) + lines.append("See the log at {log} for more details.".format(log=self.log)) + return os.linesep.join(lines) + + +def launch( + document_root: PurePath, port: int = SERVER_DEFAULT_PORT, timeout: float = 5.0 +) -> LaunchResult: + pidfile = Pidfile.load() + if pidfile and pidfile.alive(): + return LaunchResult(server_info=pidfile.server_info, already_running=True) + + log = _server_dir(ensure=True) / "log.txt" + + # N.B.: We set up line buffering for the process pipes as well as the underlying Python running + # the http server to ensure we can observe the `Serving HTTP on ...` line we need to grab the + # ephemeral port chosen. + env = {**os.environ, "PYTHONUNBUFFERED": "1"} + with log.open("w") as fp: + # Not proper daemonization, but good enough. + daemon_kwargs = ( + { + # The subprocess.{DETACHED_PROCESS,CREATE_NEW_PROCESS_GROUP} attributes are only + # defined on Windows. + "creationflags": ( + subprocess.DETACHED_PROCESS # type: ignore[attr-defined] + | subprocess.CREATE_NEW_PROCESS_GROUP # type: ignore[attr-defined] + ) + } + if Platform.current() is Platform.Windows_x86_64 + else {"preexec_fn": os.setsid} + ) + process = subprocess.Popen( + args=[sys.executable, "-m", "http.server", str(port)], + env=env, + cwd=document_root, + bufsize=1, + stdout=fp.fileno(), + stderr=subprocess.STDOUT, + close_fds=True, + **daemon_kwargs, + ) + + pidfile = Pidfile.record(server_log=log, pid=process.pid, timeout=timeout) + if not pidfile: + try: + psutil.Process(process.pid).kill() + except psutil.Error as e: + if not isinstance(e, psutil.NoSuchProcess): + raise LaunchError( + log, + additional_msg=( + f"Also failed to kill the partially launched server at pid {process.pid}: " + f"{e}" + ), + ) + raise LaunchError(log) + return LaunchResult(server_info=pidfile.server_info, already_running=False) + + +def shutdown() -> ServerInfo | None: + pidfile = Pidfile.load() + if not pidfile or not pidfile.alive(): + return None + + logger.debug(f"Killing {SERVER_NAME} {pidfile.server_info}") + pidfile.kill() + return pidfile.server_info diff --git a/science/exe.py b/science/exe.py index d7889e7..08c2653 100644 --- a/science/exe.py +++ b/science/exe.py @@ -29,6 +29,9 @@ from science import __version__, providers from science.commands import build, lift from science.commands.complete import Shell +from science.commands.doc import SERVER_NAME, LaunchError +from science.commands.doc import launch as launch_doc_server +from science.commands.doc import shutdown as shutdown_doc_server from science.commands.lift import AppInfo, FileMapping, LiftConfig, PlatformInfo from science.config import parse_config from science.context import DocConfig, ScienceConfig @@ -52,11 +55,9 @@ def _log_fatal( always_include_backtrace: bool, ) -> None: if always_include_backtrace or not isinstance(value, InputError): - click.secho("".join(traceback.format_tb(tb)), fg="yellow", file=sys.stderr, nl=False) - click.secho( - f"{type_.__module__}.{type_.__qualname__}: ", fg="yellow", file=sys.stderr, nl=False - ) - click.secho(value, fg="red", file=sys.stderr) + click.secho("".join(traceback.format_tb(tb)), fg="yellow", err=True, nl=False) + click.secho(f"{type_.__module__}.{type_.__qualname__}: ", fg="yellow", err=True, nl=False) + click.secho(value, fg="red", err=True) SEE_MANIFEST_HELP = ( @@ -202,12 +203,43 @@ def _doc(ctx: click.Context, site: str, local: Path | None) -> None: ) @click.argument("page", default=None, required=False) @pass_doc -def _open_doc(doc: DocConfig, remote: bool, page: str | None = None) -> None: +@click.pass_context +def _open_doc(ctx: click.Context, doc: DocConfig, remote: bool, page: str | None = None) -> None: """Opens the local documentation in a browser. - If an optional page argument is supplied, that page will be opened instead of the default doc site page. + If an optional page argument is supplied, that page will be opened instead of the default doc + site page. + + Documentation is served by a local HTTP server which you can shut down with `science doc close`. """ - url = doc.site if remote else f"file://{doc.local}" + if remote or not doc.local: + url = doc.site + else: + try: + launch_result = launch_doc_server(document_root=doc.local) + except LaunchError: + try: + launch_result = launch_doc_server(document_root=doc.local, port=0) + except LaunchError as e: + with open(e.log) as fp: + for line in fp: + logger.error(line.rstrip()) + logger.fatal(f"Failed to launch {SERVER_NAME}.") + ctx.exit(1) + return + + url = launch_result.server_info.url + if launch_result.already_running: + click.secho( + f"Using {SERVER_NAME} already running at {launch_result.server_info}.", + fg="cyan", + err=True, + ) + else: + click.secho( + f"Launched {SERVER_NAME} at {launch_result.server_info}", fg="green", err=True + ) + if not page: if not remote: url = f"{url}/index.html" @@ -215,9 +247,20 @@ def _open_doc(doc: DocConfig, remote: bool, page: str | None = None) -> None: url_info = urlparse(url) page = f"{page}.html" if not PurePath(page).suffix else page url = urlunparse(url_info._replace(path=f"{url_info.path}/{page}")) + click.launch(url) +@_doc.command(name="close") +def _close_doc() -> None: + """Shuts down the local documentation server.""" + server_info = shutdown_doc_server() + if server_info: + click.secho(f"Shut down the {SERVER_NAME} at {server_info}.", fg="green", err=True) + else: + click.secho("No documentation server was running.", fg="cyan", err=True) + + @_main.group(cls=DYMGroup, name="provider") def _provider() -> None: """Perform operations against provider plugins."""