Fix science doc open. (#60)

This now uses a local HTTP server to serve docs which is much more
straightforward than trying to hack around all the quirks trying to
serve a site via file:// URLs brings.

CHANGES.md

@@ -1,5 +1,10 @@
 # Release Notes
 
+## 0.3.3
+
+Fix `science doc open` to use a local documentation server. This works around all the problems
+you encounter trying to get a documentation site to function like the production via file:// URLs.
+
 ## 0.3.2
 
 Upgrade the science internal Python distribution to [PBS][PBS] CPython 3.12.2 and perform Mac

noxfile.py

@@ -7,6 +7,7 @@
 import json
 import os
 import shutil
+import subprocess
 from functools import wraps
 from pathlib import Path
 from typing import Any, Callable, Collection, Iterable, TypeVar, cast
@@ -37,7 +38,12 @@ def run_pex(session: Session, script, *args, silent=False, **env) -> Any | None:
         session.run("pex", PEX_REQUIREMENT, "--venv", "--sh-boot", "-o", str(pex_pex))
         session.run("python", "-m", "pip", "uninstall", "-y", "pex")
     return session.run(
-        "python", str(pex_pex), *args, env={"PEX_SCRIPT": script, **env}, silent=silent
+        "python",
+        str(pex_pex),
+        *args,
+        env={"PEX_SCRIPT": script, **env},
+        silent=silent,
+        stderr=subprocess.DEVNULL if silent else None,
     )
 
 
@@ -310,8 +316,8 @@ def _run_sphinx(session: Session, builder_name: str) -> Path:
 
 
 @python_session(include_project=True)
-def doc(session: Session) -> None:
-    _run_sphinx(session, builder_name="html")
+def doc(session: Session) -> Path:
+    return _run_sphinx(session, builder_name="html")
 
 
 @python_session(include_project=True, extra_reqs=["doc"])
@@ -322,7 +328,10 @@ def linkcheck(session: Session) -> None:
 @python_session()
 def run(session: Session) -> None:
     science_pyz = create_zipapp(session)
-    session.run("python", str(science_pyz), *session.posargs)
+    doc_path = doc(session)
+    session.run(
+        "python", str(science_pyz), *session.posargs, env={"SCIENCE_DOC_LOCAL": str(doc_path)}
+    )
 
 
 def _package(session: Session, docsite: Path, *extra_lift_args: str) -> None:

science/__init__.py

@@ -3,6 +3,6 @@
 
 from packaging.version import Version
 
-__version__ = "0.3.2"
+__version__ = "0.3.3"
 
 VERSION = Version(__version__)

science/commands/doc.py

@@ -0,0 +1,229 @@
+# Copyright 2024 Science project contributors.
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import re
+import shlex
+import subprocess
+import sys
+import time
+from dataclasses import dataclass
+from datetime import datetime
+from functools import cache
+from pathlib import Path, PurePath
+
+import psutil
+
+from science import __version__
+from science.cache import science_cache
+from science.platform import Platform
+
+logger = logging.getLogger(__name__)
+
+# SCIE -> SKI (rotate right) -> ISK (substitute 1) -> 1SK (ascii decimal) -> 1 83 75
+SERVER_DEFAULT_PORT = 18375
+
+SERVER_NAME = f"Science v{__version__} docs HTTP server"
+
+
+def _server_dir(ensure: bool = False) -> Path:
+    server_dir = science_cache() / "docs" / "server" / __version__
+    if ensure:
+        server_dir.mkdir(parents=True, exist_ok=True)
+    return server_dir
+
+
+def _render_unix_time(unix_time: float) -> str:
+    return datetime.fromtimestamp(unix_time).strftime("%Y-%m-%d %H:%M:%S")
+
+
+@dataclass(frozen=True)
+class ServerInfo:
+    url: str
+    pid: int
+    create_time: float
+
+    def __str__(self) -> str:
+        return f"{self.url} @ {self.pid} (started at {_render_unix_time(self.create_time)})"
+
+
+@dataclass(frozen=True)
+class Pidfile:
+    @classmethod
+    def _pidfile(cls, ensure: bool = False) -> Path:
+        return _server_dir(ensure) / "pidfile"
+
+    @classmethod
+    def load(cls) -> Pidfile | None:
+        pidfile = cls._pidfile()
+        try:
+            with pidfile.open() as fp:
+                data = json.load(fp)
+            return cls(
+                ServerInfo(url=data["url"], pid=data["pid"], create_time=data["create_time"])
+            )
+        except (OSError, ValueError, KeyError) as e:
+            logger.debug(f"Failed to load {SERVER_NAME} pid file from {pidfile}: {e}")
+            return None
+
+    @staticmethod
+    def _read_url(server_log: Path, timeout: float) -> str | None:
+        # N.B.: The simple http server module output is:
+        # Serving HTTP on 0.0.0.0 port 33539 (http://0.0.0.0:33539/) ...
+        # Or:
+        # Serving HTTP on :: port 33539 (http://[::]:33539/) ...
+        # Etc.
+
+        start = time.time()
+        while time.time() - start < timeout:
+            with server_log.open() as fp:
+                for line in fp:
+                    match = re.search(r"Serving HTTP on \S+ port (?P<port>\d+) ", line)
+                    if match:
+                        port = match.group("port")
+                        return "http://localhost:{port}".format(port=port)
+        return None
+
+    @classmethod
+    def record(cls, server_log: Path, pid: int, timeout: float = 5.0) -> Pidfile | None:
+        url = cls._read_url(server_log, timeout)
+        if not url:
+            return None
+
+        try:
+            create_time = psutil.Process(pid).create_time()
+        except psutil.Error:
+            return None
+
+        with cls._pidfile(ensure=True).open("w") as fp:
+            json.dump(dict(url=url, pid=pid, create_time=create_time), fp, indent=2, sort_keys=True)
+        return cls(ServerInfo(url=url, pid=pid, create_time=create_time))
+
+    server_info: ServerInfo
+
+    @property
+    @cache
+    def _process(self) -> psutil.Process | None:
+        try:
+            process = psutil.Process(self.server_info.pid)
+        except psutil.Error:
+            return None
+        else:
+            try:
+                create_time = process.create_time()
+            except psutil.Error:
+                return None
+            else:
+                if create_time != self.server_info.create_time:
+                    try:
+                        command = shlex.join(process.cmdline())
+                    except psutil.Error:
+                        command = "<unknown command line>"
+                    logger.debug(
+                        f"Pid has rolled over for {self.server_info} to {command} (started at "
+                        f"{_render_unix_time(create_time)})"
+                    )
+                    return None
+                return process
+
+    def alive(self) -> bool:
+        if process := self._process:
+            try:
+                return process.is_running()
+            except psutil.Error:
+                pass
+        return False
+
+    def kill(self) -> None:
+        if process := self._process:
+            process.terminate()
+
+
+@dataclass(frozen=True)
+class LaunchResult:
+    server_info: ServerInfo
+    already_running: bool
+
+
+@dataclass(frozen=True)
+class LaunchError(Exception):
+    """Indicates an error launching the doc server."""
+
+    log: PurePath
+    additional_msg: str | None = None
+
+    def __str__(self) -> str:
+        lines = ["Error launching docs server."]
+        if self.additional_msg:
+            lines.append(self.additional_msg)
+        lines.append("See the log at {log} for more details.".format(log=self.log))
+        return os.linesep.join(lines)
+
+
+def launch(
+    document_root: PurePath, port: int = SERVER_DEFAULT_PORT, timeout: float = 5.0
+) -> LaunchResult:
+    pidfile = Pidfile.load()
+    if pidfile and pidfile.alive():
+        return LaunchResult(server_info=pidfile.server_info, already_running=True)
+
+    log = _server_dir(ensure=True) / "log.txt"
+
+    # N.B.: We set up line buffering for the process pipes as well as the underlying Python running
+    # the http server to ensure we can observe the `Serving HTTP on ...` line we need to grab the
+    # ephemeral port chosen.
+    env = {**os.environ, "PYTHONUNBUFFERED": "1"}
+    with log.open("w") as fp:
+        # Not proper daemonization, but good enough.
+        daemon_kwargs = (
+            {
+                # The subprocess.{DETACHED_PROCESS,CREATE_NEW_PROCESS_GROUP} attributes are only
+                # defined on Windows.
+                "creationflags": (
+                    subprocess.DETACHED_PROCESS  # type: ignore[attr-defined]
+                    | subprocess.CREATE_NEW_PROCESS_GROUP  # type: ignore[attr-defined]
+                )
+            }
+            if Platform.current() is Platform.Windows_x86_64
+            else {"preexec_fn": os.setsid}
+        )
+        process = subprocess.Popen(
+            args=[sys.executable, "-m", "http.server", str(port)],
+            env=env,
+            cwd=document_root,
+            bufsize=1,
+            stdout=fp.fileno(),
+            stderr=subprocess.STDOUT,
+            close_fds=True,
+            **daemon_kwargs,
+        )
+
+    pidfile = Pidfile.record(server_log=log, pid=process.pid, timeout=timeout)
+    if not pidfile:
+        try:
+            psutil.Process(process.pid).kill()
+        except psutil.Error as e:
+            if not isinstance(e, psutil.NoSuchProcess):
+                raise LaunchError(
+                    log,
+                    additional_msg=(
+                        f"Also failed to kill the partially launched server at pid {process.pid}: "
+                        f"{e}"
+                    ),
+                )
+        raise LaunchError(log)
+    return LaunchResult(server_info=pidfile.server_info, already_running=False)
+
+
+def shutdown() -> ServerInfo | None:
+    pidfile = Pidfile.load()
+    if not pidfile or not pidfile.alive():
+        return None
+
+    logger.debug(f"Killing {SERVER_NAME} {pidfile.server_info}")
+    pidfile.kill()
+    return pidfile.server_info

science/exe.py

@@ -29,6 +29,9 @@
 from science import __version__, providers
 from science.commands import build, lift
 from science.commands.complete import Shell
+from science.commands.doc import SERVER_NAME, LaunchError
+from science.commands.doc import launch as launch_doc_server
+from science.commands.doc import shutdown as shutdown_doc_server
 from science.commands.lift import AppInfo, FileMapping, LiftConfig, PlatformInfo
 from science.config import parse_config
 from science.context import DocConfig, ScienceConfig
@@ -52,11 +55,9 @@ def _log_fatal(
     always_include_backtrace: bool,
 ) -> None:
     if always_include_backtrace or not isinstance(value, InputError):
-        click.secho("".join(traceback.format_tb(tb)), fg="yellow", file=sys.stderr, nl=False)
-        click.secho(
-            f"{type_.__module__}.{type_.__qualname__}: ", fg="yellow", file=sys.stderr, nl=False
-        )
-    click.secho(value, fg="red", file=sys.stderr)
+        click.secho("".join(traceback.format_tb(tb)), fg="yellow", err=True, nl=False)
+        click.secho(f"{type_.__module__}.{type_.__qualname__}: ", fg="yellow", err=True, nl=False)
+    click.secho(value, fg="red", err=True)
 
 
 SEE_MANIFEST_HELP = (
@@ -202,22 +203,64 @@ def _doc(ctx: click.Context, site: str, local: Path | None) -> None:
 )
 @click.argument("page", default=None, required=False)
 @pass_doc
-def _open_doc(doc: DocConfig, remote: bool, page: str | None = None) -> None:
+@click.pass_context
+def _open_doc(ctx: click.Context, doc: DocConfig, remote: bool, page: str | None = None) -> None:
     """Opens the local documentation in a browser.
 
-    If an optional page argument is supplied, that page will be opened instead of the default doc site page.
+    If an optional page argument is supplied, that page will be opened instead of the default doc
+    site page.
+
+    Documentation is served by a local HTTP server which you can shut down with `science doc close`.
     """
-    url = doc.site if remote else f"file://{doc.local}"
+    if remote or not doc.local:
+        url = doc.site
+    else:
+        try:
+            launch_result = launch_doc_server(document_root=doc.local)
+        except LaunchError:
+            try:
+                launch_result = launch_doc_server(document_root=doc.local, port=0)
+            except LaunchError as e:
+                with open(e.log) as fp:
+                    for line in fp:
+                        logger.error(line.rstrip())
+                logger.fatal(f"Failed to launch {SERVER_NAME}.")
+                ctx.exit(1)
+                return
+
+        url = launch_result.server_info.url
+        if launch_result.already_running:
+            click.secho(
+                f"Using {SERVER_NAME} already running at {launch_result.server_info}.",
+                fg="cyan",
+                err=True,
+            )
+        else:
+            click.secho(
+                f"Launched {SERVER_NAME} at {launch_result.server_info}", fg="green", err=True
+            )
+
     if not page:
         if not remote:
             url = f"{url}/index.html"
     else:
         url_info = urlparse(url)
         page = f"{page}.html" if not PurePath(page).suffix else page
         url = urlunparse(url_info._replace(path=f"{url_info.path}/{page}"))
+
     click.launch(url)
 
 
+@_doc.command(name="close")
+def _close_doc() -> None:
+    """Shuts down the local documentation server."""
+    server_info = shutdown_doc_server()
+    if server_info:
+        click.secho(f"Shut down the {SERVER_NAME} at {server_info}.", fg="green", err=True)
+    else:
+        click.secho("No documentation server was running.", fg="cyan", err=True)
+
+
 @_main.group(cls=DYMGroup, name="provider")
 def _provider() -> None:
     """Perform operations against provider plugins."""