Add a JSON output for pyodide xbuildenv search, better tabular outp…

…ut (#28)

## Description

This PR closes #26. It adds a `pyodide xbuildenv search --json` option
to print a JSON-based output, along with associated tests. The advantage
is that it can be saved to a file, piped to `jq` or shell functions (or
any equivalent tools), or simply imported into a Pythonic interface.

Additionally, I added a small context manager that does not do anything
but stop printing the following lines:
```
Starting new HTTPS connection (1): raw.githubusercontent.com:443
https://raw.githubusercontent.com:443 "GET /pyodide/pyodide/main/pyodide-cross-build-environments.json HTTP/11" 200 917
```

in the output, because it conflicts with receiving valid JSON. Please
let me know if this would be undesirable. If yes, I'll try to work
around it so that it gets printed for the non-JSON output (table).

CHANGELOG.md

@@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Added
+
+- The `pyodide xbuildenv search` command now accepts a `--json` flag to output the
+  search results in JSON format that is machine-readable. The design for the regular
+  tabular output has been improved.
+  [#28](https://github.com/pyodide/pyodide-build/pull/28)
+
 ### Changed
 
 - The `pyodide skeleton pypi --update` command and the `--update-patched` variant now
@@ -30,7 +37,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [0.27.2] - 2024/07/11
 
-## Changed
+### Changed
 
 - `pyodide py-compile` command now accepts `excludes` flag.
   [#9](https://github.com/pyodide/pyodide-build/pull/9)
@@ -40,7 +47,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [0.27.1] - 2024/06/28
 
-## Changed
+### Changed
 
 - ported f2c_fixes patch from https://github.com/pyodide/pyodide/pull/4822
 

pyodide_build/cli/xbuildenv.py

@@ -4,6 +4,7 @@
 
 from pyodide_build.build_env import local_versions
 from pyodide_build.common import xbuildenv_dirname
+from pyodide_build.views import MetadataView
 from pyodide_build.xbuildenv import CrossBuildEnvManager
 from pyodide_build.xbuildenv_releases import (
     cross_build_env_metadata_url,
@@ -151,6 +152,11 @@ def _search(
         "-a",
         help="search all versions, without filtering out incompatible ones",
     ),
+    json_output: bool = typer.Option(
+        False,
+        "--json",
+        help="output results in JSON format",
+    ),
 ) -> None:
     """
     Search for available versions of cross-build environment.
@@ -175,40 +181,25 @@ def _search(
         )
         raise typer.Exit(1)
 
-    table = []
-    columns = [
-        # column name, width
-        ("Version", 10),
-        ("Python", 10),
-        ("Emscripten", 10),
-        ("pyodide-build", 25),
-        ("Compatible", 10),
-    ]
-    header = [f"{name:{width}}" for name, width in columns]
-    divider = ["-" * width for _, width in columns]
-
-    table.append("\t".join(header))
-    table.append("\t".join(divider))
-
-    for release in releases:
-        compatible = (
-            "Yes"
-            if release.is_compatible(
+    # Generate views for the metadata objects (currently tabular or JSON)
+    views = [
+        MetadataView(
+            version=release.version,
+            python=release.python_version,
+            emscripten=release.emscripten_version,
+            pyodide_build={
+                "min": release.min_pyodide_build_version,
+                "max": release.max_pyodide_build_version,
+            },
+            compatible=release.is_compatible(
                 python_version=local["python"],
                 pyodide_build_version=local["pyodide-build"],
-            )
-            else "No"
+            ),
         )
-        pyodide_build_range = f"{release.min_pyodide_build_version or ''} - {release.max_pyodide_build_version or ''}"
-
-        row = [
-            f"{release.version:{columns[0][1]}}",
-            f"{release.python_version:{columns[1][1]}}",
-            f"{release.emscripten_version:{columns[2][1]}}",
-            f"{pyodide_build_range:{columns[3][1]}}",
-            f"{compatible:{columns[4][1]}}",
-        ]
-
-        table.append("\t".join(row))
+        for release in releases
+    ]
 
-    print("\n".join(table))
+    if json_output:
+        print(MetadataView.to_json(views))
+    else:
+        print(MetadataView.to_table(views))

pyodide_build/tests/test_cli_xbuildenv.py

@@ -1,3 +1,4 @@
+import json
 import os
 import shutil
 from pathlib import Path
@@ -25,6 +26,14 @@ def mock_pyodide_lock() -> PyodideLockSpec:
     )
 
 
+def is_valid_json(json_str) -> bool:
+    try:
+        json.loads(json_str)
+    except json.JSONDecodeError:
+        return False
+    return True
+
+
 @pytest.fixture()
 def mock_xbuildenv_url(tmp_path_factory, httpserver):
     """
@@ -331,14 +340,77 @@ def test_xbuildenv_search(
 
     assert result.exit_code == 0, result.stdout
 
-    header = result.stdout.splitlines()[0]
-    assert header.split() == [
+    lines = result.stdout.splitlines()
+    header = lines[1].strip().split("│")[1:-1]
+    assert [col.strip() for col in header] == [
         "Version",
         "Python",
         "Emscripten",
         "pyodide-build",
         "Compatible",
     ]
 
-    row1 = result.stdout.splitlines()[2]
-    assert row1.split() == ["0.1.0", "4.5.6", "1.39.8", "-", "No"]
+    row1 = lines[3].strip().split("│")[1:-1]
+    assert [col.strip() for col in row1] == ["0.1.0", "4.5.6", "1.39.8", "-", "No"]
+
+
+def test_xbuildenv_search_json(tmp_path, fake_xbuildenv_releases_compatible):
+    result = runner.invoke(
+        xbuildenv.app,
+        [
+            "search",
+            "--metadata",
+            str(fake_xbuildenv_releases_compatible),
+            "--json",
+            "--all",
+        ],
+    )
+
+    # Sanity check
+    assert result.exit_code == 0, result.stdout
+    assert is_valid_json(result.stdout), "Output is not valid JSON"
+
+    output = json.loads(result.stdout)
+
+    # First, check overall structure of JSON response
+    assert isinstance(output, dict), "Output should be a dictionary"
+    assert "environments" in output, "Output should have an 'environments' key"
+    assert isinstance(output["environments"], list), "'environments' should be a list"
+
+    # Now, we'll check types in each environment entry
+    for environment in output["environments"]:
+        assert isinstance(environment, dict), "Each environment should be a dictionary"
+        assert set(environment.keys()) == {
+            "version",
+            "python",
+            "emscripten",
+            "pyodide_build",
+            "compatible",
+        }, f"Environment {environment} has unexpected keys: {environment.keys()}"
+
+        assert isinstance(environment["version"], str), "version should be a string"
+        assert isinstance(environment["python"], str), "python should be a string"
+        assert isinstance(
+            environment["emscripten"], str
+        ), "emscripten should be a string"
+        assert isinstance(
+            environment["compatible"], bool
+        ), "compatible should be either True or False"
+
+        assert isinstance(
+            environment["pyodide_build"], dict
+        ), "pyodide_build should be a dictionary"
+        assert set(environment["pyodide_build"].keys()) == {
+            "min",
+            "max",
+        }, f"pyodide_build has unexpected keys: {environment['pyodide_build'].keys()}"
+        assert isinstance(
+            environment["pyodide_build"]["min"], (str, type(None))
+        ), "pyodide_build-min should be a string or None"
+        assert isinstance(
+            environment["pyodide_build"]["max"], (str, type(None))
+        ), "pyodide_build-max should be a string or None"
+
+    assert any(
+        env["compatible"] for env in output["environments"]
+    ), "There should be at least one compatible environment"

pyodide_build/views.py

@@ -0,0 +1,92 @@
+# Class for generating "views", i.e., tabular and JSON outputs from
+# metadata objects, currently used in the xbuildenv CLI (search command).
+
+
+import json
+from dataclasses import dataclass
+
+
+@dataclass
+class MetadataView:
+    version: str
+    python: str
+    emscripten: str
+    pyodide_build: dict[str, str | None]
+    compatible: bool
+
+    @classmethod
+    def to_table(cls, views: list["MetadataView"]) -> str:
+        columns = [
+            ("Version", 10),
+            ("Python", 10),
+            ("Emscripten", 10),
+            ("pyodide-build", 25),
+            ("Compatible", 10),
+        ]
+
+        # Unicode box-drawing characters
+        top_left, top_right = "┌", "┐"
+        bottom_left, bottom_right = "└", "┘"
+        horizontal, vertical = "─", "│"
+        t_down, t_up, t_right, t_left = "┬", "┴", "├", "┤"
+        cross = "┼"
+
+        # Table elements
+        top_border = (
+            top_left
+            + t_down.join(horizontal * (width + 2) for _, width in columns)
+            + top_right
+        )
+        header = (
+            vertical
+            + vertical.join(f" {name:<{width}} " for name, width in columns)
+            + vertical
+        )
+        separator = (
+            t_right
+            + cross.join(horizontal * (width + 2) for _, width in columns)
+            + t_left
+        )
+        bottom_border = (
+            bottom_left
+            + t_up.join(horizontal * (width + 2) for _, width in columns)
+            + bottom_right
+        )
+
+        ### Printing
+        table = [top_border, header, separator]
+        for view in views:
+            pyodide_build_range = (
+                f"{view.pyodide_build['min'] or ''} - {view.pyodide_build['max'] or ''}"
+            )
+            row = [
+                f"{view.version:<{columns[0][1]}}",
+                f"{view.python:<{columns[1][1]}}",
+                f"{view.emscripten:<{columns[2][1]}}",
+                f"{pyodide_build_range:<{columns[3][1]}}",
+                f"{'Yes' if view.compatible else 'No':<{columns[4][1]}}",
+            ]
+            table.append(
+                vertical + vertical.join(f" {cell} " for cell in row) + vertical
+            )
+        table.append(bottom_border)
+        return "\n".join(table)
+
+    @classmethod
+    def to_json(cls, views: list["MetadataView"]) -> str:
+        result = json.dumps(
+            {
+                "environments": [
+                    {
+                        "version": view.version,
+                        "python": view.python,
+                        "emscripten": view.emscripten,
+                        "pyodide_build": view.pyodide_build,
+                        "compatible": view.compatible,
+                    }
+                    for view in views
+                ]
+            },
+            indent=2,
+        )
+        return result

pyodide_build/xbuildenv_releases.py

@@ -1,4 +1,6 @@
+import logging
 import os
+from contextlib import contextmanager
 from functools import cache
 
 from packaging.version import Version
@@ -19,7 +21,7 @@ class CrossBuildEnvReleaseSpec(BaseModel):
     python_version: str
     # The version of the Emscripten SDK
     emscripten_version: str
-    # Minimum and maximum pyodide-build versions that is compatible with this release
+    # Minimum and maximum pyodide-build versions that are compatible with this release
     min_pyodide_build_version: str | None = None
     max_pyodide_build_version: str | None = None
     model_config = ConfigDict(extra="forbid", title="CrossBuildEnvReleasesSpec")
@@ -184,6 +186,20 @@ def get_release(
         return self.releases[version]
 
 
+@contextmanager
+def _suppress_urllib3_logging():
+    """
+    Temporarily suppresses urllib3 logging for internal use.
+    """
+    logger = logging.getLogger("urllib3")
+    original_level = logger.level
+    logger.setLevel(logging.WARNING)
+    try:
+        yield
+    finally:
+        logger.setLevel(original_level)
+
+
 def cross_build_env_metadata_url() -> str:
     """
     Get the URL to the Pyodide cross-build environment metadata
@@ -218,9 +234,11 @@ def load_cross_build_env_metadata(url_or_filename: str) -> CrossBuildEnvMetaSpec
     if url_or_filename.startswith("http"):
         import requests
 
-        response = requests.get(url_or_filename)
-        response.raise_for_status()
-        data = response.json()
+        with _suppress_urllib3_logging():
+            with requests.get(url_or_filename) as response:
+                response.raise_for_status()
+            data = response.json()
+
         return CrossBuildEnvMetaSpec.model_validate(data)
 
     with open(url_or_filename) as f: