[ci] Polishing build.py, wave 4 (taichi-dev#7857)

- Update mambaforge && sccache - Option to use developer managed Python environment - Update docs about the Python thing, and behaviors considered intrusive
quadpixels · Apr 21, 2023 · 6498808 · 6498808
1 parent 7595e2e
commit 6498808
Show file tree

Hide file tree

Showing 4 changed files with 72 additions and 42 deletions.
diff --git a/.github/workflows/scripts/ti_build/entry.py b/.github/workflows/scripts/ti_build/entry.py
@@ -20,7 +20,7 @@
 from .ospkg import setup_os_pkgs
 from .python import get_desired_python_version, setup_python
 from .sccache import setup_sccache
-from .tinysh import Command, git
+from .tinysh import Command, CommandFailed, git
 from .vulkan import setup_vulkan
 
 
@@ -88,15 +88,21 @@ def action_wheel():
     install_build_wheel_deps(python, pip)
     handle_alternate_actions()
     build_wheel(python, pip)
-    sccache("-s")
+    try:
+        sccache("--stop-server")
+    except CommandFailed:
+        pass
 
 
 def action_android():
     sccache, python, pip = setup_basic_build_env(force_vulkan=True)
     setup_android_ndk()
     handle_alternate_actions()
     build_android(python, pip)
-    sccache("-s")
+    try:
+        sccache("--stop-server")
+    except CommandFailed:
+        pass
 
 
 def action_ios():
@@ -135,7 +141,10 @@ def parse_args():
     help = "Do not build, start a shell with environment variables set instead"
     parser.add_argument("-s", "--shell", action="store_true", help=help)
 
-    help = "Python version to use, e.g. 3.7, 3.11. Defaults to the version of the current python interpreter."
+    help = (
+        "Python version to use, e.g. '3.7', '3.11', or 'native' to not use an isolated python environment. "
+        "Defaults to the same version of the current python interpreter."
+    )
     parser.add_argument("--python", default=None, help=help)
 
     help = "Continue when encounters error."

diff --git a/.github/workflows/scripts/ti_build/python.py b/.github/workflows/scripts/ti_build/python.py
@@ -12,25 +12,25 @@
 # -- own --
 from . import misc
 from .dep import download_dep
-from .misc import banner, get_cache_home, path_prepend
+from .misc import banner, get_cache_home, path_prepend, info
 from .tinysh import Command, sh
 
 
 # -- code --
-def setup_miniforge3(prefix):
+def setup_mambaforge(prefix):
     u = platform.uname()
     if u.system == "Linux":
-        url = "https://github.com/conda-forge/miniforge/releases/download/22.9.0-2/Miniforge3-22.9.0-2-Linux-x86_64.sh"
+        url = "https://github.com/conda-forge/miniforge/releases/download/23.1.0-1/Mambaforge-23.1.0-1-Linux-x86_64.sh"
         download_dep(url, prefix, args=["-bfp", str(prefix)])
     elif (u.system, u.machine) == ("Darwin", "arm64"):
-        url = "https://github.com/conda-forge/miniforge/releases/download/22.9.0-2/Miniforge3-22.9.0-2-MacOSX-arm64.sh"
+        url = "https://github.com/conda-forge/miniforge/releases/download/23.1.0-1/Mambaforge-23.1.0-1-MacOSX-arm64.sh"
         download_dep(url, prefix, args=["-bfp", str(prefix)])
     elif (u.system, u.machine) == ("Darwin", "x86_64"):
-        url = "https://github.com/conda-forge/miniforge/releases/download/22.9.0-2/Miniforge3-22.9.0-2-MacOSX-x86_64.sh"
+        url = "https://github.com/conda-forge/miniforge/releases/download/23.1.0-1/Mambaforge-23.1.0-1-MacOSX-x86_64.sh"
         download_dep(url, prefix, args=["-bfp", str(prefix)])
     elif u.system == "Windows":
         url = (
-            "https://github.com/conda-forge/miniforge/releases/download/22.9.0-2/Miniforge3-22.9.0-2-Windows-x86_64.exe"
+            "https://github.com/conda-forge/miniforge/releases/download/23.1.0-1/Mambaforge-23.1.0-1-Windows-x86_64.exe"
         )
         download_dep(
             url,
@@ -63,6 +63,8 @@ def get_desired_python_version() -> str:
         return this_version
     elif version and re.match(r"^3\.\d+$", version):
         return version
+    elif version in ("native", "Native"):
+        return "(Native)"
     else:
         raise RuntimeError(f"Unsupported Python version: {version}")
 
@@ -72,10 +74,16 @@ def setup_python(version: str) -> Tuple[Command, Command]:
     """
     Find the required Python environment and return the `python` and `pip` commands.
     """
+    if version == "(Native)":
+        info("Using your current Python interpreter as requested.")
+        python = sh.bake(sys.executable)
+        pip = python.bake("-m", "pip")
+        return python, pip
+
     windows = platform.system() == "Windows"
 
-    prefix = get_cache_home() / "miniforge3"
-    setup_miniforge3(prefix)
+    prefix = get_cache_home() / "mambaforge"
+    setup_mambaforge(prefix)
 
     if windows:
         conda_path = prefix / "Scripts" / "conda.exe"
@@ -84,9 +92,9 @@ def setup_python(version: str) -> Tuple[Command, Command]:
 
     if not conda_path.exists():
         shutil.rmtree(prefix, ignore_errors=True)
-        setup_miniforge3(prefix)
+        setup_mambaforge(prefix)
         if not conda_path.exists():
-            raise RuntimeError(f"Failed to setup miniforge3 at {prefix}")
+            raise RuntimeError(f"Failed to setup mambaforge at {prefix}")
 
     conda = sh.bake(str(conda_path))
 

diff --git a/.github/workflows/scripts/ti_build/sccache.py b/.github/workflows/scripts/ti_build/sccache.py
@@ -19,7 +19,7 @@ def setup_sccache() -> Command:
     """
     Download and install sccache, setup compiler wrappers, and return the `sccache` command.
     """
-    root = get_cache_home() / "sccache"
+    root = get_cache_home() / "sccache-v041"
     bin = root / "bin"
 
     u = platform.uname()
@@ -32,32 +32,21 @@ def setup_sccache() -> Command:
 
     if not exe.exists():
         if u.system == "Linux":
-            download_dep(
-                "https://github.com/mozilla/sccache/releases/download/v0.3.1/sccache-v0.3.1-x86_64-unknown-linux-musl.tar.gz",
-                bin,
-                strip=1,
-            )
+            url = "https://github.com/mozilla/sccache/releases/download/v0.4.1/sccache-v0.4.1-x86_64-unknown-linux-musl.tar.gz"
         elif (u.system, u.machine) == ("Darwin", "arm64"):
-            download_dep(
-                "https://github.com/mozilla/sccache/releases/download/v0.3.1/sccache-v0.3.1-aarch64-apple-darwin.tar.gz",
-                bin,
-                strip=1,
+            url = (
+                "https://github.com/mozilla/sccache/releases/download/v0.4.1/sccache-v0.4.1-aarch64-apple-darwin.tar.gz"
             )
         elif (u.system, u.machine) == ("Darwin", "x86_64"):
-            download_dep(
-                "https://github.com/mozilla/sccache/releases/download/v0.3.1/sccache-v0.3.1-x86_64-apple-darwin.tar.gz",
-                bin,
-                strip=1,
+            url = (
+                "https://github.com/mozilla/sccache/releases/download/v0.4.1/sccache-v0.4.1-x86_64-apple-darwin.tar.gz"
             )
         elif u.system == "Windows":
-            download_dep(
-                "https://github.com/mozilla/sccache/releases/download/v0.3.1/sccache-v0.3.1-x86_64-pc-windows-msvc.tar.gz",
-                bin,
-                strip=1,
-            )
+            url = "https://github.com/mozilla/sccache/releases/download/v0.4.1/sccache-v0.4.1-x86_64-pc-windows-msvc.tar.gz"
         else:
             raise RuntimeError(f"Unsupported platform: {u.system} {u.machine}")
 
+        download_dep(url, bin, strip=1)
         exe.chmod(0o755)
 
     os.environ["SCCACHE_LOG"] = "error"

diff --git a/docs/lang/articles/contribution/dev_install.md b/docs/lang/articles/contribution/dev_install.md
@@ -447,14 +447,14 @@ Or you can find it at:
 
 A typical cache dir will contain sub folders below:
 
-| Sub Folder       | Purpose                                                  |
-| ---------------- | -------------------------------------------------------- |
-| bootstrap        | Contains Python packages used by `build.py` itself       |
-| deps             | Downloaded external dependencies, before extract/install |
-| llvm15           | Managed pre-built LLVM binaries                          |
-| miniforge3       | Conda environment dedicated to build / develop Taichi    |
-| sccache          | Compile cache                                            |
-| vulkan-1.x.xxx.x | Vulkan SDK location                                      |
+| Sub Folder       | Purpose                                                       | Code Responsible                                                                                                 |
+| ---------------- | ------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- |
+| bootstrap        | Contains Python packages used by `build.py` itself            | [bootstrap.py](https://github.com/taichi-dev/taichi/blob/master/.github/workflows/scripts/ti_build/bootstrap.py) |
+| deps             | Downloaded external dependencies, before extract/install      | [dep.py](https://github.com/taichi-dev/taichi/blob/master/.github/workflows/scripts/ti_build/dep.py)             |
+| llvm15           | Managed pre-built LLVM binaries                               | [llvm.py](https://github.com/taichi-dev/taichi/blob/master/.github/workflows/scripts/ti_build/llvm.py)           |
+| mambaforge       | Managed conda environment dedicated to build / develop Taichi | [python.py](https://github.com/taichi-dev/taichi/blob/master/.github/workflows/scripts/ti_build/python.py)       |
+| sccache          | Compile cache                                                 | [sccache.py](https://github.com/taichi-dev/taichi/blob/master/.github/workflows/scripts/ti_build/sccache.py)     |
+| vulkan-1.x.xxx.x | Vulkan SDK location                                           | [vulkan.py](https://github.com/taichi-dev/taichi/blob/master/.github/workflows/scripts/ti_build/vulkan.py)       |
 
 The whole cache folder can be safely removed.
 
@@ -464,9 +464,20 @@ The whole cache folder can be safely removed.
 On Debian/Ubuntu systems, `apt install python3-distutils` is required due to packaging quirks.
 :::
 
-### Build / develop for a different Python interpreter version
+:::caution Behaviors considered intrusive
+
+1. On Ubuntu systems, there's an attempt to install missing development libraries at [ospkg.py](https://github.com/taichi-dev/taichi/blob/master/.github/workflows/scripts/ti_build/ospkg.py) by invoking `sudo apt install libxxxx-dev`
+   if a terminal is detected. It can be skipped by telling `apt` not to install them.
+
+2. Installing Vulkan SDK on Windows requires elevated privileges, and the installer will set several machine scoped environment variables (`VULKAN_SDK` and `VK_SDK_PATH`).
+
+:::
+
+
+### Choose your desired Python version, or use your own Python environment.
 
 By default, `build.py` assumes that the same Python version used to invoke it will also be used for building Taichi.
+`build.py` will then create an isolated Python environment and use it for all the subsequent Python related tasks.
 To use a different version, please specify the desired version via `--python` option:
 
 ```shell
@@ -477,6 +488,19 @@ To use a different version, please specify the desired version via `--python` op
 ./build.py --python=3.10 --shell
 ```
 
+If you prefer to manage Python environments yourself, you could specify `--python=native`, and `build.py` will not attempt to use a managed Python environment.
+
+```shell
+# Use your own conda
+conda activate my-own-conda-env
+
+# Build a wheel
+./build.py --python=native
+
+# Or enter development shell
+./build.py --python=native --shell
+```
+
 ## Troubleshooting and debugging
 
 ### Permission denied