Isolate paka's pulumi from system pulumi

e2e/pytest_kind/cluster.py

@@ -13,14 +13,7 @@
 from kubernetes import config
 
 from paka.k8s.utils import setup_port_forward
-
-
-def get_latest_kind_version() -> str:
-    url = "https://api.github.com/repos/kubernetes-sigs/kind/releases/latest"
-    response = requests.get(url)
-    response.raise_for_status()
-    data = response.json()
-    return data["tag_name"]
+from paka.utils import get_gh_release_latest_version
 
 
 def get_latest_kubectl_version() -> str:
@@ -30,7 +23,9 @@ def get_latest_kubectl_version() -> str:
     return response.text
 
 
-KIND_VERSION = os.environ.get("KIND_VERSION", get_latest_kind_version())
+KIND_VERSION = os.environ.get(
+    "KIND_VERSION", get_gh_release_latest_version("kubernetes-sigs/kind")
+)
 KUBECTL_VERSION = os.environ.get("KUBECTL_VERSION", get_latest_kubectl_version())
 
 

e2e/test_pulumi_install.py

@@ -0,0 +1,35 @@
+import os
+import shutil
+import tempfile
+from unittest.mock import patch
+
+import pytest
+
+from paka.cluster.pulumi import ensure_pulumi
+from paka.constants import HOME_ENV_VAR
+
+
+@pytest.mark.parametrize(
+    "system, arch",
+    [
+        ("darwin", "amd64"),
+        ("darwin", "arm64"),
+        ("linux", "amd64"),
+        ("linux", "arm64"),
+        ("windows", "amd64"),
+        ("windows", "arm64"),
+    ],
+)
+def test_installation(system: str, arch: str) -> None:
+    with patch("platform.system", return_value=system), patch(
+        "platform.machine", return_value=arch
+    ), tempfile.TemporaryDirectory() as temp_dir:
+        os.environ[HOME_ENV_VAR] = temp_dir
+
+        ensure_pulumi()
+
+        bin = "pulumi"
+        if system == "windows":
+            bin += ".exe"
+
+        assert shutil.which(bin) is not None

paka/cli/utils.py

@@ -243,3 +243,5 @@ def init_pulumi() -> None:
     os.environ["PULUMI_BACKEND_URL"] = os.environ.get(
         "PULUMI_BACKEND_URL", f"file://{pulumi_root}"
     )
+    os.environ["PULUMI_DEBUG"] = os.environ.get("PULUMI_DEBUG", "false")
+    os.environ["PULUMI_HOME"] = os.environ.get("PULUMI_HOME", pulumi_root)

paka/cluster/manager/base.py

@@ -5,6 +5,7 @@
 
 from pulumi import automation as auto
 
+from paka.cluster.pulumi import ensure_pulumi
 from paka.config import CloudConfig, Config
 from paka.k8s.model_group.service import create_model_group_service
 from paka.logger import logger
@@ -46,6 +47,8 @@ def _stack_for_program(self, program: auto.PulumiFn) -> auto.Stack:
 
     @property
     def _stack(self) -> auto.Stack:
+        ensure_pulumi()
+
         def program() -> None:
             self.provision_k8s()
 

paka/cluster/pulumi.py

@@ -0,0 +1,120 @@
+import os
+import platform
+import shutil
+import tarfile
+import tempfile
+import zipfile
+from pathlib import Path
+
+import requests
+
+from paka.logger import logger
+from paka.utils import calculate_sha256, get_project_data_dir
+
+# Pin the Pulumi version to avoid breaking changes
+PULUMI_VERSION = "v3.114.0"
+
+
+def change_permissions_recursive(path: Path, mode: int) -> None:
+    for child in path.iterdir():
+        if child.is_file():
+            child.chmod(mode)
+        elif child.is_dir():
+            child.chmod(mode)
+            change_permissions_recursive(child, mode)
+
+
+def ensure_pulumi() -> None:
+    paka_home = Path(get_project_data_dir())
+
+    bin_dir = paka_home / "bin"
+    bin_dir.mkdir(parents=True, exist_ok=True)
+
+    current_path = os.environ.get("PATH", "")
+
+    pulumi_files = list(bin_dir.glob("pulumi-*"))
+    if pulumi_files:
+        os.environ["PATH"] = f"{pulumi_files[0]}:{current_path}"
+        return
+
+    pulumi_version = PULUMI_VERSION
+
+    new_pulumi_path = bin_dir / f"pulumi-{pulumi_version}"
+
+    system = platform.system().lower()
+    arch = platform.machine().lower()
+
+    if arch in ["amd64", "x86_64"]:
+        arch = "x64"
+    elif arch == "arm64":
+        arch = "arm64"
+    else:
+        raise Exception(f"Unsupported architecture: {arch}")
+
+    pulumi_file = f"pulumi-{pulumi_version}-{system}-{arch}"
+
+    if system == "windows":
+        pulumi_file = f"{pulumi_file}.zip"
+    else:
+        pulumi_file = f"{pulumi_file}.tar.gz"
+
+    # First of all, download the checksum file
+    checksum_url = f"https://github.com/pulumi/pulumi/releases/download/{pulumi_version}/pulumi-{pulumi_version[1:]}-checksums.txt"
+
+    response = requests.get(checksum_url)
+    response.raise_for_status()
+    file_sha256_dict = {}
+    # Iterate over the lines in the checksum file and split by sha256 and filename
+    for line in response.text.strip().split("\n"):
+        expected_sha256, filename = line.strip().split()
+        file_sha256_dict[filename] = expected_sha256
+
+    url = f"https://github.com/pulumi/pulumi/releases/download/{pulumi_version}/{pulumi_file}"
+
+    logger.info(f"Downloading {pulumi_file}...")
+
+    with tempfile.NamedTemporaryFile() as tf:
+        with requests.get(url, stream=True) as r:
+            r.raise_for_status()
+            for chunk in r.iter_content(chunk_size=8192):
+                tf.write(chunk)
+
+        tf.flush()
+        os.fsync(tf.fileno())
+
+        archive_file = tf.name
+
+        archive_file_sha256 = calculate_sha256(archive_file)
+
+        if pulumi_file not in file_sha256_dict:
+            raise Exception(f"SHA256 not found for {pulumi_file}")
+
+        expected_sha256 = file_sha256_dict[pulumi_file]
+
+        if archive_file_sha256 != expected_sha256:
+            raise Exception(
+                f"SHA256 mismatch: {archive_file_sha256} != {expected_sha256}"
+            )
+
+        if system == "windows":
+            with zipfile.ZipFile(archive_file, "r") as zip_ref:
+                zip_ref.extractall(bin_dir)
+        else:
+            with tarfile.open(archive_file, "r:gz") as tar:
+                tar.extractall(bin_dir)
+
+    pulumi_path = bin_dir / "pulumi"
+    change_permissions_recursive(pulumi_path, 0o755)
+    pulumi_path = pulumi_path.rename(new_pulumi_path)
+
+    # For windows, the Pulumi binary is under pulumi_path/bin
+    # For other platforms, the Pulumi binary is under pulumi_path
+    if system == "windows":
+        windows_bin_path = pulumi_path / "bin"
+        for file in windows_bin_path.iterdir():
+            if file.is_file():
+                shutil.move(str(file), str(pulumi_path))
+
+    logger.info("Pulumi installed successfully.")
+
+    os.environ["PATH"] = f"{pulumi_path}:{current_path}"

paka/container/pack.py

@@ -1,4 +1,3 @@
-import hashlib
 import os
 import platform
 import tarfile
@@ -9,25 +8,11 @@
 import requests
 
 from paka.logger import logger
-from paka.utils import get_project_data_dir
-
-
-def get_latest_pack_version() -> str:
-    url = "https://api.github.com/repos/buildpacks/pack/releases/latest"
-    response = requests.get(url)
-    response.raise_for_status()
-    data = response.json()
-    return data["tag_name"]
-
-
-def calculate_sha256(file_path: str) -> str:
-    sha256_hash = hashlib.sha256()
-
-    with open(file_path, "rb") as f:
-        for byte_block in iter(lambda: f.read(4096), b""):
-            sha256_hash.update(byte_block)
-
-    return sha256_hash.hexdigest()
+from paka.utils import (
+    calculate_sha256,
+    get_gh_release_latest_version,
+    get_project_data_dir,
+)
 
 
 def ensure_pack() -> str:
@@ -40,7 +25,7 @@ def ensure_pack() -> str:
     if pack_files:
         return str(pack_files[0])
 
-    pack_version = get_latest_pack_version()
+    pack_version = get_gh_release_latest_version("buildpacks/pack")
 
     new_pack_path = bin_dir / f"pack-{pack_version}"
 

paka/utils.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import hashlib
 import json
 import os
 import random
@@ -10,6 +11,7 @@
 from pathlib import Path
 from typing import Any, Callable, Dict, Optional
 
+import requests
 from ruamel.yaml import YAML
 
 from paka.constants import HOME_ENV_VAR, PROJECT_NAME
@@ -186,7 +188,7 @@ def get_pulumi_root() -> str:
     Returns:
         str: The pulumi data directory.
     """
-    return get_project_data_dir()
+    return str(Path(get_project_data_dir()) / "pulumi")
 
 
 def save_kubeconfig(cluster_name: str, kubeconfig_json: Optional[str]) -> None:
@@ -319,3 +321,41 @@ def random_str(length: int = 5) -> str:
         str: The generated random string.
     """
     return "".join(random.choices(string.ascii_letters + string.digits, k=length))
+
+
+def calculate_sha256(file_path: str) -> str:
+    """
+    Calculate the SHA-256 hash of a file.
+
+    Args:
+        file_path (str): The path to the file for which the SHA-256 hash is to be calculated.
+
+    Returns:
+        str: The SHA-256 hash of the file, as a hexadecimal string.
+    """
+    sha256_hash = hashlib.sha256()
+
+    with open(file_path, "rb") as f:
+        for byte_block in iter(lambda: f.read(4096), b""):
+            sha256_hash.update(byte_block)
+
+    return sha256_hash.hexdigest()
+
+
+def get_gh_release_latest_version(repo: str) -> str:
+    """
+    Get the latest release version of a GitHub repository.
+
+    This function queries the GitHub API to get the latest release version of a repository.
+
+    Args:
+        repo (str): The GitHub repository in the format 'owner/repo'.
+
+    Returns:
+        str: The latest release version of the repository.
+    """
+    url = f"https://api.github.com/repos/{repo}/releases/latest"
+    response = requests.get(url)
+    response.raise_for_status()
+    data = response.json()
+    return data["tag_name"]