From 25425b5940d800c832a7c96d2579562ef6e00a05 Mon Sep 17 00:00:00 2001 From: paulacamargo25 Date: Wed, 18 Oct 2023 15:01:01 -0700 Subject: [PATCH] Platform-specific vsix (#89) * Update nox file and pre release * update args * update value * see params * echo vars * fix args * fix args * Fix error obj to string * fix command line error * update args * update how access values * Add args * Send arguments * tried ith arguments * update value * remove args extra * update run * send as env * add to see other values * update env variables * use VSCETARGET * fix variable * Fix macOs error * Fix lint in nox file * Fix nox hash comparison * Fix default value * Remove unnecessary code * Update platform url and debugpy version * Fix dict key name * Remove vsix for all platforms * Fix pr comments * fix download function * fix hash code * Read from json pypypackage * add json file and session that creates it * Reformat nox file * resolve comments * fix lint error * update pipeline * fix build error * dont hardcode the hash --- build/azure-pipeline.pre-release.yml | 36 ++++- debugpy_info.json | 32 ++++ noxfile.py | 213 +++++++++++---------------- 3 files changed, 147 insertions(+), 134 deletions(-) create mode 100644 debugpy_info.json diff --git a/build/azure-pipeline.pre-release.yml b/build/azure-pipeline.pre-release.yml index b0dc7ab1..6c085735 100644 --- a/build/azure-pipeline.pre-release.yml +++ b/build/azure-pipeline.pre-release.yml @@ -28,6 +28,32 @@ extends: template: azure-pipelines/extension/pre-release.yml@templates parameters: l10nSourcePaths: ./src + buildPlatforms: + - name: Linux + packageArch: arm64 + vsceTarget: linux-arm64 + - name: Linux + packageArch: arm + vsceTarget: linux-armhf + - name: Linux + packageArch: x64 + vsceTarget: linux-x64 + - name: MacOS + packageArch: arm64 + vsceTarget: darwin-arm64 + - name: MacOS + packageArch: x64 + vsceTarget: darwin-x64 + - name: Windows + packageArch: arm + vsceTarget: win32-arm64 + - name: Windows + packageArch: ia32 + vsceTarget: win32-ia32 + - name: Windows + packageArch: x64 + vsceTarget: win32-x64 + buildSteps: - task: NodeTool@0 inputs: @@ -47,14 +73,14 @@ extends: - script: python -m pip install -U pip displayName: Upgrade pip - - script: python -m pip install wheel - displayName: Install wheel - - - script: python -m pip install nox - displayName: Install wheel + - script: python -m pip install wheel nox + displayName: Install wheel and nox + # update according packageArch - script: python -m nox --session install_bundled_libs displayName: Install Python dependencies + env: + VSCETARGET: ${{ variables.VSCETARGET }} - script: python ./build/update_ext_version.py --for-publishing displayName: Update build number diff --git a/debugpy_info.json b/debugpy_info.json new file mode 100644 index 00000000..76266411 --- /dev/null +++ b/debugpy_info.json @@ -0,0 +1,32 @@ +{ + "macOS": { + "url": "https://files.pythonhosted.org/packages/bd/a3/5e37ce13c7dd850b72a52be544a058ed49606ebbbf8b95b2ba3c1db5620a/debugpy-1.7.0-cp311-cp311-macosx_11_0_universal2.whl", + "hash": { + "sha256": "538765a41198aa88cc089295b39c7322dd598f9ef1d52eaae12145c63bf9430a" + } + }, + "win32": { + "url": "https://files.pythonhosted.org/packages/52/59/3591e9f709b7ee4d3a926a8903a395669cd0e0279204a94b6acccf6ed6ee/debugpy-1.7.0-cp311-cp311-win32.whl", + "hash": { + "sha256": "18a69f8e142a716310dd0af6d7db08992aed99e2606108732efde101e7c65e2a" + } + }, + "win64": { + "url": "https://files.pythonhosted.org/packages/51/59/84ebd58d3e9de33a54ca8aa4532e03906e5458092dafe240264c2937a99b/debugpy-1.7.0-cp311-cp311-win_amd64.whl", + "hash": { + "sha256": "7515a5ba5ee9bfe956685909c5f28734c1cecd4ee813523363acfe3ca824883a" + } + }, + "linux": { + "url": "https://files.pythonhosted.org/packages/b4/fc/087324d46dab8e21e084ce2cf670fa7e524ab5e7691692438e4987bd3ecb/debugpy-1.7.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "hash": { + "sha256": "c7e8cf91f8f3f9b5fad844dd88427b85d398bda1e2a0cd65d5a21312fcbc0c6f" + } + }, + "any": { + "url": "https://files.pythonhosted.org/packages/39/2f/c8a8cfac6c7fa3d9e163a6bf46e6d27d027b7a1331028e99a6ef7fd3699d/debugpy-1.7.0-py2.py3-none-any.whl", + "hash": { + "sha256": "f6de2e6f24f62969e0f0ef682d78c98161c4dca29e9fb05df4d2989005005502" + } + } +} \ No newline at end of file diff --git a/noxfile.py b/noxfile.py index f2133ee0..80bcfa92 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. """All the action we need during build""" +import hashlib import io import json import os @@ -12,26 +13,6 @@ import nox # pylint: disable=import-error -def _install_bundle(session: nox.Session) -> None: - session.install( - "-t", - "./bundled/libs", - "--no-cache-dir", - "--implementation", - "py", - "--no-deps", - "--upgrade", - "-r", - "./requirements.txt", - ) - session.install("packaging") - _install_package(f"{os.getcwd()}/bundled/libs", "debugpy", "1.7.0") - - -def _update_pip_packages(session: nox.Session) -> None: - session.run("pip-compile", "--generate-hashes", "--upgrade", "./requirements.in") - - @nox.session() def lint(session: nox.Session) -> None: """Runs linter and formatter checks on python files.""" @@ -58,81 +39,80 @@ def tests(session: nox.Session) -> None: session.run("npm", "run", "test") -def _get_package_data(package): - json_uri = f"https://registry.npmjs.org/{package}" - with url_lib.urlopen(json_uri) as response: - return json.loads(response.read()) - - -def _update_npm_packages(session: nox.Session) -> None: - pinned = { - "vscode-languageclient", - "@types/vscode", - "@types/node", - } - package_json_path = pathlib.Path(__file__).parent / "package.json" - package_json = json.loads(package_json_path.read_text(encoding="utf-8")) - - for package in package_json["dependencies"]: - if package not in pinned: - data = _get_package_data(package) - latest = "^" + data["dist-tags"]["latest"] - package_json["dependencies"][package] = latest - - for package in package_json["devDependencies"]: - if package not in pinned: - data = _get_package_data(package) - latest = "^" + data["dist-tags"]["latest"] - package_json["devDependencies"][package] = latest - - # Ensure engine matches the package - if ( - package_json["engines"]["vscode"] - != package_json["devDependencies"]["@types/vscode"] - ): - print( - "Please check VS Code engine version and @types/vscode version in package.json." - ) +@nox.session() +def install_bundled_libs(session): + """Installs the libraries that will be bundled with the extension.""" + session.install("wheel") + session.install( + "-t", + "./bundled/libs", + "--no-cache-dir", + "--implementation", + "py", + "--no-deps", + "--require-hashes", + "--only-binary", + ":all:", + "-r", + "./requirements.txt", + ) + session.install("packaging") - new_package_json = json.dumps(package_json, indent=4) - # JSON dumps uses \n for line ending on all platforms by default - if not new_package_json.endswith("\n"): - new_package_json += "\n" - package_json_path.write_text(new_package_json, encoding="utf-8") + debugpy_info_json_path = pathlib.Path(__file__).parent / "debugpy_info.json" + debugpy_info = json.loads(debugpy_info_json_path.read_text(encoding="utf-8")) + + target = os.environ.get("VSCETARGET", "") + print("target:", target) + if "darwin" in target: + download_url(debugpy_info["macOS"]) + elif "win32-ia32" == target: + download_url(debugpy_info["win32"]) + elif "win32-x64" == target: + download_url(debugpy_info["win64"]) + elif "linux-x64" == target: + download_url(debugpy_info["linux"]) + else: + download_url(debugpy_info["any"]) - session.run("npm", "audit", "fix", external=True) - session.run("npm", "install", external=True) +def download_url(value): + with url_lib.urlopen(value["url"]) as response: + data = response.read() + hash_algorithm, hash_value = [ + (key, value) for key, value in value["hash"].items() + ][0] + if hashlib.new(hash_algorithm, data).hexdigest() != hash_value: + raise ValueError("Failed hash verification for {}.".format(value["url"])) -def _setup_template_environment(session: nox.Session) -> None: - session.install("wheel", "pip-tools") - _update_pip_packages(session) - _install_bundle(session) + print("Download: ", value["url"]) + with zipfile.ZipFile(io.BytesIO(data), "r") as wheel: + libs_dir = pathlib.Path.cwd() / "bundled" / "libs" + for zip_info in wheel.infolist(): + print("\t" + zip_info.filename) + wheel.extract(zip_info.filename, libs_dir) -@nox.session(python="3.7") -def install_bundled_libs(session): - """Installs the libraries that will be bundled with the extension.""" - session.install("wheel") - _install_bundle(session) - +@nox.session() +def update_build_number(session: nox.Session) -> None: + """Updates build number for the extension.""" + if not len(session.posargs): + session.log("No updates to package version") + return -@nox.session(python="3.7") -def setup(session: nox.Session) -> None: - """Sets up the extension for development.""" - _setup_template_environment(session) + package_json_path = pathlib.Path(__file__).parent / "package.json" + session.log(f"Reading package.json at: {package_json_path}") + package_json = json.loads(package_json_path.read_text(encoding="utf-8")) -@nox.session() -def update_packages(session: nox.Session) -> None: - """Update pip and npm packages.""" - session.install("wheel", "pip-tools") - _update_pip_packages(session) - _update_npm_packages(session) + parts = re.split(r"\.|-", package_json["version"]) + major, minor = parts[:2] + version = f"{major}.{minor}.{session.posargs[0]}" + version = version if len(parts) == 3 else f"{version}-{''.join(parts[3:])}" -def _contains(s, parts=()): - return any(p for p in parts if p in s) + session.log(f"Updating version from {package_json['version']} to {version}") + package_json["version"] = version + package_json_path.write_text(json.dumps(package_json, indent=4), encoding="utf-8") def _get_pypi_package_data(package_name): @@ -143,59 +123,34 @@ def _get_pypi_package_data(package_name): return json.loads(response.read()) -def _get_urls(data, version): - return list( - r["url"] for r in data["releases"][version] if _contains(r["url"], ("cp37",)) - ) - - -def _download_and_extract(root, url): - if "manylinux" in url or "macosx" in url or "win_amd64" in url: - root = os.getcwd() if root is None or root == "." else root - print(url) - with url_lib.urlopen(url) as response: - data = response.read() - with zipfile.ZipFile(io.BytesIO(data), "r") as wheel: - for zip_info in wheel.infolist(): - # Ignore dist info since we are merging multiple wheels - if ".dist-info/" in zip_info.filename: - continue - print("\t" + zip_info.filename) - wheel.extract(zip_info.filename, root) - - -def _install_package(root, package_name, version="latest"): +def _get_debugpy_info(version="latest", platform="none-any", cp="cp311"): from packaging.version import parse as version_parser - data = _get_pypi_package_data(package_name) + data = _get_pypi_package_data("debugpy") if version == "latest": use_version = max(data["releases"].keys(), key=version_parser) else: use_version = version - for url in _get_urls(data, use_version): - _download_and_extract(root, url) + return list( + {"url": r["url"], "hash": {"sha256": r["digests"]["sha256"]}} + for r in data["releases"][use_version] + if f"{cp}-{platform}" in r["url"] or f"py3-{platform}" in r["url"] + )[0] @nox.session() -def update_build_number(session: nox.Session) -> None: - """Updates build number for the extension.""" - if len(session.posargs) == 0: - session.log("No updates to package version") - return - - package_json_path = pathlib.Path(__file__).parent / "package.json" - session.log(f"Reading package.json at: {package_json_path}") - - package_json = json.loads(package_json_path.read_text(encoding="utf-8")) - - parts = re.split("\\.|-", package_json["version"]) - major, minor = parts[:2] - - version = f"{major}.{minor}.{session.posargs[0]}" - version = version if len(parts) == 3 else f"{version}-{''.join(parts[3:])}" - - session.log(f"Updating version from {package_json['version']} to {version}") - package_json["version"] = version - package_json_path.write_text(json.dumps(package_json, indent=4), encoding="utf-8") +def create_debugpy_json(session: nox.Session, version="1.7.0", cp="cp311"): + platforms = [ + ("macOS", "macosx"), + ("win32", "win32"), + ("win64", "win_amd64"), + ("linux", "manylinux"), + ("any", "none-any"), + ] + debugpy_info_json_path = pathlib.Path(__file__).parent / "debugpy_info.json" + debugpy_info = {p: _get_debugpy_info(version, id, cp) for p, id in platforms} + debugpy_info_json_path.write_text( + json.dumps(debugpy_info, indent=4), encoding="utf-8" + )