Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[platform] Install ltchiptool in separate virtual environment #166

Merged
merged 5 commits into from
Sep 10, 2023
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
[platform] Install ltchiptool in separate virtual environment
kuba2k2 committed Sep 7, 2023

Verified

This commit was signed with the committer’s verified signature.
kuba2k2 Kuba Szczodrzyński
commit 935ffdf0c629829f39c8a4470cdebdf419d6acc0
9 changes: 8 additions & 1 deletion builder/main.py
Original file line number Diff line number Diff line change
@@ -17,14 +17,21 @@
platform: PlatformBase = env.PioPlatform()
board: PlatformBoardConfig = env.BoardConfig()

python_deps = {
"ltchiptool": ">=4.5.0,<5.0",
}
env.SConscript("python-venv.py", exports="env")
env.ConfigurePythonVenv()
env.InstallPythonDependencies(python_deps)

# Utilities
env.SConscript("utils/config.py", exports="env")
env.SConscript("utils/cores.py", exports="env")
env.SConscript("utils/env.py", exports="env")
env.SConscript("utils/flash.py", exports="env")
env.SConscript("utils/libs-external.py", exports="env")
env.SConscript("utils/libs-queue.py", exports="env")
env.SConscript("utils/ltchiptool.py", exports="env")
env.SConscript("utils/ltchiptool-util.py", exports="env")

# Firmware name
if env.get("PROGNAME", "program") == "program":
94 changes: 94 additions & 0 deletions builder/python-venv.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
# Copyright (c) Kuba Szczodrzyński 2023-09-07.

import json
import site
import subprocess
from pathlib import Path

import semantic_version
from platformio.compat import IS_WINDOWS
from platformio.package.version import pepver_to_semver
from platformio.platform.base import PlatformBase
from SCons.Script import DefaultEnvironment, Environment

env: Environment = DefaultEnvironment()
platform: PlatformBase = env.PioPlatform()

# code borrowed and modified from espressif32/builder/frameworks/espidf.py


def env_configure_python_venv(env: Environment):
venv_path = Path(env.subst("${PROJECT_CORE_DIR}"), "penv", ".libretiny")

if not venv_path.is_dir():
pip_path = venv_path.joinpath(
"Scripts" if IS_WINDOWS else "bin",
"pip" + (".exe" if IS_WINDOWS else ""),
)
if not pip_path.is_file():
# Use the built-in PlatformIO Python to create a standalone virtual env
env.Execute(
env.VerboseAction(
f'"${PYTHONEXE}" -m venv --clear "{venv_path.absolute()}"',
"LibreTiny: Creating a virtual environment for Python dependencies",
)
)

assert (
pip_path.is_file()
), "Error: Failed to create a proper virtual environment. Missing the pip binary!"

python_path = venv_path.joinpath(
"Scripts" if IS_WINDOWS else "bin",
"python" + (".exe" if IS_WINDOWS else ""),
)

assert (
python_path.is_file()
), f"Error: Missing Python executable file `{python_path.absolute()}`"

env.Replace(LTPYTHONEXE=python_path.absolute(), LTPYTHONENV=venv_path.absolute())
site.addsitedir(str(venv_path.absolute()))


def env_install_python_dependencies(env: Environment, dependencies: dict):
try:
pip_output = subprocess.check_output(
[
env.subst("${LTPYTHONEXE}"),
"-m",
"pip",
"list",
"--format=json",
"--disable-pip-version-check",
]
)
pip_data = json.loads(pip_output)
packages = {p["name"]: pepver_to_semver(p["version"]) for p in pip_data}
except:
print(
"LibreTiny: Warning! Couldn't extract the list of installed Python packages"
)
packages = {}

to_install = []
for name, spec in dependencies.items():
install_spec = f'"{name}{dependencies[name]}"'
if name not in packages:
to_install.append(install_spec)
elif spec:
version_spec = semantic_version.Spec(spec)
if not version_spec.match(packages[name]):
to_install.append(install_spec)

if to_install:
env.Execute(
env.VerboseAction(
'"${LTPYTHONEXE}" -m pip install -U ' + " ".join(to_install),
"LibreTiny: Installing Python dependencies",
)
)


env.AddMethod(env_configure_python_venv, "ConfigurePythonVenv")
env.AddMethod(env_install_python_dependencies, "InstallPythonDependencies")
2 changes: 1 addition & 1 deletion builder/utils/env.py
Original file line number Diff line number Diff line change
@@ -77,7 +77,7 @@ def env_configure(
# ltchiptool config:
# -r output raw log messages
# -i 1 indent log messages
LTCHIPTOOL='"${PYTHONEXE}" -m ltchiptool -r -i 1',
LTCHIPTOOL='"${LTPYTHONEXE}" -m ltchiptool -r -i 1 -L "${LT_DIR}"',
# Fix for link2bin to get tmpfile name in argv
LINKCOM="${LINK} ${LINKARGS}",
LINKARGS="${TEMPFILE('-o $TARGET $LINKFLAGS $__RPATH $SOURCES $_LIBDIRFLAGS $_LIBFLAGS', '$LINKCOMSTR')}",
File renamed without changes.
90 changes: 14 additions & 76 deletions platform.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
# Copyright (c) Kuba Szczodrzyński 2022-04-20.

import importlib
import json
import os
import platform
import site
import sys
from os.path import dirname
from subprocess import Popen
from pathlib import Path
from typing import Dict, List

import click
@@ -15,73 +15,8 @@
from platformio.package.meta import PackageItem
from platformio.platform.base import PlatformBase
from platformio.platform.board import PlatformBoardConfig
from semantic_version import SimpleSpec, Version

LTCHIPTOOL_VERSION = "^4.2.3"


# Install & import tools
def check_ltchiptool(install: bool):
if install:
# update ltchiptool to a supported version
print("Installing/updating ltchiptool")
p = Popen(
[
sys.executable,
"-m",
"pip",
"install",
"-U",
"--force-reinstall",
f"ltchiptool >= {LTCHIPTOOL_VERSION[1:]}, < 5.0",
],
)
p.wait()

# unload all modules from the old version
for name, module in list(sorted(sys.modules.items())):
if not name.startswith("ltchiptool"):
continue
del sys.modules[name]
del module

# try to import it
ltchiptool = importlib.import_module("ltchiptool")

# check if the version is known
version = Version.coerce(ltchiptool.get_version()).truncate()
if version in SimpleSpec(LTCHIPTOOL_VERSION):
return
if not install:
raise ImportError(f"Version incompatible: {version}")


def try_check_ltchiptool():
install_modes = [False, True]
exception = None
for install in install_modes:
try:
check_ltchiptool(install)
return
except (ImportError, AttributeError) as ex:
exception = ex
print(
"!!! Installing ltchiptool failed, or version outdated. "
"Please install ltchiptool manually using pip. "
f"Cannot continue. {type(exception).name}: {exception}"
)
raise exception


try_check_ltchiptool()
import ltchiptool

# Remove current dir so it doesn't conflict with PIO
if dirname(__file__) in sys.path:
sys.path.remove(dirname(__file__))

# Let ltchiptool know about LT's location
ltchiptool.lt_set_path(dirname(__file__))
site.addsitedir(Path(__file__).absolute().parent.joinpath("tools"))


def get_os_specifiers():
@@ -119,6 +54,12 @@ def __init__(self, manifest_path):
super().__init__(manifest_path)
self.custom_opts = {}
self.versions = {}
self.verbose = (
"-v" in sys.argv
or "--verbose" in sys.argv
or "PIOVERBOSE=1" in sys.argv
or os.environ.get("PIOVERBOSE", "0") == "1"
)

def print(self, *args, **kwargs):
if not self.verbose:
@@ -137,11 +78,8 @@ def get_package_spec(self, name, version=None):
return spec

def configure_default_packages(self, options: dict, targets: List[str]):
from ltchiptool.util.dict import RecursiveDict
from libretiny import RecursiveDict

self.verbose = (
"-v" in sys.argv or "--verbose" in sys.argv or "PIOVERBOSE=1" in sys.argv
)
self.print(f"configure_default_packages(targets={targets})")

pioframework = options.get("pioframework") or ["base"]
@@ -298,19 +236,19 @@ def get_boards(self, id_=None):
return result

def update_board(self, board: PlatformBoardConfig):
from libretiny import Board, Family, merge_dicts

if "_base" in board:
board._manifest = ltchiptool.Board.get_data(board._manifest)
board._manifest = Board.get_data(board._manifest)
board._manifest.pop("_base")

if self.custom("board"):
from ltchiptool.util.dict import merge_dicts

with open(self.custom("board"), "r") as f:
custom_board = json.load(f)
board._manifest = merge_dicts(board._manifest, custom_board)

family = board.get("build.family")
family = ltchiptool.Family.get(short_name=family)
family = Family.get(short_name=family)
# add "frameworks" key with the default "base"
board.manifest["frameworks"] = ["base"]
# add "arduino" framework if supported
14 changes: 14 additions & 0 deletions tools/libretiny/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# Copyright (c) Kuba Szczodrzyński 2023-09-07.

from .board import Board
from .dict import RecursiveDict, merge_dicts
from .family import Family

# TODO refactor and remove all this from here

__all__ = [
"Board",
"Family",
"RecursiveDict",
"merge_dicts",
]
34 changes: 34 additions & 0 deletions tools/libretiny/board.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
# Copyright (c) Kuba Szczodrzyński 2022-07-29.

from typing import Union

from genericpath import isfile

from .dict import merge_dicts
from .fileio import readjson
from .lvm import lvm_load_json


class Board:
@staticmethod
def get_data(board: Union[str, dict]) -> dict:
if not isinstance(board, dict):
if isfile(board):
board = readjson(board)
if not board:
raise FileNotFoundError(f"Board not found: {board}")
else:
source = board
board = lvm_load_json(f"boards/{board}.json")
board["source"] = source
if "_base" in board:
base = board["_base"]
if not isinstance(base, list):
base = [base]
result = {}
for base_name in base:
board_base = lvm_load_json(f"boards/_base/{base_name}.json")
merge_dicts(result, board_base)
merge_dicts(result, board)
board = result
return board
65 changes: 65 additions & 0 deletions tools/libretiny/dict.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
# Copyright (c) Kuba Szczodrzyński 2022-07-29.

from .obj import get, has, pop, set_


class RecursiveDict(dict):
def __init__(self, data: dict = None):
if data:
data = {
key: RecursiveDict(value) if isinstance(value, dict) else value
for key, value in data.items()
}
super().__init__(data)
else:
super().__init__()

def __getitem__(self, key):
if "." not in key:
return super().get(key, None)
return get(self, key)

def __setitem__(self, key, value):
if "." not in key:
return super().__setitem__(key, value)
set_(self, key, value, newtype=RecursiveDict)

def __delitem__(self, key):
if "." not in key:
return super().pop(key, None)
return pop(self, key)

def __contains__(self, key) -> bool:
if "." not in key:
return super().__contains__(key)
return has(self, key)

def get(self, key, default=None):
if "." not in key:
return super().get(key, default)
return get(self, key) or default

def pop(self, key, default=None):
if "." not in key:
return super().pop(key, default)
return pop(self, key, default)


def merge_dicts(d1, d2):
# force RecursiveDict instances to be treated as regular dicts
d1_type = dict if isinstance(d1, RecursiveDict) else type(d1)
d2_type = dict if isinstance(d2, RecursiveDict) else type(d2)
if d1 is not None and d1_type != d2_type:
raise TypeError(f"d1 and d2 are of different types: {type(d1)} vs {type(d2)}")
if isinstance(d2, list):
if d1 is None:
d1 = []
d1.extend(merge_dicts(None, item) for item in d2)
elif isinstance(d2, dict):
if d1 is None:
d1 = {}
for key in d2:
d1[key] = merge_dicts(d1.get(key, None), d2[key])
else:
d1 = d2
return d1
97 changes: 97 additions & 0 deletions tools/libretiny/family.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
# Copyright (c) Kuba Szczodrzyński 2022-06-02.

from dataclasses import dataclass, field
from typing import List, Optional, Union

from .lvm import lvm_load_json, lvm_path

LT_FAMILIES: List["Family"] = []


@dataclass
class Family:
name: str
parent: Union["Family", None]
code: str
description: str
id: Optional[int] = None
short_name: Optional[str] = None
package: Optional[str] = None
mcus: List[str] = field(default_factory=lambda: [])
children: List["Family"] = field(default_factory=lambda: [])

# noinspection PyTypeChecker
def __post_init__(self):
if self.id:
self.id = int(self.id, 16)
self.mcus = set(self.mcus)

@classmethod
def get_all(cls) -> List["Family"]:
global LT_FAMILIES
if LT_FAMILIES:
return LT_FAMILIES
families = lvm_load_json("families.json")
LT_FAMILIES = [
cls(name=k, **v) for k, v in families.items() if isinstance(v, dict)
]
# attach parents and children to all families
for family in LT_FAMILIES:
if family.parent is None:
continue
try:
parent = next(f for f in LT_FAMILIES if f.name == family.parent)
except StopIteration:
raise ValueError(
f"Family parent '{family.parent}' of '{family.name}' doesn't exist"
)
family.parent = parent
parent.children.append(family)
return LT_FAMILIES

@classmethod
def get(
cls,
any: str = None,
id: Union[str, int] = None,
short_name: str = None,
name: str = None,
code: str = None,
description: str = None,
) -> "Family":
if any:
id = any
short_name = any
name = any
code = any
description = any
if id and isinstance(id, str) and id.startswith("0x"):
id = int(id, 16)
for family in cls.get_all():
if id and family.id == id:
return family
if short_name and family.short_name == short_name.upper():
return family
if name and family.name == name.lower():
return family
if code and family.code == code.lower():
return family
if description and family.description == description:
return family
if any:
raise ValueError(f"Family not found - {any}")
items = [hex(id) if id else None, short_name, name, code, description]
text = ", ".join(filter(None, items))
raise ValueError(f"Family not found - {text}")

@property
def has_arduino_core(self) -> bool:
if lvm_path().joinpath("cores", self.name, "arduino").is_dir():
return True
if self.parent:
return self.parent.has_arduino_core
return False

@property
def target_package(self) -> Optional[str]:
return self.package or self.parent and self.parent.target_package
17 changes: 17 additions & 0 deletions tools/libretiny/fileio.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# Copyright (c) Kuba Szczodrzyński 2022-06-10.

import json
from json import JSONDecodeError
from os.path import isfile
from typing import Optional, Union


def readjson(file: str) -> Optional[Union[dict, list]]:
"""Read a JSON file into a dict or list."""
if not isfile(file):
return None
with open(file, "r", encoding="utf-8") as f:
try:
return json.load(f)
except JSONDecodeError:
return None
19 changes: 19 additions & 0 deletions tools/libretiny/lvm.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# Copyright (c) Kuba Szczodrzyński 2023-3-18.

import json
from pathlib import Path
from typing import Dict, Union

json_cache: Dict[str, Union[list, dict]] = {}
libretiny_path = Path(__file__).parents[2]


def lvm_load_json(path: str) -> Union[list, dict]:
if path not in json_cache:
with libretiny_path.joinpath(path).open("rb") as f:
json_cache[path] = json.load(f)
return json_cache[path]


def lvm_path() -> Path:
return libretiny_path
62 changes: 62 additions & 0 deletions tools/libretiny/obj.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
# Copyright (c) Kuba Szczodrzyński 2022-06-02.

from enum import Enum
from typing import Type

# The following helpers force using base dict class' methods.
# Because RecursiveDict uses these helpers, this prevents it
# from running into endless nested loops.


def get(data: dict, path: str):
if not isinstance(data, dict) or not path:
return None
if dict.__contains__(data, path):
return dict.get(data, path, None)
key, _, path = path.partition(".")
return get(dict.get(data, key, None), path)


def pop(data: dict, path: str, default=None):
if not isinstance(data, dict) or not path:
return default
if dict.__contains__(data, path):
return dict.pop(data, path, default)
key, _, path = path.partition(".")
return pop(dict.get(data, key, None), path, default)


def has(data: dict, path: str) -> bool:
if not isinstance(data, dict) or not path:
return False
if dict.__contains__(data, path):
return True
key, _, path = path.partition(".")
return has(dict.get(data, key, None), path)


def set_(data: dict, path: str, value, newtype=dict):
if not isinstance(data, dict) or not path:
return
# can't use __contains__ here, as we're setting,
# so it's obvious 'data' doesn't have the item
if "." not in path:
dict.__setitem__(data, path, value)
else:
key, _, path = path.partition(".")
# allow creation of parent objects
if key in data:
sub_data = dict.__getitem__(data, key)
else:
sub_data = newtype()
dict.__setitem__(data, key, sub_data)
set_(sub_data, path, value)


def str2enum(cls: Type[Enum], key: str):
if not key:
return None
try:
return next(e for e in cls if e.name.lower() == key.lower())
except StopIteration:
return None