From eaa5c8dc64bad40d7561d3793a2a44703ab09798 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Mon, 7 Oct 2024 20:12:11 +0200 Subject: [PATCH] IDF Arduino libs compile (#84) --- builder/build_lib/CMakeLists.txt | 1 + builder/build_lib/arduino-lib-builder-as.S | 0 builder/build_lib/arduino-lib-builder-cpp.cpp | 0 builder/build_lib/arduino-lib-builder-gcc.c | 0 builder/build_lib/idf_component.yml | 18 +++ builder/build_lib/sketch.cpp | 10 ++ builder/frameworks/arduino.py | 37 +++++- builder/frameworks/espidf.py | 119 +++++++++++++++++- builder/main.py | 1 + platform.json | 2 +- platform.py | 2 + 11 files changed, 180 insertions(+), 10 deletions(-) create mode 100644 builder/build_lib/CMakeLists.txt create mode 100644 builder/build_lib/arduino-lib-builder-as.S create mode 100644 builder/build_lib/arduino-lib-builder-cpp.cpp create mode 100644 builder/build_lib/arduino-lib-builder-gcc.c create mode 100644 builder/build_lib/idf_component.yml create mode 100644 builder/build_lib/sketch.cpp diff --git a/builder/build_lib/CMakeLists.txt b/builder/build_lib/CMakeLists.txt new file mode 100644 index 000000000..1ab83d39a --- /dev/null +++ b/builder/build_lib/CMakeLists.txt @@ -0,0 +1 @@ +idf_component_register(SRCS "sketch.cpp" "arduino-lib-builder-gcc.c" "arduino-lib-builder-cpp.cpp" "arduino-lib-builder-as.S" INCLUDE_DIRS ".") diff --git a/builder/build_lib/arduino-lib-builder-as.S b/builder/build_lib/arduino-lib-builder-as.S new file mode 100644 index 000000000..e69de29bb diff --git a/builder/build_lib/arduino-lib-builder-cpp.cpp b/builder/build_lib/arduino-lib-builder-cpp.cpp new file mode 100644 index 000000000..e69de29bb diff --git a/builder/build_lib/arduino-lib-builder-gcc.c b/builder/build_lib/arduino-lib-builder-gcc.c new file mode 100644 index 000000000..e69de29bb diff --git a/builder/build_lib/idf_component.yml b/builder/build_lib/idf_component.yml new file mode 100644 index 000000000..6e5f92897 --- /dev/null +++ b/builder/build_lib/idf_component.yml @@ -0,0 +1,18 @@ +dependencies: + # Required IDF version + idf: ">=5.1" + espressif/cmake_utilities: + version: "0.*" + espressif/esp32-camera: + version: "master" + git: https://github.com/espressif/esp32-camera.git + require: public + rules: + - if: "target in [esp32, esp32s2, esp32s3, esp32p4]" + espressif/fb_gfx: + version: "master" + path: components/fb_gfx + git: https://github.com/espressif/esp32-arduino-lib-builder.git + require: public + rules: + - if: "target in [esp32, esp32s2, esp32s3, esp32p4]" diff --git a/builder/build_lib/sketch.cpp b/builder/build_lib/sketch.cpp new file mode 100644 index 000000000..10a17a8f2 --- /dev/null +++ b/builder/build_lib/sketch.cpp @@ -0,0 +1,10 @@ +#include "Arduino.h" + +void setup() { + Serial.begin(115200); +} + +void loop() { + Serial.println("Hello World!"); + delay(1000); +} diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index 51826e871..e8a3bcedc 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -25,6 +25,7 @@ import subprocess import json import semantic_version +import os from os.path import join from SCons.Script import COMMAND_LINE_TARGETS, DefaultEnvironment, SConscript @@ -32,6 +33,8 @@ env = DefaultEnvironment() platform = env.PioPlatform() +board = env.BoardConfig() +mcu = board.get("build.mcu", "esp32") extra_flags = ''.join([element.replace("-D", " ") for element in env.BoardConfig().get("build.extra_flags", "")]) build_flags = ''.join([element.replace("-D", " ") for element in env.GetProjectOption("build_flags")]) @@ -44,10 +47,31 @@ FRAMEWORK_DIR = platform.get_package_dir("framework-arduino-ITEAD") elif "arduino" in env.subst("$PIOFRAMEWORK") and "CORE32SOLO1" not in extra_flags and "FRAMEWORK_ARDUINO_SOLO1" not in build_flags and "CORE32ITEAD" not in extra_flags and "FRAMEWORK_ARDUINO_ITEAD" not in build_flags: FRAMEWORK_DIR = platform.get_package_dir("framework-arduinoespressif32") -ARDUINO_FRAMEWORK_DIR = FRAMEWORK_DIR -if "espidf" not in env.subst("$PIOFRAMEWORK"): - SConscript(join(FRAMEWORK_DIR, "tools", "platformio-build.py")) +# TODO not working check for "custom_sdkconfig" +# config = env.GetProjectConfig() +# flag_test = config.has_section("custom_sdkconfig") +# print("flag test", flag_test) + +flag_custom_sdkonfig = False +try: + if env.GetProjectOption("custom_sdkconfig"): + flag_custom_sdkonfig = True +except: + flag_custom_sdkonfig = False + +if flag_custom_sdkonfig == True: + custom_lib_config = join(platform.get_package_dir("framework-arduinoespressif32"),"tools","esp32-arduino-libs","sdkconfig."+env["PIOENV"]) + # check if custom libs are already compiled and there. TODO better check and remove old and restore standard... + if bool(os.path.isfile(custom_lib_config)): + flag_custom_sdkonfig = False + else: + flag_custom_sdkonfig = True + +if flag_custom_sdkonfig == True: + if env.subst("$ARDUINO_LIB_COMPILE_FLAG") in ("False", "Inactive"): + print("Compile Arduino IDF libs for %s" % env["PIOENV"]) + SConscript("espidf.py") def install_python_deps(): def _get_installed_pip_packages(): @@ -101,8 +125,11 @@ def _get_installed_pip_packages(): ] ) ), - "Installing Python dependencies", + "Installing Arduino Python dependencies", ) ) + return -install_python_deps() +if "arduino" in env.subst("$PIOFRAMEWORK") and "espidf" not in env.subst("$PIOFRAMEWORK") and env.subst("$ARDUINO_LIB_COMPILE_FLAG") in ("Inactive", "True"): + install_python_deps() + SConscript(join(FRAMEWORK_DIR, "tools", "platformio-build.py")) diff --git a/builder/frameworks/espidf.py b/builder/frameworks/espidf.py index 904218338..6277b8f8c 100644 --- a/builder/frameworks/espidf.py +++ b/builder/frameworks/espidf.py @@ -26,6 +26,7 @@ import sys import shutil import os +from os.path import join import re import platform as sys_platform @@ -89,7 +90,7 @@ # Arduino framework as a component is not compatible with ESP-IDF >5.2 if "arduino" in env.subst("$PIOFRAMEWORK"): - ARDUINO_FRAMEWORK_DIR = platform.get_package_dir("framework-arduinoespressif32") + ARDUINO_FRAMEWORK_DIR = platform.get_package_dir("framework-arduinoespressif32") # todo Tasmota ? # Possible package names in 'package@version' format is not compatible with CMake if "@" in os.path.basename(ARDUINO_FRAMEWORK_DIR): new_path = os.path.join( @@ -109,6 +110,71 @@ os.path.join(PROJECT_DIR, "sdkconfig.%s" % env.subst("$PIOENV")), )) +# +# generate modified Arduino IDF sdkconfig, applying settings from "custom_sdkconfig" +# +flag_custom_sdkonfig = False +try: # TODO better check if env exists + if env.GetProjectOption("custom_sdkconfig"): + flag_custom_sdkonfig = True +except: + flag_custom_sdkonfig = False + +def HandleArduinoIDFsettings(env): + if flag_custom_sdkonfig == True: + print("Add \"custom_sdkconfig\" settings to IDF sdkconfig.defaults!") + idf_config_flags = env.GetProjectOption("custom_sdkconfig").splitlines() + sdkconfig_src = join(ARDUINO_FRAMEWORK_DIR,"tools","esp32-arduino-libs",mcu,"sdkconfig") + + def get_flag(line): + if line.startswith("#") and "is not set" in line: + return line.split(" ")[1] + elif not line.startswith("#") and len(line.split("=")) > 1: + return line.split("=")[0] + else: + return None + + with open(sdkconfig_src) as src: + sdkconfig_dst = os.path.join(PROJECT_DIR, "sdkconfig.defaults") + dst = open(sdkconfig_dst,"w") + dst.write("# TASMOTA\n") + while line := src.readline(): + flag = get_flag(line) + # print(flag) + if flag is None: + dst.write(line) + else: + no_match = True + for item in idf_config_flags: + if flag in item: + dst.write(item+"\n") + no_match = False + print("Replace:",line," with: ",item) + if no_match: + dst.write(line) + dst.close() + return + else: + return + +if flag_custom_sdkonfig: + HandleArduinoIDFsettings(env) + LIB_SOURCE = os.path.join(env.subst("$PROJECT_CORE_DIR"), "platforms", "espressif32", "builder", "build_lib") + if not bool(os.path.exists(os.path.join(PROJECT_DIR, ".dummy"))): + shutil.copytree(LIB_SOURCE, os.path.join(PROJECT_DIR, ".dummy")) + PROJECT_SRC_DIR = os.path.join(PROJECT_DIR, ".dummy") + env.Replace( + PROJECT_SRC_DIR=PROJECT_SRC_DIR, + BUILD_FLAGS="", + BUILD_UNFLAGS="", + LINKFLAGS="", + PIOFRAMEWORK="arduino", + ARDUINO_LIB_COMPILE_FLAG="Build", + ) +# env.Append( +# BUILD_FLAGS="-mtext-section-literals" if mcu in ("esp32", "esp32s2", "esp32s3") else "" +# ) + env["INTEGRATION_EXTRA_DATA"].update({"arduino_lib_compile_flag": env.subst("$ARDUINO_LIB_COMPILE_FLAG")}) def get_project_lib_includes(env): project = ProjectAsLibBuilder(env, "$PROJECT_DIR") @@ -1060,7 +1126,8 @@ def generate_empty_partition_image(binary_path, image_size): ), ) - env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", empty_partition) + if flag_custom_sdkonfig == False: + env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", empty_partition) def get_partition_info(pt_path, pt_offset, pt_params): @@ -1557,7 +1624,8 @@ def get_python_exe(): # Compile bootloader # -env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", build_bootloader(sdk_config)) +if flag_custom_sdkonfig == False: + env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", build_bootloader(sdk_config)) # # Target: ESP-IDF menuconfig @@ -1772,6 +1840,44 @@ def _skip_prj_source_files(node): if os.path.isdir(ulp_dir) and os.listdir(ulp_dir) and mcu not in ("esp32c2", "esp32c3", "esp32h2"): env.SConscript("ulp.py", exports="env sdk_config project_config app_includes idf_variant") +# +# Compile Arduino sources +# + +if "arduino" in env.get("PIOFRAMEWORK") and "espidf" not in env.get("PIOFRAMEWORK"): + def idf_lib_copy(source, target, env): + lib_src = join(env["PROJECT_BUILD_DIR"],env["PIOENV"],"esp-idf") + lib_dst = join(ARDUINO_FRAMEWORK_DIR,"tools","esp32-arduino-libs",mcu,"lib") + src = [join(lib_src,x) for x in os.listdir(lib_src)] + src = [folder for folder in src if not os.path.isfile(folder)] # folders only + for folder in src: + # print(folder) + files = [join(folder,x) for x in os.listdir(folder)] + for file in files: + if file.strip().endswith(".a"): + # print(file.split("/")[-1]) + shutil.copyfile(file,join(lib_dst,file.split("/")[-1])) + if not bool(os.path.isfile(join(ARDUINO_FRAMEWORK_DIR,"tools","esp32-arduino-libs",mcu,"sdkconfig.orig"))): + shutil.move(join(ARDUINO_FRAMEWORK_DIR,"tools","esp32-arduino-libs",mcu,"sdkconfig"),join(ARDUINO_FRAMEWORK_DIR,"tools","esp32-arduino-libs",mcu,"sdkconfig.orig")) + shutil.copyfile(join(env.subst("$PROJECT_DIR"),"sdkconfig."+env["PIOENV"]),join(ARDUINO_FRAMEWORK_DIR,"tools","esp32-arduino-libs",mcu,"sdkconfig")) + shutil.copyfile(join(env.subst("$PROJECT_DIR"),"sdkconfig."+env["PIOENV"]),join(ARDUINO_FRAMEWORK_DIR,"tools","esp32-arduino-libs","sdkconfig."+env["PIOENV"])) + print("*** Copied compiled IDF libraries to Arduino framework ***") + + # TODO Check if Windows path is correct, see line 1427 + pio_exe_path = shutil.which("platformio"+(".exe" if IS_WINDOWS else "")) + # print("Platformio exe path", pio_exe_path) + pio_cmd = env["PIOENV"] + env.Execute( + env.VerboseAction( + ( + '"%s" run -e ' % pio_exe_path + + " ".join(['"%s"' % pio_cmd]) + ), + "Arduino compile %s with custom libraries" % pio_cmd, + ) + ) + env.AddPostAction("checkprogsize", idf_lib_copy) + # # Process OTA partition and image # @@ -1786,7 +1892,6 @@ def _skip_prj_source_files(node): # Generate an empty image if OTA is enabled in partition table ota_partition_image = os.path.join("$BUILD_DIR", "ota_data_initial.bin") if "arduino" in env.subst("$PIOFRAMEWORK"): - from arduino import ARDUINO_FRAMEWORK_DIR ota_partition_image = os.path.join(ARDUINO_FRAMEWORK_DIR, "tools", "partitions", "boot_app0.bin") else: generate_empty_partition_image(ota_partition_image, ota_partition_params["size"]) @@ -1801,6 +1906,12 @@ def _skip_prj_source_files(node): ) ] ) + EXTRA_IMG_DIR = join(env.subst("$PROJECT_DIR"), "variants", "tasmota") + env.Append( + FLASH_EXTRA_IMAGES=[ + (offset, join(EXTRA_IMG_DIR, img)) for offset, img in board.get("upload.arduino.flash_extra_images", []) + ] + ) def _parse_size(value): if isinstance(value, int): diff --git a/builder/main.py b/builder/main.py index ceee23ea4..9dca6b81f 100644 --- a/builder/main.py +++ b/builder/main.py @@ -291,6 +291,7 @@ def __fetch_fs_size(target, source, env): ), ESP32_APP_OFFSET=env.get("INTEGRATION_EXTRA_DATA").get("application_offset"), + ARDUINO_LIB_COMPILE_FLAG="Inactive", PROGSUFFIX=".elf" ) diff --git a/platform.json b/platform.json index 0255b3a01..92425ed11 100644 --- a/platform.json +++ b/platform.json @@ -51,7 +51,7 @@ "type": "framework", "optional": true, "owner": "Jason2866", - "version": "https://github.com/tasmota/esp-idf/releases/download/v5.3.1.240924/esp-idf-v5.3.1.zip" + "version": "https://github.com/tasmota/esp-idf/releases/download/v5.3.1.240926/esp-idf-v5.3.1.zip" }, "toolchain-xtensa-esp-elf": { "type": "toolchain", diff --git a/platform.py b/platform.py index dcfd9de95..97a4f41c9 100644 --- a/platform.py +++ b/platform.py @@ -44,6 +44,8 @@ def configure_default_packages(self, variables, targets): if variables.get("custom_sdkconfig") is not None: frameworks.append("espidf") + self.packages["framework-espidf"]["optional"] = False + self.packages["framework-arduinoespressif32"]["optional"] = False if "arduino" in frameworks: if "CORE32SOLO1" in core_variant_board or "FRAMEWORK_ARDUINO_SOLO1" in core_variant_build: