IDF Arduino libs compile (#84)

builder/build_lib/CMakeLists.txt

@@ -0,0 +1 @@
+idf_component_register(SRCS "sketch.cpp" "arduino-lib-builder-gcc.c" "arduino-lib-builder-cpp.cpp" "arduino-lib-builder-as.S" INCLUDE_DIRS ".")

builder/build_lib/idf_component.yml

@@ -0,0 +1,18 @@
+dependencies:
+  # Required IDF version
+  idf: ">=5.1"
+  espressif/cmake_utilities:
+    version: "0.*"
+  espressif/esp32-camera:
+    version: "master"
+    git: https://github.com/espressif/esp32-camera.git
+    require: public
+    rules:
+      - if: "target in [esp32, esp32s2, esp32s3, esp32p4]"
+  espressif/fb_gfx:
+    version: "master"
+    path: components/fb_gfx
+    git: https://github.com/espressif/esp32-arduino-lib-builder.git
+    require: public
+    rules:
+      - if: "target in [esp32, esp32s2, esp32s3, esp32p4]"

builder/build_lib/sketch.cpp

@@ -0,0 +1,10 @@
+#include "Arduino.h"
+
+void setup() {
+  Serial.begin(115200);
+}
+
+void loop() {
+  Serial.println("Hello World!");
+  delay(1000);
+}

builder/frameworks/arduino.py

@@ -25,13 +25,16 @@
 import subprocess
 import json
 import semantic_version
+import os
 from os.path import join
 
 from SCons.Script import COMMAND_LINE_TARGETS, DefaultEnvironment, SConscript
 from platformio.package.version import pepver_to_semver
 
 env = DefaultEnvironment()
 platform = env.PioPlatform()
+board = env.BoardConfig()
+mcu = board.get("build.mcu", "esp32")
 
 extra_flags = ''.join([element.replace("-D", " ") for element in env.BoardConfig().get("build.extra_flags", "")])
 build_flags = ''.join([element.replace("-D", " ") for element in env.GetProjectOption("build_flags")])
@@ -44,10 +47,31 @@
     FRAMEWORK_DIR = platform.get_package_dir("framework-arduino-ITEAD")
 elif "arduino" in env.subst("$PIOFRAMEWORK") and "CORE32SOLO1" not in extra_flags and "FRAMEWORK_ARDUINO_SOLO1" not in build_flags and "CORE32ITEAD" not in extra_flags and "FRAMEWORK_ARDUINO_ITEAD" not in build_flags:
     FRAMEWORK_DIR = platform.get_package_dir("framework-arduinoespressif32")
-ARDUINO_FRAMEWORK_DIR = FRAMEWORK_DIR
 
-if "espidf" not in env.subst("$PIOFRAMEWORK"):
-    SConscript(join(FRAMEWORK_DIR, "tools", "platformio-build.py"))
+# TODO not working check for "custom_sdkconfig"
+# config = env.GetProjectConfig()
+# flag_test = config.has_section("custom_sdkconfig")
+# print("flag test", flag_test)
+
+flag_custom_sdkonfig = False
+try:
+    if env.GetProjectOption("custom_sdkconfig"):
+        flag_custom_sdkonfig = True
+except:
+    flag_custom_sdkonfig = False
+
+if flag_custom_sdkonfig == True:
+    custom_lib_config = join(platform.get_package_dir("framework-arduinoespressif32"),"tools","esp32-arduino-libs","sdkconfig."+env["PIOENV"])
+    # check if custom libs are already compiled and there. TODO better check and remove old and restore standard...
+    if bool(os.path.isfile(custom_lib_config)):
+        flag_custom_sdkonfig = False
+    else:
+        flag_custom_sdkonfig = True
+
+if flag_custom_sdkonfig == True:
+    if env.subst("$ARDUINO_LIB_COMPILE_FLAG") in ("False", "Inactive"):
+        print("Compile Arduino IDF libs for %s" % env["PIOENV"])
+        SConscript("espidf.py")
 
 def install_python_deps():
     def _get_installed_pip_packages():
@@ -101,8 +125,11 @@ def _get_installed_pip_packages():
                         ]
                     )
                 ),
-                "Installing Python dependencies",
+                "Installing Arduino Python dependencies",
             )
         )
+    return
 
-install_python_deps()
+if "arduino" in env.subst("$PIOFRAMEWORK") and "espidf" not in env.subst("$PIOFRAMEWORK") and env.subst("$ARDUINO_LIB_COMPILE_FLAG") in ("Inactive", "True"):
+    install_python_deps()
+    SConscript(join(FRAMEWORK_DIR, "tools", "platformio-build.py"))

builder/frameworks/espidf.py

@@ -26,6 +26,7 @@
 import sys
 import shutil
 import os
+from os.path import join
 import re
 import platform as sys_platform
 
@@ -89,7 +90,7 @@
 
 # Arduino framework as a component is not compatible with ESP-IDF >5.2
 if "arduino" in env.subst("$PIOFRAMEWORK"):
-    ARDUINO_FRAMEWORK_DIR = platform.get_package_dir("framework-arduinoespressif32")
+    ARDUINO_FRAMEWORK_DIR = platform.get_package_dir("framework-arduinoespressif32") # todo Tasmota ?
     # Possible package names in 'package@version' format is not compatible with CMake
     if "@" in os.path.basename(ARDUINO_FRAMEWORK_DIR):
         new_path = os.path.join(
@@ -109,6 +110,71 @@
         os.path.join(PROJECT_DIR, "sdkconfig.%s" % env.subst("$PIOENV")),
 ))
 
+#
+# generate modified Arduino IDF sdkconfig, applying settings from "custom_sdkconfig"
+#
+flag_custom_sdkonfig = False
+try: # TODO better check if env exists
+    if env.GetProjectOption("custom_sdkconfig"):
+        flag_custom_sdkonfig = True
+except:
+    flag_custom_sdkonfig = False
+
+def HandleArduinoIDFsettings(env):
+    if flag_custom_sdkonfig == True:
+        print("Add \"custom_sdkconfig\" settings to IDF sdkconfig.defaults!")
+        idf_config_flags = env.GetProjectOption("custom_sdkconfig").splitlines()
+        sdkconfig_src = join(ARDUINO_FRAMEWORK_DIR,"tools","esp32-arduino-libs",mcu,"sdkconfig")
+
+        def get_flag(line):
+            if line.startswith("#") and "is not set" in line:
+                return line.split(" ")[1]
+            elif not line.startswith("#") and len(line.split("=")) > 1:
+                return line.split("=")[0]
+            else:
+                return None
+
+        with open(sdkconfig_src) as src:
+            sdkconfig_dst = os.path.join(PROJECT_DIR, "sdkconfig.defaults")
+            dst = open(sdkconfig_dst,"w")
+            dst.write("# TASMOTA\n")
+            while line := src.readline():
+                flag = get_flag(line)
+                # print(flag)
+                if flag is None:
+                    dst.write(line)
+                else:
+                    no_match = True
+                    for item in idf_config_flags:
+                        if flag in item:
+                            dst.write(item+"\n")
+                            no_match = False
+                            print("Replace:",line," with: ",item)
+                    if no_match:
+                        dst.write(line)
+            dst.close()
+        return
+    else:
+        return
+
+if flag_custom_sdkonfig:
+    HandleArduinoIDFsettings(env)
+    LIB_SOURCE = os.path.join(env.subst("$PROJECT_CORE_DIR"), "platforms", "espressif32", "builder", "build_lib")
+    if not bool(os.path.exists(os.path.join(PROJECT_DIR, ".dummy"))):
+        shutil.copytree(LIB_SOURCE, os.path.join(PROJECT_DIR, ".dummy"))
+    PROJECT_SRC_DIR = os.path.join(PROJECT_DIR, ".dummy")
+    env.Replace(
+        PROJECT_SRC_DIR=PROJECT_SRC_DIR,
+        BUILD_FLAGS="",
+        BUILD_UNFLAGS="",
+        LINKFLAGS="",
+        PIOFRAMEWORK="arduino",
+        ARDUINO_LIB_COMPILE_FLAG="Build",
+    )
+#    env.Append(
+#        BUILD_FLAGS="-mtext-section-literals" if mcu in ("esp32", "esp32s2", "esp32s3") else ""
+#    )
+    env["INTEGRATION_EXTRA_DATA"].update({"arduino_lib_compile_flag": env.subst("$ARDUINO_LIB_COMPILE_FLAG")})
 
 def get_project_lib_includes(env):
     project = ProjectAsLibBuilder(env, "$PROJECT_DIR")
@@ -1060,7 +1126,8 @@ def generate_empty_partition_image(binary_path, image_size):
         ),
     )
 
-    env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", empty_partition)
+    if flag_custom_sdkonfig == False:
+        env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", empty_partition)
 
 
 def get_partition_info(pt_path, pt_offset, pt_params):
@@ -1557,7 +1624,8 @@ def get_python_exe():
 # Compile bootloader
 #
 
-env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", build_bootloader(sdk_config))
+if flag_custom_sdkonfig == False:
+    env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", build_bootloader(sdk_config))
 
 #
 # Target: ESP-IDF menuconfig
@@ -1772,6 +1840,44 @@ def _skip_prj_source_files(node):
 if os.path.isdir(ulp_dir) and os.listdir(ulp_dir) and mcu not in ("esp32c2", "esp32c3", "esp32h2"):
     env.SConscript("ulp.py", exports="env sdk_config project_config app_includes idf_variant")
 
+#
+# Compile Arduino sources
+#
+
+if "arduino" in env.get("PIOFRAMEWORK") and "espidf" not in env.get("PIOFRAMEWORK"):
+    def idf_lib_copy(source, target, env):
+        lib_src = join(env["PROJECT_BUILD_DIR"],env["PIOENV"],"esp-idf")
+        lib_dst = join(ARDUINO_FRAMEWORK_DIR,"tools","esp32-arduino-libs",mcu,"lib")
+        src = [join(lib_src,x) for x in os.listdir(lib_src)]
+        src = [folder for folder in src if not os.path.isfile(folder)] # folders only
+        for folder in src:
+            # print(folder)
+            files = [join(folder,x) for x in os.listdir(folder)]
+            for file in files:
+                if file.strip().endswith(".a"):
+                    # print(file.split("/")[-1])
+                    shutil.copyfile(file,join(lib_dst,file.split("/")[-1]))
+        if not bool(os.path.isfile(join(ARDUINO_FRAMEWORK_DIR,"tools","esp32-arduino-libs",mcu,"sdkconfig.orig"))):
+            shutil.move(join(ARDUINO_FRAMEWORK_DIR,"tools","esp32-arduino-libs",mcu,"sdkconfig"),join(ARDUINO_FRAMEWORK_DIR,"tools","esp32-arduino-libs",mcu,"sdkconfig.orig"))
+        shutil.copyfile(join(env.subst("$PROJECT_DIR"),"sdkconfig."+env["PIOENV"]),join(ARDUINO_FRAMEWORK_DIR,"tools","esp32-arduino-libs",mcu,"sdkconfig"))
+        shutil.copyfile(join(env.subst("$PROJECT_DIR"),"sdkconfig."+env["PIOENV"]),join(ARDUINO_FRAMEWORK_DIR,"tools","esp32-arduino-libs","sdkconfig."+env["PIOENV"]))
+        print("*** Copied compiled IDF libraries to Arduino framework ***")
+
+        # TODO Check if Windows path is correct, see line 1427
+        pio_exe_path = shutil.which("platformio"+(".exe" if IS_WINDOWS else ""))
+        # print("Platformio exe path", pio_exe_path)
+        pio_cmd = env["PIOENV"]
+        env.Execute(
+            env.VerboseAction(
+                (
+                    '"%s" run -e ' % pio_exe_path
+                    + " ".join(['"%s"' % pio_cmd])
+                ),
+                "Arduino compile %s with custom libraries" % pio_cmd,
+            )
+        )
+    env.AddPostAction("checkprogsize", idf_lib_copy)
+
 #
 # Process OTA partition and image
 #
@@ -1786,7 +1892,6 @@ def _skip_prj_source_files(node):
     # Generate an empty image if OTA is enabled in partition table
     ota_partition_image = os.path.join("$BUILD_DIR", "ota_data_initial.bin")
     if "arduino" in env.subst("$PIOFRAMEWORK"):
-        from arduino import ARDUINO_FRAMEWORK_DIR
         ota_partition_image = os.path.join(ARDUINO_FRAMEWORK_DIR, "tools", "partitions", "boot_app0.bin")
     else:
         generate_empty_partition_image(ota_partition_image, ota_partition_params["size"])
@@ -1801,6 +1906,12 @@ def _skip_prj_source_files(node):
             )
         ]
     )
+    EXTRA_IMG_DIR = join(env.subst("$PROJECT_DIR"), "variants", "tasmota")
+    env.Append(
+        FLASH_EXTRA_IMAGES=[
+            (offset, join(EXTRA_IMG_DIR, img)) for offset, img in board.get("upload.arduino.flash_extra_images", [])
+        ]
+    )
 
 def _parse_size(value):
     if isinstance(value, int):

builder/main.py

@@ -291,6 +291,7 @@ def __fetch_fs_size(target, source, env):
     ),
 
     ESP32_APP_OFFSET=env.get("INTEGRATION_EXTRA_DATA").get("application_offset"),
+    ARDUINO_LIB_COMPILE_FLAG="Inactive",
 
     PROGSUFFIX=".elf"
 )

platform.json

@@ -51,7 +51,7 @@
       "type": "framework",
       "optional": true,
       "owner": "Jason2866",
-      "version": "https://github.com/tasmota/esp-idf/releases/download/v5.3.1.240924/esp-idf-v5.3.1.zip"
+      "version": "https://github.com/tasmota/esp-idf/releases/download/v5.3.1.240926/esp-idf-v5.3.1.zip"
     },
     "toolchain-xtensa-esp-elf": {
       "type": "toolchain",

platform.py

@@ -44,6 +44,8 @@ def configure_default_packages(self, variables, targets):
 
         if variables.get("custom_sdkconfig") is not None:
             frameworks.append("espidf")
+            self.packages["framework-espidf"]["optional"] = False
+            self.packages["framework-arduinoespressif32"]["optional"] = False
 
         if "arduino" in frameworks:
             if "CORE32SOLO1" in core_variant_board or "FRAMEWORK_ARDUINO_SOLO1" in core_variant_build: