From b07d9cbb699be71a64dc0f927c26d03fa12d5f9c Mon Sep 17 00:00:00 2001 From: valeros Date: Fri, 10 Jan 2025 14:13:03 +0200 Subject: [PATCH 1/5] Update IDF package to v5.4 Resolves #1526 --- platform.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform.json b/platform.json index ee0745b68..74c3078c6 100644 --- a/platform.json +++ b/platform.json @@ -88,7 +88,7 @@ "type": "framework", "optional": true, "owner": "platformio", - "version": "~3.50301.0", + "version": "~3.50400.0", "optionalVersions": ["~3.40407.0"] }, "tool-esptoolpy": { From aab73c61b35f5a98a695a2baed4c580e3cb218cb Mon Sep 17 00:00:00 2001 From: valeros Date: Fri, 10 Jan 2025 14:13:39 +0200 Subject: [PATCH 2/5] Update toolchain packages to v14.2.0 --- platform.json | 4 ++-- platform.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/platform.json b/platform.json index 74c3078c6..ddb4a23f2 100644 --- a/platform.json +++ b/platform.json @@ -52,13 +52,13 @@ "optional": true, "owner": "espressif", "version": "8.4.0+2021r2-patch5", - "optionalVersions": ["13.2.0+20240530"] + "optionalVersions": ["14.2.0+20241119"] }, "toolchain-xtensa-esp-elf": { "type": "toolchain", "optional": true, "owner": "platformio", - "version": "13.2.0+20240530" + "version": "14.2.0+20241119" }, "toolchain-esp32ulp": { "type": "toolchain", diff --git a/platform.py b/platform.py index 895948e06..e7bdf9848 100644 --- a/platform.py +++ b/platform.py @@ -137,7 +137,7 @@ def configure_default_packages(self, variables, targets): self.packages["toolchain-riscv32-esp"]["owner"] = "platformio" self.packages["toolchain-riscv32-esp"][ "version" - ] = "13.2.0+20240530" + ] = "14.2.0+20241119" if "arduino" in frameworks: # Disable standalone GDB packages for Arduino and Arduino/IDF projects From 1442a35adbab75051d28670aef4cae90711e393f Mon Sep 17 00:00:00 2001 From: valeros Date: Fri, 10 Jan 2025 14:16:58 +0200 Subject: [PATCH 3/5] Update ULP build script - Added support for ESP32-C6 ULP - Updates according to changes in IDF v5.4.0 Resolves #1507 --- builder/frameworks/espidf.py | 6 ++- builder/frameworks/ulp.py | 90 ++++++++++++++++++++++++++++-------- 2 files changed, 74 insertions(+), 22 deletions(-) diff --git a/builder/frameworks/espidf.py b/builder/frameworks/espidf.py index 8f03c0410..7cbe81644 100644 --- a/builder/frameworks/espidf.py +++ b/builder/frameworks/espidf.py @@ -1804,8 +1804,10 @@ def _skip_prj_source_files(node): # ulp_dir = os.path.join(PROJECT_DIR, "ulp") -if os.path.isdir(ulp_dir) and os.listdir(ulp_dir) and mcu not in ("esp32c3", "esp32c6"): - env.SConscript("ulp.py", exports="env sdk_config project_config idf_variant") +if os.path.isdir(ulp_dir) and os.listdir(ulp_dir) and mcu != "esp32c3": + env.SConscript( + "ulp.py", exports="env sdk_config project_config idf_variant" + ) # # Process OTA partition and image diff --git a/builder/frameworks/ulp.py b/builder/frameworks/ulp.py index 89aa42011..982742295 100644 --- a/builder/frameworks/ulp.py +++ b/builder/frameworks/ulp.py @@ -28,7 +28,10 @@ FRAMEWORK_DIR = platform.get_package_dir("framework-espidf") BUILD_DIR = ulp_env.subst("$BUILD_DIR") ULP_BUILD_DIR = os.path.join( - BUILD_DIR, "esp-idf", project_config["name"].replace("__idf_", ""), "ulp_main" + BUILD_DIR, + "esp-idf", + project_config["name"].replace("__idf_", ""), + "ulp_main", ) @@ -36,9 +39,15 @@ def prepare_ulp_env_vars(env): ulp_env.PrependENVPath("IDF_PATH", FRAMEWORK_DIR) toolchain_path = platform.get_package_dir( - "toolchain-xtensa-esp-elf" - if "arduino" not in env.subst("$PIOFRAMEWORK") - else "toolchain-xtensa-%s" % idf_variant + "toolchain-riscv32-esp" + if idf_variant in ("esp32c3", "esp32c6") + else ( + ( + "toolchain-xtensa-esp-elf" + if "arduino" not in env.subst("$PIOFRAMEWORK") + else "toolchain-xtensa-%s" % idf_variant + ) + ) ) additional_packages = [ @@ -72,9 +81,9 @@ def get_component_includes(target_config): if source["path"].endswith("ulp_main.bin.S"): return [ inc["path"] - for inc in target_config["compileGroups"][source["compileGroupIndex"]][ - "includes" - ] + for inc in target_config["compileGroups"][ + source["compileGroupIndex"] + ]["includes"] ] return [os.path.join(BUILD_DIR, "config")] @@ -85,7 +94,9 @@ def _generate_ulp_configuration_action(env, target, source): riscv_ulp_enabled = sdk_config.get("ULP_COPROC_TYPE_RISCV", False) cmd = ( - os.path.join(platform.get_package_dir("tool-cmake"), "bin", "cmake"), + os.path.join( + platform.get_package_dir("tool-cmake"), "bin", "cmake" + ), "-DCMAKE_GENERATOR=Ninja", "-DCMAKE_TOOLCHAIN_FILE=" + os.path.join( @@ -93,21 +104,52 @@ def _generate_ulp_configuration_action(env, target, source): "components", "ulp", "cmake", - "toolchain-%sulp%s.cmake" - % ( - "" if riscv_ulp_enabled else idf_variant + "-", - "-riscv" if riscv_ulp_enabled else "", + ( + "toolchain-lp-core-riscv.cmake" + if sdk_config.get("ULP_COPROC_TYPE_LP_CORE", False) + else "toolchain-%sulp%s.cmake" + % ( + "" if riscv_ulp_enabled else idf_variant + "-", + "-riscv" if riscv_ulp_enabled else "", + ) ), ), - "-DULP_S_SOURCES=%s" % ";".join([fs.to_unix_path(s.get_abspath()) for s in source]), + "-DULP_S_SOURCES=%s" + % ";".join([fs.to_unix_path(s.get_abspath()) for s in source]), "-DULP_APP_NAME=ulp_main", - "-DCOMPONENT_DIR=" + os.path.join(ulp_env.subst("$PROJECT_DIR"), "ulp"), - "-DCOMPONENT_INCLUDES=%s" % ";".join(get_component_includes(target_config)), + "-DCOMPONENT_DIR=" + + os.path.join(ulp_env.subst("$PROJECT_DIR"), "ulp"), + "-DCOMPONENT_INCLUDES=%s" + % ";".join(get_component_includes(target_config)), "-DIDF_TARGET=%s" % idf_variant, "-DIDF_PATH=" + fs.to_unix_path(FRAMEWORK_DIR), - "-DSDKCONFIG_HEADER=" + os.path.join(BUILD_DIR, "config", "sdkconfig.h"), + "-DSDKCONFIG_HEADER=" + + os.path.join(BUILD_DIR, "config", "sdkconfig.h"), "-DPYTHON=" + env.subst("$PYTHONEXE"), - "-DSDKCONFIG_CMAKE=" + os.path.join(BUILD_DIR, "config", "sdkconfig.cmake"), + "-DSDKCONFIG_CMAKE=" + + os.path.join(BUILD_DIR, "config", "sdkconfig.cmake"), + "-DULP_COCPU_IS_RISCV=" + + ( + "ON" + if idf_variant in ("esp32s2", "esp32s3") + and sdk_config.get("ULP_COPROC_TYPE_RISCV", False) + else "" + ), + "-DULP_COCPU_IS_LP_CORE=" + + ( + "ON" + if sdk_config.get("ULP_COPROC_TYPE_LP_CORE", False) + else "" + ), + "-DESP_ROM_HAS_LP_ROM=" + + ( + "ON" + if sdk_config.get("ESP_ROM_HAS_LP_ROM", False) + else "" + ), + "-DCMAKE_MODULE_PATH=" + + fs.to_unix_path( + os.path.join(FRAMEWORK_DIR, "components", "ulp", "cmake")), "-GNinja", "-B", ULP_BUILD_DIR, @@ -152,7 +194,9 @@ def compile_ulp_binary(): os.path.join(ULP_BUILD_DIR, "ulp_main.bin"), ], None, - ulp_binary_env.VerboseAction(" ".join(cmd), "Generating ULP project files $TARGETS"), + ulp_binary_env.VerboseAction( + " ".join(cmd), "Generating ULP project files $TARGETS" + ), ) @@ -164,14 +208,20 @@ def generate_ulp_assembly(): "-DFILE_TYPE=BINARY", "-P", os.path.join( - FRAMEWORK_DIR, "tools", "cmake", "scripts", "data_file_embed_asm.cmake" + FRAMEWORK_DIR, + "tools", + "cmake", + "scripts", + "data_file_embed_asm.cmake", ), ) return ulp_env.Command( os.path.join(BUILD_DIR, "ulp_main.bin.S"), os.path.join(ULP_BUILD_DIR, "ulp_main.bin"), - ulp_env.VerboseAction(" ".join(cmd), "Generating ULP assembly file $TARGET"), + ulp_env.VerboseAction( + " ".join(cmd), "Generating ULP assembly file $TARGET" + ), ) From 5a74961dcd25e61ea3777a48a6130f89244391c7 Mon Sep 17 00:00:00 2001 From: valeros Date: Fri, 10 Jan 2025 16:29:19 +0200 Subject: [PATCH 4/5] Better handling of IDF virtual environment Resolves #1525 --- builder/frameworks/espidf.py | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/builder/frameworks/espidf.py b/builder/frameworks/espidf.py index 7cbe81644..99b89f75a 100644 --- a/builder/frameworks/espidf.py +++ b/builder/frameworks/espidf.py @@ -1343,11 +1343,37 @@ def get_idf_venv_dir(): def ensure_python_venv_available(): + def _get_idf_venv_python_version(): + try: + version = subprocess.check_output( + [ + get_python_exe(), + "-c", + "import sys;print('{0}.{1}.{2}-{3}.{4}'.format(*list(sys.version_info)))" + ], text=True + ) + return version.strip() + except subprocess.CalledProcessError as e: + print("Failed to extract Python version from IDF virtual env!") + return None + def _is_venv_outdated(venv_data_file): try: with open(venv_data_file, "r", encoding="utf8") as fp: venv_data = json.load(fp) if venv_data.get("version", "") != IDF_ENV_VERSION: + print( + "Warning! IDF virtual environment version changed!" + ) + return True + if ( + venv_data.get("python_version", "") + != _get_idf_venv_python_version() + ): + print( + "Warning! Python version in the IDF virtual environment" + " differs from the current Python!" + ) return True return False except: @@ -1387,8 +1413,13 @@ def _create_venv(venv_dir): venv_data_file = os.path.join(venv_dir, "pio-idf-venv.json") if not os.path.isfile(venv_data_file) or _is_venv_outdated(venv_data_file): _create_venv(venv_dir) + + install_python_deps() with open(venv_data_file, "w", encoding="utf8") as fp: - venv_info = {"version": IDF_ENV_VERSION} + venv_info = { + "version": IDF_ENV_VERSION, + "python_version": _get_idf_venv_python_version() + } json.dump(venv_info, fp, indent=2) @@ -1407,11 +1438,10 @@ def get_python_exe(): # -# ESP-IDF requires Python packages with specific versions in a virtual environment +# Ensure Python environment contains everything required for IDF # ensure_python_venv_available() -install_python_deps() # ESP-IDF package doesn't contain .git folder, instead package version is specified # in a special file "version.h" in the root folder of the package From e972aab961a3f446aa90e96afb09e4fe4bdbac6e Mon Sep 17 00:00:00 2001 From: valeros Date: Fri, 10 Jan 2025 16:30:02 +0200 Subject: [PATCH 5/5] Skip unnecessary link flags --- builder/frameworks/espidf.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/builder/frameworks/espidf.py b/builder/frameworks/espidf.py index 99b89f75a..d7c865195 100644 --- a/builder/frameworks/espidf.py +++ b/builder/frameworks/espidf.py @@ -1646,7 +1646,15 @@ def get_python_exe(): # Extra flags which need to be explicitly specified in LINKFLAGS section because SCons # cannot merge them correctly extra_flags = filter_args( - link_args["LINKFLAGS"], ["-T", "-u", "-Wl,--start-group", "-Wl,--end-group"] + link_args["LINKFLAGS"], + [ + "-T", + "-u", + "-Wl,--start-group", + "-Wl,--end-group", + "-Wl,--whole-archive", + "-Wl,--no-whole-archive", + ], ) link_args["LINKFLAGS"] = sorted(list(set(link_args["LINKFLAGS"]) - set(extra_flags)))