Add a way to mutate JSON within manifests

.github/workflows/build.yaml

@@ -159,7 +159,7 @@ jobs:
           done
 
           # Build it
-          python3 tools/build_project.py \
+          /opt/venv/bin/python3 tools/build_project.py \
             $sdk_args \
             $toolchain_args \
             --manifest "${{ matrix.manifest }}" \

Dockerfile

@@ -16,10 +16,15 @@ RUN \
        default-jre-headless \
        patch \
        python3 \
-       python3-ruamel.yaml \
+       python3-pip \
+       python3-virtualenv \
        unzip \
        xz-utils
 
+RUN \
+    virtualenv /opt/venv \
+    && /opt/venv/bin/pip install jq ruamel.yaml
+
 # Install Simplicity Commander (unfortunately no stable URL available, this
 # is known to be working with Commander_linux_x86_64_1v15p0b1306.tar.bz).
 RUN \

README.md

@@ -70,7 +70,7 @@ tool will automatically determine which SDK and toolchain to use.
 > automatically found so these flags can be omitted.
 
 ```bash
-pip install ruamel.yaml  # Only dependency
+pip install jq ruamel.yaml
 
 python tools/build_project.py \
     # The following SDK and toolchain flags can be omitted on macOS

tools/build_project.py

@@ -19,12 +19,31 @@
 import multiprocessing
 from datetime import datetime, timezone
 
+import jq
 from ruamel.yaml import YAML
 
 
-SLC = ["slc", "--daemon", "--daemon-timeout", "1"]
-
 LOGGER = logging.getLogger(__name__)
+SLC = ["slc", "--daemon", "--daemon-timeout", "1"]
+DEFAULT_JSON_CONFIG = [
+    # Fix a few paths by default
+    {
+        "file": "config/zcl/zcl_config.zap",
+        "jq": '(.package[] | select(.type == "zcl-properties")).path = $zcl_zap',
+        "args": {
+            "zcl_zap": "template:{sdk}/app/zcl/zcl-zap.json",
+        },
+        "skip_if_missing": True,
+    },
+    {
+        "file": "config/zcl/zcl_config.zap",
+        "jq": '(.package[] | select(.type == "gen-templates-json")).path = $gen_templates',
+        "args": {
+            "gen_templates": "template:{sdk}/protocol/zigbee/app/framework/gen-template/gen-templates.json",
+        },
+        "skip_if_missing": True,
+    },
+]
 
 
 yaml = YAML(typ="safe")
@@ -38,6 +57,14 @@ def evaulate_f_string(f_string: str, variables: dict[str, typing.Any]) -> str:
     return eval("f" + repr(f_string), variables)
 
 
+def expand_template(value: typing.Any, env: dict[str, typing.Any]) -> typing.Any:
+    """Expand a template string."""
+    if isinstance(value, str) and value.startswith("template:"):
+        return evaulate_f_string(value.replace("template:", "", 1), env)
+    else:
+        return value
+
+
 def ensure_folder(path: str | pathlib.Path) -> pathlib.Path:
     """Ensure that the path exists and is a folder."""
     path = pathlib.Path(path)
@@ -274,10 +301,10 @@ def main():
 
     # Ensure we can load the correct SDK and toolchain
     sdks = load_sdks(args.sdks)
-    sdk, sdk_version = next(
+    sdk, sdk_name_version = next(
         (path, version) for path, version in sdks.items() if version == manifest["sdk"]
     )
-    sdk_name = sdk_version.split(":", 1)[0]
+    sdk_name, _, sdk_version = sdk_name_version.partition(":")
 
     toolchains = load_toolchains(args.toolchains)
     toolchain = next(
@@ -370,6 +397,35 @@ def main():
     with (args.build_dir / "gbl_metadata.yaml").open("w") as f:
         yaml.dump(manifest["gbl"], f)
 
+    # Template variables
+    value_template_env = {
+        "git_repo_hash": get_git_commit_id(repo=pathlib.Path(__file__).parent.parent),
+        "manifest_name": args.manifest.stem,
+        "now": datetime.now(timezone.utc),
+        "sdk": sdk,
+        "sdk_version": sdk_version,
+    }
+
+    for json_config in DEFAULT_JSON_CONFIG + manifest.get("json_config", []):
+        json_path = build_template_path / json_config["file"]
+
+        if json_config.get("skip_if_missing", False) and not json_path.exists():
+            continue
+
+        jq_args = {
+            k: expand_template(v, value_template_env)
+            for k, v in json_config.get("args", {}).items()
+        }
+
+        LOGGER.debug("Substituting JQ args for %s: %s", json_path, jq_args)
+
+        result = (
+            jq.compile(json_config["jq"], args=jq_args)
+            .input_text(json_path.read_text())
+            .first()
+        )
+        json_path.write_text(json.dumps(result, indent=2))
+
     # Next, generate a chip-specific project from the modified base project
     print(f"Generating project for {manifest['device']}")
 
@@ -399,13 +455,6 @@ def main():
             LOGGER.error("Referenced extension not present in SDK: %s", expected_dir)
             sys.exit(1)
 
-    # Template variables for C defines
-    value_template_env = {
-        "git_repo_hash": get_git_commit_id(repo=pathlib.Path(__file__).parent.parent),
-        "manifest_name": args.manifest.stem,
-        "now": datetime.now(timezone.utc),
-    }
-
     # Actually search for C defines within config
     unused_defines = set(manifest.get("c_defines", {}).keys())
 
@@ -416,7 +465,7 @@ def main():
             new_config_h_lines = []
 
             for index, line in enumerate(config_h_lines):
-                for define, value_template in manifest.get("c_defines", {}).items():
+                for define, value in manifest.get("c_defines", {}).items():
                     if f"#define {define} " not in line:
                         continue
 
@@ -441,15 +490,7 @@ def main():
                         assert re.match(r'#warning ".*? not configured"', prev_line)
                         new_config_h_lines.pop(index - 1)
 
-                    value_template = str(value_template)
-
-                    if value_template.startswith("template:"):
-                        value = value_template.replace("template:", "", 1).format(
-                            **value_template_env
-                        )
-                    else:
-                        value = value_template
-
+                    value = expand_template(value, value_template_env)
                     new_config_h_lines.append(f"#define {define}{alignment}{value}")
                     written_config[define] = value