From 8003cbe66c216b5a347057ad764054933a757529 Mon Sep 17 00:00:00 2001 From: Alexander Williams Date: Thu, 30 Jan 2025 13:50:26 -0800 Subject: [PATCH] [virtual-vlnv] Add CLI option to add virtual providers Add --add-vlnv to enable adding user-requested VLNVs that are not under the "system" or top core's explicit dependency tree. The intention of this argument is to allow users to specify cores that provide the implementations of virtual VLNVs in these "side" trees. The CLI option allows users to provide these without requiring someone to write a core file with the explicit deps, so targets may be reused across multiple different configurations, and users don't need to have duplicate core files that only differ by virtual providers. Signed-off-by: Alexander Williams --- fusesoc/coremanager.py | 45 +++++++++++++++++++----------- fusesoc/edalizer.py | 6 +++- fusesoc/fusesoc.py | 3 +- fusesoc/main.py | 17 +++++++++++- tests/test_coremanager.py | 58 +++++++++++++++++++++++++++++++++++++-- 5 files changed, 107 insertions(+), 22 deletions(-) diff --git a/fusesoc/coremanager.py b/fusesoc/coremanager.py index 4666d306..db5c4ab2 100644 --- a/fusesoc/coremanager.py +++ b/fusesoc/coremanager.py @@ -110,8 +110,8 @@ def _hash_flags_dict(self, flags): h ^= hash(pair) return h - def solve(self, top_core, flags): - return self._solve(top_core, flags) + def solve(self, top_core, flags, added_cores): + return self._solve(top_core, flags, added_cores) def _get_conflict_map(self): """Return a map of cores to their conflicts @@ -142,7 +142,7 @@ def _get_conflict_map(self): conflict_set.remove(real_pkg) return conflict_map - def _solve(self, top_core, flags={}, only_matching_vlnv=False): + def _solve(self, top_core, flags={}, added_cores=(), only_matching_vlnv=False): def eq_vln(this, that): return ( this.vendor == that.vendor @@ -151,7 +151,12 @@ def eq_vln(this, that): ) # Try to return a cached result - solver_cache_key = (top_core, self._hash_flags_dict(flags), only_matching_vlnv) + requested_cores = frozenset(added_cores + (top_core,)) + solver_cache_key = ( + requested_cores, + self._hash_flags_dict(flags), + only_matching_vlnv, + ) cached_solution = self._solver_cache_lookup(solver_cache_key) if cached_solution: return cached_solution @@ -206,12 +211,13 @@ def eq_vln(this, that): repo.add_package(package) request = Request() - _top_dep = "{} {} {}".format( - self._package_name(top_core), - top_core.relation, - self._package_version(top_core), - ) - request.install(Requirement._from_string(_top_dep)) + for requested_core in requested_cores: + dep = "{} {} {}".format( + self._package_name(requested_core), + requested_core.relation, + self._package_version(requested_core), + ) + request.install(Requirement._from_string(dep)) installed_repository = Repository() pool = Pool([repo]) @@ -249,11 +255,12 @@ def eq_vln(this, that): ] # Print a warning for all virtual selections that has no concrete requirement selection for virtual in virtual_selection.values(): - logger.warning( - "Non-deterministic selection of virtual core {} selected {}".format( - virtual[1], virtual[0] + if virtual[0] not in requested_cores: + logger.warning( + "Non-deterministic selection of virtual core {} selected {}".format( + virtual[1], virtual[0] + ) ) - ) result = [op.package.core for op in transaction.operations] @@ -402,13 +409,16 @@ def get_libraries(self): """Get all registered libraries""" return self._lm.get_libraries() - def get_depends(self, core, flags): + def get_depends(self, core, flags, added_cores): """Get an ordered list of all dependencies of a core All direct and indirect dependencies are resolved into a dependency tree, the tree is flattened, and an ordered list of dependencies is created. + The list is augmented by any added_cores required by the call, which can + deterministically satisfy virtual VLNV dependencies. + The first element in the list is a leaf dependency, the last element is the core at the root of the dependency tree. """ @@ -417,8 +427,11 @@ def get_depends(self, core, flags): core.relation, str(core), str(flags) ) ) + logger.debug( + " User added cores to request: " + ", ".join([str(c) for c in added_cores]) + ) resolved_core = self.db.find(core) - deps = self.db.solve(resolved_core.name, flags) + deps = self.db.solve(resolved_core.name, flags, added_cores) logger.debug(" Resolved core to {}".format(str(resolved_core.name))) logger.debug(" with dependencies " + ", ".join([str(c.name) for c in deps])) return deps diff --git a/fusesoc/edalizer.py b/fusesoc/edalizer.py index 30918c4c..05fed1c8 100644 --- a/fusesoc/edalizer.py +++ b/fusesoc/edalizer.py @@ -49,12 +49,14 @@ def __init__( export_root=None, system_name=None, resolve_env_vars=False, + added_core_reqs=(), ): logger.debug("Building EDAM structure") self.toplevel = toplevel self.flags = flags self.core_manager = core_manager + self.added_core_reqs = added_core_reqs self.work_root = work_root self.export_root = export_root self.system_name = system_name @@ -74,7 +76,9 @@ def cores(self): @property def resolved_cores(self): """Get a list of all "used" cores after the dependency resolution""" - return self.core_manager.get_depends(self.toplevel, self.flags) + return self.core_manager.get_depends( + self.toplevel, self.flags, self.added_core_reqs + ) @property def discovered_cores(self): diff --git a/fusesoc/fusesoc.py b/fusesoc/fusesoc.py index 5f163fc1..74eef941 100644 --- a/fusesoc/fusesoc.py +++ b/fusesoc/fusesoc.py @@ -115,7 +115,7 @@ def get_work_root(self, core, flags): return work_root - def get_backend(self, core, flags, backendargs=[]): + def get_backend(self, core, flags, added_core_reqs=(), backendargs=[]): work_root = self.get_work_root(core, flags) @@ -156,6 +156,7 @@ def get_backend(self, core, flags, backendargs=[]): export_root=export_root, system_name=self.config.system_name, resolve_env_vars=self.config.resolve_env_vars_early, + added_core_reqs=added_core_reqs, ) try: diff --git a/fusesoc/main.py b/fusesoc/main.py index 47c7fb30..a388da2a 100644 --- a/fusesoc/main.py +++ b/fusesoc/main.py @@ -29,6 +29,7 @@ from fusesoc.coremanager import DependencyError from fusesoc.fusesoc import Fusesoc from fusesoc.librarymanager import Library +from fusesoc.vlnv import Vlnv logger = logging.getLogger(__name__) @@ -303,6 +304,11 @@ def run(fs, args): else: flags[flag] = True + added_core_reqs = [] + for vlnv in args.add_vlnv: + added_core_reqs.append(Vlnv(vlnv)) + added_core_reqs = tuple(added_core_reqs) + core = _get_core(fs, args.system) try: @@ -326,7 +332,9 @@ def run(fs, args): # Frontend/backend separation try: - edam_file, backend = fs.get_backend(core, flags, args.backendargs) + edam_file, backend = fs.get_backend( + core, flags, added_core_reqs, args.backendargs + ) except RuntimeError as e: logger.error(str(e)) @@ -592,6 +600,13 @@ def get_parser(): parser_run.add_argument("--run", action="store_true", help="Execute run stage") parser_run.add_argument("--target", help="Override default target") parser_run.add_argument("--tool", help="Override default tool for target") + parser_run.add_argument( + "--add-vlnv", + help="Add a VLNV to the build as a dependency of the 'system'. Intended for explicit virtual providers. Multiple uses allowed.", + action="append", + default=[], + metavar="VLNV", + ) parser_run.add_argument( "--flag", help="Set custom use flags. Can be specified multiple times", diff --git a/tests/test_coremanager.py b/tests/test_coremanager.py index 679371fb..a293a19a 100644 --- a/tests/test_coremanager.py +++ b/tests/test_coremanager.py @@ -88,7 +88,7 @@ def test_deptree(tmp_path): edam = edalizer.run() # Check dependency tree (after running all generators) - deps = cm.get_depends(root_core.name, {}) + deps = cm.get_depends(root_core.name, {}, ()) deps_names = [str(c) for c in deps] all_core_names = set() @@ -274,7 +274,7 @@ def test_virtual(): ) edalizer.run() - deps = cm.get_depends(root_core.name, {}) + deps = cm.get_depends(root_core.name, {}, ()) deps_names = [str(c) for c in deps] assert deps_names == expected_deps @@ -355,7 +355,7 @@ def test_virtual_non_deterministic_virtual(caplog): edalizer.run() assert "Non-deterministic selection of virtual core" in caplog.text - deps = cm.get_depends(root_core.name, {}) + deps = cm.get_depends(root_core.name, {}, ()) deps_names = [str(c) for c in deps] for dependency in deps_names: @@ -365,3 +365,55 @@ def test_virtual_non_deterministic_virtual(caplog): "::user:0", "::top_non_deterministic:0", ] + + +def test_virtual_explicit_providers(caplog): + """ + Test virtual core selection when there are explicit selected implementations on the side. + This shall NOT result in a warning that the virtual core selection is non-deteministic. + """ + import logging + import os + import tempfile + + from fusesoc.config import Config + from fusesoc.coremanager import CoreManager + from fusesoc.edalizer import Edalizer + from fusesoc.librarymanager import Library + from fusesoc.vlnv import Vlnv + + flags = {"tool": "icarus"} + + build_root = tempfile.mkdtemp(prefix="export_") + work_root = os.path.join(build_root, "work") + + core_dir = os.path.join(os.path.dirname(__file__), "capi2_cores", "virtual") + + cm = CoreManager(Config()) + cm.add_library(Library("virtual", core_dir), []) + + root_core = cm.get_core(Vlnv("::top_non_deterministic")) + + added_core_options = ((Vlnv("::impl1:0"),), (Vlnv("::impl2:0"),)) + + for added_cores in added_core_options: + edalizer = Edalizer( + toplevel=root_core.name, + flags=flags, + core_manager=cm, + work_root=work_root, + added_core_reqs=added_cores, + ) + + with caplog.at_level(logging.WARNING): + edalizer.run() + assert "Non-deterministic selection of virtual core" not in caplog.text + + deps = cm.get_depends(root_core.name, {}, added_cores) + deps_names = [str(c) for c in deps] + + for dependency in deps_names: + assert dependency in [ + "::user:0", + "::top_non_deterministic:0", + ] + [str(c) for c in added_cores]