From 9dc5254c54a72e55d7b2018eb2b804e445dd0320 Mon Sep 17 00:00:00 2001 From: Meagan Lang Date: Mon, 24 Jun 2024 16:10:11 -0400 Subject: [PATCH] Added debug message for tool configuration Added flag to get config diff Added debug message for getting CFLAGS from environment Check configuration options for default_executable during compilation tool registration Add gnu to set of compatible toolsets for MSVC linker & archiver Fix bug in use of build_env for cmake --- yggdrasil/command_line.py | 9 +- yggdrasil/config.py | 141 ++++++++++++++++------- yggdrasil/drivers/CMakeModelDriver.py | 4 +- yggdrasil/drivers/CModelDriver.py | 5 +- yggdrasil/drivers/CompiledModelDriver.py | 13 ++- yggdrasil/tools.py | 2 +- 6 files changed, 124 insertions(+), 50 deletions(-) diff --git a/yggdrasil/command_line.py b/yggdrasil/command_line.py index 5c05329ca..81d3638f6 100755 --- a/yggdrasil/command_line.py +++ b/yggdrasil/command_line.py @@ -1412,7 +1412,12 @@ class update_config(SubCommand): {'action': 'store_true', 'default': None, 'help': ('Don\'t set the KMP_DUPLICATE_LIB_OK environment variable ' 'when running models (see help for \'--allow-multiple-omp\' ' - 'for more information).')})] + 'for more information).')}), + (('--diff', ), + {'action': 'store_true', + 'help': ('Display the diff between the updated configuration ' + 'and the existing configuration without actually ' + 'updating the configuration file.')})] + [(('--%s-compiler' % k, ), {'help': ('Name or path to compiler that should be used to compile ' 'models written in %s.' % k)}) @@ -1523,7 +1528,7 @@ def func(cls, args): lang_kwargs[k][x] = getattr(args, '%s_%s' % (k, x)) config.update_language_config( args.languages, overwrite=args.overwrite, - verbose=(not args.quiet), + verbose=(not args.quiet), diff=args.diff, disable_languages=args.disable_languages, enable_languages=args.enable_languages, allow_multiple_omp=args.allow_multiple_omp, diff --git a/yggdrasil/config.py b/yggdrasil/config.py index 73296a66e..947b8de26 100644 --- a/yggdrasil/config.py +++ b/yggdrasil/config.py @@ -14,6 +14,7 @@ import warnings import configparser import copy +import pprint import argparse from contextlib import contextmanager from collections import OrderedDict @@ -257,6 +258,38 @@ def get(self, section, option, default=None, **kwargs): return self.backwards_str2val(out) return default + def diff(self, solf): + r"""Get the difference between this and another YggConfigParser. + + Args: + solf (YggConfigParser): Parser to compare against this one. + + Returns: + dict: Sets of added, missing and changed parameters between + this config parser and solf. + + """ + out = {k: [] for k in ['added', 'missing', 'changed']} + for s in self.sections(): + if not solf.has_section(s): + out['missing'] += [(s, opt) for opt in self.options(s)] + continue + for opt in self.options(s): + if not solf.has_option(s, opt): + out['missing'].append((s, opt)) + elif self.get(s, opt) != solf.get(s, opt): + out['changed'].append((s, opt, + self.get(s, opt), + solf.get(s, opt))) + for s in solf.sections(): + if not self.has_section(s): + out['added'] += [(s, opt) for opt in solf.options(s)] + continue + for opt in solf.options(s): + if not self.has_option(s, opt): + out['added'].append((s, opt)) + return out + def get_language_order(drivers): r"""Get the correct language order, including any base languages. @@ -306,7 +339,7 @@ def get_language_order(drivers): def update_language_config(languages=None, skip_warnings=False, disable_languages=None, enable_languages=None, allow_multiple_omp=None, lang_kwargs=None, - overwrite=False, verbose=False): + overwrite=False, verbose=False, diff=False): r"""Update configuration options for a language driver. Args: @@ -327,14 +360,17 @@ def update_language_config(languages=None, skip_warnings=False, Defaults to False. verbose (bool, optional): If True, information about the config file will be displayed. Defaults to False. + diff (bool, optional): If True, display the difference between + the updated configuration and the existing one, without + actually updating the file. Defaults to False. lang_kwargs (dict, optional): Dictionary containing language specific keyword arguments. Defaults to {}. """ from yggdrasil.components import import_component - if verbose: - logger.info("Updating user configuration file for yggdrasil at:\n\t%s" - % usr_config_file) + if verbose and not diff: + logger.info(f"Updating user configuration file for yggdrasil " + f"at:\n\t{usr_config_file}") miss = [] if (languages is None) or overwrite: all_languages = tools.get_supported_lang() @@ -350,44 +386,65 @@ def update_language_config(languages=None, skip_warnings=False, enable_languages = [] if lang_kwargs is None: lang_kwargs = {} - if overwrite: - shutil.copy(def_config_file, usr_config_file) - ygg_cfg_usr.reload() - if allow_multiple_omp is not None: - if not ygg_cfg_usr.has_section('general'): - ygg_cfg_usr.add_section('general') - ygg_cfg_usr.set('general', 'allow_multiple_omp', allow_multiple_omp) - drivers = OrderedDict([(lang, import_component('model', lang)) - for lang in languages]) - drv = list(get_language_order(drivers).values()) - for idrv in drv: - if (((idrv.language in disable_languages) - and (idrv.language in enable_languages))): - logger.info(("%s language both enabled and disabled. " - "No action will be taken.") % idrv.language) - elif idrv.language in disable_languages: - if not ygg_cfg_usr.has_section(idrv.language): - ygg_cfg_usr.add_section(idrv.language) - ygg_cfg_usr.set(idrv.language, 'disable', 'True') - elif idrv.language in enable_languages: - if not ygg_cfg_usr.has_section(idrv.language): - ygg_cfg_usr.add_section(idrv.language) - ygg_cfg_usr.set(idrv.language, 'disable', 'False') - if ygg_cfg_usr.get(idrv.language, 'disable', 'False').lower() == 'true': - continue # pragma: no cover - miss += idrv.configure(ygg_cfg_usr, - **lang_kwargs.get(idrv.language, {})) - ygg_cfg_usr.update_file() - ygg_cfg.reload() - if not skip_warnings: - for sect, opt, desc in miss: # pragma: windows - warnings.warn( - f"Could not set option {opt} in section {sect}. Please " - f"set this in {ygg_cfg_usr.file_to_update} to: {desc}", - RuntimeWarning) - if verbose: - with open(usr_config_file, 'r') as fd: - print(fd.read()) + existing = None + cached = '_cached'.join(os.path.splitext(usr_config_file)) + if diff: + shutil.copy2(usr_config_file, cached) + existing = YggConfigParser.from_files( + [def_config_file, cached, loc_config_file]) + try: + if overwrite: + shutil.copy(def_config_file, usr_config_file) + ygg_cfg_usr.reload() + if allow_multiple_omp is not None: + if not ygg_cfg_usr.has_section('general'): + ygg_cfg_usr.add_section('general') + ygg_cfg_usr.set('general', 'allow_multiple_omp', + allow_multiple_omp) + drivers = OrderedDict([(lang, import_component('model', lang)) + for lang in languages]) + drv = list(get_language_order(drivers).values()) + for idrv in drv: + if (((idrv.language in disable_languages) + and (idrv.language in enable_languages))): + logger.info(("%s language both enabled and disabled. " + "No action will be taken.") % idrv.language) + elif idrv.language in disable_languages: + if not ygg_cfg_usr.has_section(idrv.language): + ygg_cfg_usr.add_section(idrv.language) + ygg_cfg_usr.set(idrv.language, 'disable', 'True') + elif idrv.language in enable_languages: + if not ygg_cfg_usr.has_section(idrv.language): + ygg_cfg_usr.add_section(idrv.language) + ygg_cfg_usr.set(idrv.language, 'disable', 'False') + if ygg_cfg_usr.get(idrv.language, 'disable', 'False').lower() == 'true': + continue # pragma: no cover + miss += idrv.configure(ygg_cfg_usr, + **lang_kwargs.get(idrv.language, {})) + ygg_cfg_usr.update_file() + ygg_cfg.reload() + if not skip_warnings: + for sect, opt, desc in miss: # pragma: windows + warnings.warn( + f"Could not set option {opt} in section {sect}. Please " + f"set this in {ygg_cfg_usr.file_to_update} to: {desc}", + RuntimeWarning) + if verbose: + with open(usr_config_file, 'r') as fd: + print(fd.read()) + if diff: + diffdict = existing.diff(ygg_cfg) + msg = (f"Cached: {cached}\n" + f"Updated: {usr_config_file}\n") + for k in ['missing', 'added', 'changed']: + msg += f"{k.title():7}: {pprint.pformat(diffdict[k])}\n" + print(msg) + finally: + if diff: + shutil.move(cached, usr_config_file) + ygg_cfg_usr.reload() + ygg_cfg.reload() + assert not any(existing.diff(ygg_cfg).values()) def get_ygg_loglevel(cfg=None, default='DEBUG'): diff --git a/yggdrasil/drivers/CMakeModelDriver.py b/yggdrasil/drivers/CMakeModelDriver.py index bd7a95f8f..59a13f739 100644 --- a/yggdrasil/drivers/CMakeModelDriver.py +++ b/yggdrasil/drivers/CMakeModelDriver.py @@ -648,10 +648,10 @@ def create_dep(cls, without_wrapper=False, **kwargs): if platform._is_win and out['target_dep'].tool('compiler').is_gnu: gcc = out['target_dep'].tool('compiler').get_executable( full_path=True) - env = out['build_env'] + env = out.get('builder_env', for_build=True) path = cls.prune_sh_gcc(env, gcc) env['PATH'] = path - out.set('build_env', env) + out.set('builder_env', env) if not shutil.which('sh', path=path): # pragma: appveyor # This will not be run on Github actions where # the shell is always set diff --git a/yggdrasil/drivers/CModelDriver.py b/yggdrasil/drivers/CModelDriver.py index 09ddab31c..22a543ede 100755 --- a/yggdrasil/drivers/CModelDriver.py +++ b/yggdrasil/drivers/CModelDriver.py @@ -169,7 +169,7 @@ class ClangCompiler(CCompilerBase): compatible_toolsets = ['gnu'] next_stage_flag_flag = '-Xlinker' version_regex = [ - r'(?P(?:Apple )?clang version \d+\.\d+\.\d+)'] + r'(?P(?:Apple )?(?:(?:clang)|(?:LLVM)) version \d+\.\d+\.\d+)'] flag_options = OrderedDict(list(CCompilerBase.flag_options.items()) + [('sysroot', '--sysroot'), ('isysroot', {'key': '-isysroot', @@ -371,6 +371,7 @@ class MSVCLinker(LinkerBase): platforms = MSVCCompiler.platforms languages = MSVCCompiler.languages toolset = MSVCCompiler.toolset + compatible_toolsets = ['llvm', 'gnu'] aliases = [] output_key = '/OUT:%s' output_first = True @@ -440,7 +441,7 @@ class MSVCArchiver(ArchiverBase): libtype_flags = {} output_key = '/OUT:%s' toolset = 'msvc' - compatible_toolsets = ['llvm'] + compatible_toolsets = ['llvm', 'gnu'] search_path_envvar = ['LIB'] diff --git a/yggdrasil/drivers/CompiledModelDriver.py b/yggdrasil/drivers/CompiledModelDriver.py index 7f5b07d36..63f7ee809 100644 --- a/yggdrasil/drivers/CompiledModelDriver.py +++ b/yggdrasil/drivers/CompiledModelDriver.py @@ -4191,6 +4191,7 @@ def before_registration(cls): to registration including things like platform dependent properties and checking environment variables for default settings. """ + from yggdrasil.config import ygg_cfg if cls.toolname is None: # pragma: debug raise CompilationToolError("Registering unnamed compilation tool.") cls.is_gnu = (cls.toolset == 'gnu') @@ -4201,6 +4202,9 @@ def before_registration(cls): # Copy so that list modification is not propagated to subclasses setattr(cls, k, copy.deepcopy(getattr(cls, k, []))) # Set attributes based on environment variables or sysconfig + if cls.default_executable is None and cls.languages: + cls.default_executable = ygg_cfg.get( + cls.languages[0], f'{cls.toolname}_executable', None) if cls.default_executable is None: cls.default_executable = cls.env_matches_tool() if cls.default_executable is None: @@ -4670,6 +4674,10 @@ def env_matches_tool(cls, use_sysconfig=False, env=None, env.update(sysconfig.get_config_vars()) else: env.update(os.environ) + # TODO: temp + if (('-arch arm64' in env.get('CFLAGS', '') + and '-arch x86_64' in env.get('CFLAGS', ''))): + verbose = True envi_full = '' if isinstance(cls.default_executable_env, str): envi_full = env.get(cls.default_executable_env, '').split( @@ -6918,6 +6926,8 @@ def configure(cls, cfg, **kwargs): cfg.set(cls.language, k, vtool.toolname) setattr(cls, f'default_{k}', vtool.toolname) if os.path.isfile(v): + setattr(_tool_registry.tool(k, vtool.toolname), + 'default_executable', v) cfg.set(cls.language, f'{vtool.toolname}_executable', v) # Clear dependencies that should be set based on tool if kwargs: @@ -7004,6 +7014,7 @@ def configure_libraries(cls, cfg): kws[k] = cls.get_tool(k) except InvalidCompilationTool: pass + print(f"CONFIGURATION SPECIALIZATION:\n{pprint.pformat(kws)}") libs = cls.libraries.specialized(**kws) for v in libs.libraries.values(): v.from_cache(cfg) @@ -7084,7 +7095,7 @@ def compile_dependencies(cls, dep=None, add_all_products=False, dep.products(kwargs['products'], include_dependencies=True, skip_build_products=True) - dep.logInfo() + dep.logInfo(f'compiled: {dep}') @classmethod def cleanup_dependencies(cls, dep=None, products=None, diff --git a/yggdrasil/tools.py b/yggdrasil/tools.py index a0f314088..f257e4eb3 100644 --- a/yggdrasil/tools.py +++ b/yggdrasil/tools.py @@ -1036,7 +1036,7 @@ def get_supported_comm(dont_include_value=False): for k in excl_list: if k in out: out.remove(k) - return list(set(out)) + return out # list(set(out)) def is_lang_installed(lang):