Warn if api_key in Config (#1049)

Fixes #927 

Look for the presence of `api_key` as a key when loading config files,
raise a warning to the user if the file is readable by group or others.

## Verification

List the steps needed to make sure this thing works

``` yaml
---
run:
    seed:
    eval_threshold: 0.5
    generations: 1
    probe_tags:

plugins:
    model_type: test
    model_name: Lipsum
    detector_spec: auto
    api_key: test
    probe_spec: test.Test

reporting:
    report_prefix: test
```

run `garak --config config.yaml`

NOTE: The warning does not get surfaced to the user directly, which
shouldn't be the case when we warn, but it is written to the log. This
is an artifact of our default verbosity and it may make sense to `print`
the warning here.

garak/_config.py

@@ -7,11 +7,13 @@
 
 # logging should be set up before config is loaded
 
+import platform
 from collections import defaultdict
 from dataclasses import dataclass
 import importlib
 import logging
 import os
+import stat
 import pathlib
 from typing import List
 import yaml
@@ -116,13 +118,25 @@ def _nested_dict():
 # generator, probe, detector, buff = {}, {}, {}, {}
 
 
+def _key_exists(d: dict, key: str) -> bool:
+    # Check for the presence of a key in a nested dict.
+    if not isinstance(d, dict) and not isinstance(d, list):
+        return False
+    if isinstance(d, list):
+        return any([_key_exists(item, key) for item in d])
+    if isinstance(d, dict) and key in d.keys():
+        return True
+    else:
+        return any([_key_exists(val, key) for val in d.values()])
+
+
 def _set_settings(config_obj, settings_obj: dict):
     for k, v in settings_obj.items():
         setattr(config_obj, k, v)
     return config_obj
 
 
-def _combine_into(d: dict, combined: dict) -> None:
+def _combine_into(d: dict, combined: dict) -> dict:
     if d is None:
         return combined
     for k, v in d.items():
@@ -141,6 +155,22 @@ def _load_yaml_config(settings_filenames) -> dict:
         with open(settings_filename, encoding="utf-8") as settings_file:
             settings = yaml.safe_load(settings_file)
             if settings is not None:
+                if _key_exists(settings, "api_key"):
+                    if platform.system() == "Windows":
+                        msg = (f"A possibly secret value (`api_key`) was detected in {settings_filename}. "
+                               f"We recommend removing potentially sensitive values from config files or "
+                               f"ensuring the file is readable only by you.")
+                        logging.warning(msg)
+                        print(f"⚠️  {msg}")
+                    else:
+                        logging.info(f"API key found in {settings_filename}. Checking readability...")
+                        res = os.stat(settings_filename)
+                        if res.st_mode & stat.S_IROTH or res.st_mode & stat.S_IRGRP:
+                            msg = (f"A possibly secret value (`api_key`) was detected in {settings_filename}, "
+                                   f"which is readable by users other than yourself. "
+                                   f"Consider changing permissions on this file to only be readable by you.")
+                            logging.warning(msg)
+                            print(f"⚠️  {msg}")
                 config = _combine_into(settings, config)
     return config
 

tests/test_config.py

@@ -826,3 +826,10 @@ async def main():
         "/", headers={"User-Agent": AGENT_TEST}
     ).respond_with_data("")
     asyncio.run(main())
+
+
+def test_api_key_in_config():
+    importlib.reload(_config)
+
+    _config.plugins.generators["a"]["b"]["c"]["api_key"] = "something"
+    assert _config._key_exists(_config.plugins.generators, "api_key")