diff --git a/requirements.txt b/requirements.txt index 85a7681..eba7fd0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,7 @@ MarkupSafe==1.1.1 matplotlib==3.3.4 numpy==1.20.1 Pillow==8.1.2 +psycopg2==2.8.6 pyerfa==1.7.2 pyparsing==2.4.7 python-dateutil==2.8.1 diff --git a/trail/trail/config.py b/trail/trail/config.py new file mode 100644 index 0000000..c3f2f4e --- /dev/null +++ b/trail/trail/config.py @@ -0,0 +1,186 @@ +import os +import stat + +import yaml +import boto3 + + +__all__ = ["CONF_FILE_ENVVAR", "CONF_FILE_PATH", "Config"] + + +CONF_FILE_ENVVAR = "TARILBLAZER_CONFIG" +"""Default name of the environmental variable that contains the path to the +configuration file. When the env var does not exist, configuration is assumed +to exist at its default location (see ``CONF_FILE_PATH``).""" + +CONF_FILE_PATH = "~/.trail/secrets.yaml" +"""Default path at which it is expected the config file can be found. Will be +ignored if ``CONF_FILE_ENVVAR`` env var exists.""" + + +class Config(): + """Represents a general YAML configuration file, with keys being mapped to + attributes. Optionally resolving existing secrets via AWS Secrets Manager. + + Parameters + ---------- + confDict : `dict` + Dictionary whose keys will be mapped to attributes of the class. + useAwsSecrets : `bool`, optional + Resolve secrets using AWS Secrets manager. False by default. + awsRegion : `str`, optional + Region of the secret manager to use. Default: `us-west-2`. + + Notes + ----- + Secrets Manager can and will support any kind of string as a secret. For + RDS it will tests showed that secrets are stored as a JSON key-value string + pairs (i.e. output looks like a ``str(dict)``). This presents 3 different + scenarios when keys get resolved and set as Config attributes: + 1) resolve a secret key into multiple keys and insert them, replacing the + secret key with the recieved key-value pairs; + 2) resolve a secret and insert under original key, when returned secrets + are simple strings so the name of the secret is replaced with the secret + itself; + 3) and insert a key-value pair named in the YAML config file. + """ + + configKey = "*" + """Key which is read to create a config, the value `*` selects all keys.""" + + secretsKeys = [] + """Specifies which keys are to be resolved as secrets.""" + + def __init__(self, confDict, useAwsSecrets=False, awsRegion="us-west-2"): + self._keys = [] + self._subConfs = [] + self._recurseDownDicts(confDict, useAwsSecrets, awsRegion=awsRegion) + + def _recurseDownDicts(self, confDict, useAwsSecrets, awsRegion): + """Recursively walks the dictionary keys and values and maps keys to + instance attributes, resolving any existing secrets along the way. + + Parameters + ---------- + confDict : `dict` + Dictionary whose keys will be mapped to attributes of the class. + useAwsSecrets : `bool`, optional + Resolve secrets using AWS Secrets manager. False by default. + awsRegion : `str`, optional + Region of the secret manager to use. Default: `us-west-2`. + """ + if self.configKey != "*": + if self.configKey not in confDict: + raise ValueError(f"Required config key {self.configKey} does " + "not exist in the config dictionary.") + confDict = confDict[self.configKey] + + # if a region is set in the config use it, otherwise use the default + region = confDict.get("aws-region", awsRegion) + + for key, val in confDict.items(): + if isinstance(val, dict): + self._subConfs.append(key) + setattr(self, key, Config(val)) + else: + # of course this is now ugly... + if useAwsSecrets and key in self.secretsKeys: + secrets = self._parseAwsSecrets(val, region) + if isinstance(secrets, dict): + # scenario 1, replacing key with many + for secretkey, secretval in secrets.items(): + if secretkey not in self._keys: + self._keys.append(secretkey) + setattr(self, secretkey, secretval) + # skip inserting the replaced key + continue + else: + # scenario 2, resolve simple secret as key + val = secrets + # scenario 2 or 3, insert key-value pair, resolving secrets + self._keys.append(key) + setattr(self, key, val) + + @staticmethod + def _parseAwsSecrets(name, region): + smClient = boto3.client("secretsmanager", region_name=region) + secretString = smClient.get_secret_value(SecretId=name)["SecretString"] + # JSON is like YAML, right? + return yaml.safe_load(secretString) + + + def __repr__(self): + reprStr = f"{self.__class__.__name__}(" + + for key in self._subConfs: + reprStr += f"{key}={getattr(self, key)}, " + + for key in self._keys: + reprStr += key + ", " + reprStr = reprStr[:-2] + + return reprStr+")" + + + def __eq__(self, other): + equal = True + + for key, subConf in zip(self._keys, self._subConfs): + try: + equal = equal and getattr(self, key) == getattr(other, key) + equal = equal and getattr(self, subConf) == getattr(other, subConf) + except AttributeError: + # other does not have a key, but self has - not equal + # or other does not have a subConf, but self has - not equal + return False + + return equal + + @classmethod + def fromYaml(cls, filePath=None, useAwsSecrets=False, awsRegion="us-west-2"): + """Create a new Config instance from a YAML file. By default will + look at location pointed to by the environmental variable named by + `CONF_FILE_ENVVAR`. If the env var is not set it will default to + location set by `CONF_FILE_PATH`. + + Parameters + ---------- + filePath : `str` or `None`, Optional + A file path to the YAML configuration. When not specified, first + the ``CONF_FILE_ENVVAR`` is used. If it doesn't exist the + ``CONF_FILE_PATH`` is used. + useAwsSecrets : `bool`, optional + Resolve secrets using AWS Secrets manager. False by default. + awsRegion : `str`, optional + Region of the secret manager to use. Default: `us-west-2`. + """ + # resolve config file path + if filePath is None: + if CONF_FILE_ENVVAR in os.environ: + filePath = os.path.expanduser(os.environ[CONF_FILE_ENVVAR]) + else: + filePath = os.path.expanduser(CONF_FILE_PATH) + + # make sure file exists and its permissions are at 600 or more + if not os.path.isfile(filePath): + raise FileNotFoundError(f"No configuration file found: {filePath}") + + mode = os.stat(filePath).st_mode + if mode & (stat.S_IRWXG | stat.S_IRWXO) != 0: + raise PermissionError(f"Configuration file {filePath} has " + f"incorrect permissions: {mode:o}") + + with open(filePath, 'r') as stream: + confDict = yaml.safe_load(stream) + + return cls(confDict, useAwsSecrets, awsRegion) + + +class DbAuth(Config): + configKey = 'db' + secretsKeys = ["secret_name", ] + + +class SiteConfig(Config): + configKey = 'settings' + secretsKeys = ["secret_key", ] diff --git a/trail/trail/settings.py b/trail/trail/settings.py index ed399ce..ccbb9b5 100644 --- a/trail/trail/settings.py +++ b/trail/trail/settings.py @@ -13,6 +13,11 @@ from pathlib import Path import os +from .config import DbAuth, SiteConfig + +siteConfig = SiteConfig.fromYaml() +dbConfig = DbAuth.fromYaml() + # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -24,14 +29,7 @@ # See https://docs.djangoproject.com/en/3.1/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -home = str(Path.home()) -try: - secret_key_file = os.path.join(home, '.trail/secret_key.txt') - with open(secret_key_file) as f: - SECRET_KEY = f.read().strip() -except FileNotFoundError: - print('Unable to find secret key file {secret_key_file}.') - +SECRET_KEY = siteConfig.secret_key # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True @@ -90,12 +88,17 @@ DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': REPO_DIR / 'db.sqlite3', - } + 'ENGINE': dbConfig.engine, + 'NAME' : dbConfig.name, + 'USER' : dbConfig.user, + 'PASSWORD' : dbConfig.password, + 'HOST' : dbConfig.host, + 'PORT' : dbConfig.port + } } + # Password validation # https://docs.djangoproject.com/en/3.1/ref/settings/#auth-password-validators diff --git a/trail/trail/tests/__init__.py b/trail/trail/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/trail/trail/tests/config/awsSecretsConf.yaml b/trail/trail/tests/config/awsSecretsConf.yaml new file mode 100644 index 0000000..8fe51a2 --- /dev/null +++ b/trail/trail/tests/config/awsSecretsConf.yaml @@ -0,0 +1,2 @@ +db: + secret_name: "db-secret" diff --git a/trail/trail/tests/config/badPermissionConf.yaml b/trail/trail/tests/config/badPermissionConf.yaml new file mode 100644 index 0000000..e69de29 diff --git a/trail/trail/tests/config/conf.yaml b/trail/trail/tests/config/conf.yaml new file mode 100644 index 0000000..5c76f34 --- /dev/null +++ b/trail/trail/tests/config/conf.yaml @@ -0,0 +1,11 @@ +settings: + secret_key: nonsense + static_root: ~/trail/static + media_root: ~/trail/media +db: + engine: django.db.backend.postgresql_psycopg2 + name: dbname + user: dbuser + password: dbpassword + host: dbhost.alala.com + port: 5432 diff --git a/trail/trail/tests/test_config.py b/trail/trail/tests/test_config.py new file mode 100644 index 0000000..2783da0 --- /dev/null +++ b/trail/trail/tests/test_config.py @@ -0,0 +1,119 @@ +import os + +from django.test import TestCase +from moto import mock_secretsmanager +import boto3 +import yaml + + +import trail.config as ConfigModule +from trail.config import Config, DbAuth, SiteConfig + + +TESTDIR = os.path.abspath(os.path.dirname(__file__)) + + +class ConfigTestCase(TestCase): + testConfigDir = os.path.join(TESTDIR, "config") + + def setUp(self): + self.badConf = os.path.join(self.testConfigDir, "badPermissionConf.yaml") + self.goodConf = os.path.join(self.testConfigDir, "conf.yaml") + self.noExists = os.path.join(self.testConfigDir, "noexist.yaml") + + def tearDown(self): + pass + + def testInstantiation(self): + # Test 600 permissions + with self.assertRaises(PermissionError): + Config.fromYaml(self.badConf) + + # Test missing file + with self.assertRaises(FileNotFoundError): + Config.fromYaml(self.noExists) + + # Test that fromYaml and direct instantiation produce same result + # Test it's possible to instantiate without errors, test env var and + # global var default instantiations. + try: + conf1 = Config.fromYaml(self.goodConf) + except Exception as e: + self.fail(f"ConfigTestCase.testConfig conf1 failed with:\n{e}") + + with open(self.goodConf, 'r') as stream: + confDict = yaml.safe_load(stream) + try: + conf2 = Config(confDict) + except Exception as e: + self.fail(f"ConfigTestCase.testConfig conf2 failed with:\n{e}") + + self.assertEqual(conf1, conf2) + + ConfigModule.CONF_FILE_PATH = self.goodConf + try: + conf3 = Config.fromYaml() + except Exception as e: + self.fail(f"ConfigTestCase.testConfig conf3 failed with:\n{e}") + + self.assertEqual(conf2, conf3) + + # Switch to a different conf file to verify overriding with env var + # works as intended + os.environ[ConfigModule.CONF_FILE_ENVVAR] = self.badConf + with self.assertRaises(PermissionError): + conf4 = Config.fromYaml() + + def testConfigKey(self): + Config.configKey = "noexists" + with self.assertRaises(ValueError): + Config.fromYaml(self.goodConf) + + # this is a bit silly I think because it doesn't test correctness? + Config.configKey = "db" + conf1 = Config.fromYaml(self.goodConf) + conf2 = DbAuth.fromYaml(self.goodConf) + self.assertEqual(conf1, conf2) + + +class AwsSecretsTestCase(TestCase): + testConfigDir = os.path.join(TESTDIR, "config") + + def setUp(self): + self.goodConf = os.path.join(self.testConfigDir, "conf.yaml") + self.awsSecretsConf = os.path.join(self.testConfigDir, "awsSecretsConf.yaml") + + @mock_secretsmanager + def testSimpleAwsSecrets(self): + smClient = boto3.client("secretsmanager", region_name="us-west-2") + smClient.create_secret(Name="nonsense", SecretString="test-secret-key") + + conf = SiteConfig.fromYaml(self.goodConf, useAwsSecrets=True) + + self.assertEqual(conf.secret_key, "test-secret-key") + + @mock_secretsmanager + def testMultiKeyedSecret(self): + multiKeyedSecret = { + "engine": "postgresql", + "name": "dbname", + "user": "dbuser", + "password": "dbpassword", + "host": "dbhost.alala.com", + "port": 5432, + } + smClient = boto3.client("secretsmanager", region_name="us-west-2") + smClient.create_secret(Name="db-secret", SecretString=str(multiKeyedSecret)) + + conf = DbAuth.fromYaml(self.awsSecretsConf, useAwsSecrets=True) + + # verify the secret_key was expanded + for key, val in multiKeyedSecret.items(): + self.assertEqual(getattr(conf, key), val) + + # verify that the replaced key was not inserted + with self.assertRaises(AttributeError): + conf.secret_name + + +