From 29ae24086c2fde478403b564052ba387eb566b22 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 14 Aug 2024 14:42:05 -0600 Subject: [PATCH] idaholab/Malcolm#530, work in progress on netbox plugin installation --- .dockerignore | 2 + Dockerfiles/netbox.Dockerfile | 1 + docker-compose-dev.yml | 8 +- docker-compose.yml | 8 +- docs/contributing-local-modifications.md | 6 + docs/custom-rules.md | 7 +- malcolm-iso/build.sh | 1 + netbox/custom-plugins/requirements/.gitignore | 3 + netbox/scripts/netbox_init.py | 282 +++++++++++++++--- netbox/supervisord.conf | 1 + scripts/malcolm_appliance_packager.sh | 1 + scripts/malcolm_utils.py | 6 +- 12 files changed, 281 insertions(+), 45 deletions(-) create mode 100644 netbox/custom-plugins/requirements/.gitignore diff --git a/.dockerignore b/.dockerignore index 4172fe569..b8dc8dc67 100644 --- a/.dockerignore +++ b/.dockerignore @@ -42,3 +42,5 @@ suricata-logs netbox/netbox/media netbox/netbox/postgres netbox/netbox/redis +netbox/custom-plugins +zeek/custom \ No newline at end of file diff --git a/Dockerfiles/netbox.Dockerfile b/Dockerfiles/netbox.Dockerfile index 758cef153..01e93ba27 100644 --- a/Dockerfiles/netbox.Dockerfile +++ b/Dockerfiles/netbox.Dockerfile @@ -76,6 +76,7 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') procps \ psmisc \ python3-dev \ + ripgrep \ rsync \ supervisor \ tini && \ diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 15533c5f1..a8989ea3e 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -1105,7 +1105,7 @@ services: bind: create_host_path: false source: ./netbox/config - target: /etc/netbox/config + target: /etc/netbox/config/configmap read_only: true - type: bind bind: @@ -1118,6 +1118,12 @@ services: source: ./netbox/preload target: /opt/netbox-preload/configmap read_only: true + - type: bind + bind: + create_host_path: false + source: ./netbox/custom-plugins + target: /opt/netbox-custom-plugins + read_only: true healthcheck: test: ["CMD", "curl", "--silent", "http://localhost:8080/netbox/api/"] interval: 60s diff --git a/docker-compose.yml b/docker-compose.yml index 67b3140aa..705f3587a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -979,7 +979,7 @@ services: bind: create_host_path: false source: ./netbox/config - target: /etc/netbox/config + target: /etc/netbox/config/configmap read_only: true - type: bind bind: @@ -992,6 +992,12 @@ services: source: ./netbox/preload target: /opt/netbox-preload/configmap read_only: true + - type: bind + bind: + create_host_path: false + source: ./netbox/custom-plugins + target: /opt/netbox-custom-plugins + read_only: true healthcheck: test: ["CMD", "curl", "--silent", "http://localhost:8080/netbox/api/"] interval: 60s diff --git a/docs/contributing-local-modifications.md b/docs/contributing-local-modifications.md index c58be6d16..2cd99546a 100644 --- a/docs/contributing-local-modifications.md +++ b/docs/contributing-local-modifications.md @@ -463,6 +463,12 @@ services: source: ./netbox/preload target: /opt/netbox-preload/configmap read_only: true + - type: bind + bind: + create_host_path: false + source: ./netbox/custom-plugins + target: /opt/netbox-custom-plugins + read_only: true netbox-postgres: volumes: - type: bind diff --git a/docs/custom-rules.md b/docs/custom-rules.md index 2a69b77f0..64f62ed4c 100644 --- a/docs/custom-rules.md +++ b/docs/custom-rules.md @@ -1,8 +1,9 @@ -# Custom Rules and Scripts +# Custom Rules, Scripts and Plugins * [Suricata](#Suricata) * [Zeek](#Zeek) * [YARA](#YARA) +* [NetBox](#NetBox) * [Other Customizations](#Other) Much of Malcolm's behavior can be adjusted through [environment variable files](malcolm-config.md#MalcolmConfigEnvVars). However, some components allow further customization through the use of custom scripts, configuration files, and rules. @@ -73,6 +74,10 @@ docker compose exec file-monitor supervisorctl restart yara If the `EXTRACTED_FILE_YARA_CUSTOM_ONLY` [environment variable](malcolm-config.md#MalcolmConfigEnvVars) is set to `true`, Malcolm will bypass the default Yara rulesets ([Neo23x0/signature-base](https://github.com/Neo23x0/signature-base), [reversinglabs/reversinglabs-yara-rules](https://github.com/reversinglabs/reversinglabs-yara-rules), and [bartblaze/Yara-rules](https://github.com/bartblaze/Yara-rules)) and use only user-defined rules in `./yara/rules`. +## NetBox + +TODO documentation + ## Other Customizations There are other areas of Malcolm that can be modified and customized to fit users' needs. Please see these other sections of the documentation for more information. diff --git a/malcolm-iso/build.sh b/malcolm-iso/build.sh index dc325fcb2..00954d792 100755 --- a/malcolm-iso/build.sh +++ b/malcolm-iso/build.sh @@ -100,6 +100,7 @@ if [ -d "$WORKDIR" ]; then mkdir -p "$MALCOLM_DEST_DIR/htadmin/" mkdir -p "$MALCOLM_DEST_DIR/logstash/certs/" mkdir -p "$MALCOLM_DEST_DIR/logstash/maps/" + mkdir -p "$MALCOLM_DEST_DIR/netbox/custom-plugins/requirements/" mkdir -p "$MALCOLM_DEST_DIR/netbox/media/" mkdir -p "$MALCOLM_DEST_DIR/netbox/postgres/" mkdir -p "$MALCOLM_DEST_DIR/netbox/redis/" diff --git a/netbox/custom-plugins/requirements/.gitignore b/netbox/custom-plugins/requirements/.gitignore new file mode 100644 index 000000000..a5baada18 --- /dev/null +++ b/netbox/custom-plugins/requirements/.gitignore @@ -0,0 +1,3 @@ +* +!.gitignore + diff --git a/netbox/scripts/netbox_init.py b/netbox/scripts/netbox_init.py index c9b64af17..6b836df40 100755 --- a/netbox/scripts/netbox_init.py +++ b/netbox/scripts/netbox_init.py @@ -4,6 +4,7 @@ # Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. import argparse +import ast import glob import gzip import ipaddress @@ -26,6 +27,7 @@ from collections.abc import Iterable from distutils.dir_util import copy_tree from datetime import datetime +from packaging.version import Version from slugify import slugify ################################################################################################### @@ -36,45 +38,6 @@ ################################################################################################### -def get_iterable(x): - if isinstance(x, Iterable) and not isinstance(x, str): - return x - else: - return (x,) - - -def is_ip_address(x): - try: - ipaddress.ip_address(x) - return True - except Exception: - return False - - -def is_ip_v4_address(x): - try: - ipaddress.IPv4Address(x) - return True - except Exception: - return False - - -def is_ip_v6_address(x): - try: - ipaddress.IPv6Address(x) - return True - except Exception: - return False - - -def is_ip_network(x): - try: - ipaddress.ip_network(x) - return True - except Exception: - return False - - def min_hash_value_by_value(x): return next( iter(list({k: v for k, v in sorted(x.items(), key=lambda item: item[1])}.values())), @@ -105,6 +68,104 @@ def max_hash_value_by_key(x): return last +def GetInstalledPackages(venvPy): + packagesInstalled = {} + cmd = [ + venvPy, + "-m", + "pip", + "--no-color", + "--no-input", + "--disable-pip-version-check", + "list", + "--local", + "--format", + "json", + "--verbose", + ] + err, results = malcolm_utils.run_process(cmd, stderr=False, logger=logging) + if (err == 0) and results and (len(results) > 0): + try: + packagesInstalled = {item['name']: item for item in malcolm_utils.LoadStrIfJson(results[0])} + except Exception as e: + logging.error(f"{type(e).__name__} getting list of installed Python packages: {e}") + + return packagesInstalled + + +def InstallPackageDirIfNeeded( + packageDir, + venvPy, + preinstalledPackagesDict={}, +): + installResult = False + + # First do a "dry run" install to determine what would happen. The report from this will + # help us determine if the package actually needs installed or not, as pip always treats + # installations from local directories as "new installs" and would uninstall/reinstall + # no matter what, which we want to avoid if we don't need it. + pluginNeedsInstall = False + with malcolm_utils.temporary_filename(suffix='.json') as dryRunInstallReportFileName: + cmd = [ + venvPy, + "-m", + "pip", + "--no-color", + "--no-input", + "--disable-pip-version-check", + "install", + "--upgrade", + "--dry-run", + "--progress-bar", + "off", + "--report", + dryRunInstallReportFileName, + packageDir, + ] + err, results = malcolm_utils.run_process(cmd, logger=logging) + if (err == 0) and os.path.isfile(dryRunInstallReportFileName): + with open(dryRunInstallReportFileName, 'r') as f: + dryRunReport = malcolm_utils.LoadFileIfJson(f) + wouldInstallInfo = { + malcolm_utils.deep_get(installItem, ['metadata', 'name']): malcolm_utils.deep_get( + installItem, ['metadata', 'version'] + ) + for installItem in dryRunReport.get('install', []) + } + pluginNeedsInstall = any( + [ + package_name + for package_name, new_version in wouldInstallInfo.items() + if (package_name not in preinstalledPackagesDict) + or (Version(new_version) > Version(preinstalledPackagesDict[package_name]['version'])) + ] + ) + else: + pluginNeedsInstall = True + + if pluginNeedsInstall: + with malcolm_utils.temporary_filename(suffix='.json') as installReportFileName: + cmd = [ + venvPy, + "-m", + "pip", + "--no-color", + "--no-input", + "--disable-pip-version-check", + "install", + "--upgrade", + "--progress-bar", + "off", + "--report", + installReportFileName, + packageDir, + ] + err, results = malcolm_utils.run_process(cmd, logger=logging) + installResult = err == 0 + + return installResult + + ################################################################################################### # main def main(): @@ -219,6 +280,14 @@ def main(): required=False, help="NetBox installation directory", ) + parser.add_argument( + '--netbox-config', + dest='netboxConfigDir', + type=str, + default=os.getenv('NETBOX_CONFIG_PATH', '/etc/netbox/config'), + required=False, + help="NetBox config directory (containing plugins.py, etc.)", + ) parser.add_argument( '-l', '--library', @@ -228,6 +297,15 @@ def main(): required=False, help="Directory containing NetBox Device-Type-Library-Import project and library repo", ) + parser.add_argument( + '-c', + '--custom-plugins', + dest='customPluginsDir', + type=str, + default=os.getenv('NETBOX_CUSTOM_PLUGINS_PATH', '/opt/netbox-custom-plugins'), + required=False, + help="Parent directory containing custom NetBox plugins to install", + ) parser.add_argument( '-p', '--preload', @@ -297,6 +375,132 @@ def main(): netboxVenvPy = os.path.join(os.path.join(os.path.join(args.netboxDir, 'venv'), 'bin'), 'python') manageScript = os.path.join(os.path.join(args.netboxDir, 'netbox'), 'manage.py') + # CUSTOM PLUGIN INSTALLATION ################################################################################# + if os.path.isdir(args.customPluginsDir) and os.path.isfile(os.path.join(args.netboxConfigDir, 'plugins.py')): + + # get a list of what packages/plugins already installed (package names and versions in a dict) + packagesInstalled = GetInstalledPackages(netboxVenvPy) + + # if there is a "requirements" subdirectory, handle that first as it contains dependencies + if os.path.isdir(os.path.join(args.customPluginsDir, 'requirements')): + requirementsSubDirs = [ + malcolm_utils.remove_suffix(f.path, '/') + for f in os.scandir(os.path.join(args.customPluginsDir, 'requirements')) + if f.is_dir() + ] + for packageDir in requirementsSubDirs: + packageInstalled = InstallPackageDirIfNeeded(packageDir, netboxVenvPy, packagesInstalled) + logging.info( + f"{os.path.basename(packageDir)} (dependency): {'' if packageInstalled else 'not ' }installed" + ) + + # now install the plugins directories + installedOrUpdatedPlugins = [] + customPluginSubdirs = [ + malcolm_utils.remove_suffix(f.path, '/') + for f in os.scandir(args.customPluginsDir) + if f.is_dir() and (os.path.basename(f) != 'requirements') + ] + for pluginDir in customPluginSubdirs: + if pluginInstalled := InstallPackageDirIfNeeded(pluginDir, netboxVenvPy, packagesInstalled): + installedOrUpdatedPlugins.append(pluginDir) + logging.info(f"{os.path.basename(pluginDir)}: {'' if pluginInstalled else 'not ' }installed") + + # for any packages that were newly installed (or updated, we'll be thorough) we need to make + # sure the package name is in the plugins.py + logging.info(f"Plugins installed or updated: {installedOrUpdatedPlugins}") + if installedOrUpdatedPlugins: + # get updated list of installed packages + packagesInstalled = GetInstalledPackages(netboxVenvPy) + + # now get the names of the NetBox plugins installed + pluginNames = [] + + # first get a list of __init__.py files for potential plugins installed in the package location(s) + cmd = [ + '/usr/bin/rg', + '--files-with-matches', + '--iglob', + '__init__.py', + r'\bPluginConfig\b', + list({package['location'] for package in packagesInstalled.values() if 'location' in package}), + ] + err, results = malcolm_utils.run_process(cmd, stderr=False, logger=logging) + if results: + # process each of those potential plugin __init__.py files + for pluginInitFileName in results: + try: + if os.path.isfile(pluginInitFileName): + # parse the Python of the __init__.py into an abstract syntax tree + with open(pluginInitFileName, 'r') as f: + node = ast.parse(f.read()) + # look at each Class defined in this code + for c in [n for n in node.body if isinstance(n, ast.ClassDef)]: + # plugins are classes with "PluginConfig" for a parent + if any([baseClass.id == 'PluginConfig' for baseClass in c.bases]): + # this ia a plugin class, so iterate over its members (functions, + # variables, etc.) to find its name + for item in c.body: + # the name is defined as an assignment (ast.Assign) + if isinstance(item, ast.Assign): + # does this assignment have a target called 'name'? + for target in item.targets: + if isinstance(target, ast.Name) and target.id == 'name': + # check if the value assigned to 'name' is a constant + if isinstance(item.value, ast.Constant): + pluginNames.append(item.value.value) + except Exception as e: + logging.error(f"{type(e).__name__} identifying NetBox plugin names: {e}") + + if pluginNames: + pluginNames = list(set(pluginNames)) + # at this point we have a list of plugin names for all of the plugin classes! + # we need to make sure they exist in plugins.py + + # Load and parse the plugins.py file + pluginsListFound = False + with open(os.path.join(args.netboxConfigDir, 'plugins.py'), 'r') as pluginFile: + code = pluginFile.read() + tree = ast.parse(code) + + # Walk the AST to find the PLUGINS assignment + class PluginListModifier(ast.NodeTransformer): + def visit_Assign(self, node): + global pluginsListFound + if isinstance(node.targets[0], ast.Name) and node.targets[0].id == 'PLUGINS': + pluginsListFound = True + # Check if the node's value is a list + if isinstance(node.value, ast.List): + # Get the existing plugin names in the list + existingPlugins = {elt.s for elt in node.value.elts if isinstance(elt, ast.Str)} + # Add new plugins if they aren't already in the list + for plugin in pluginNames: + if plugin not in existingPlugins: + node.value.elts.append(ast.Constant(value=plugin)) + return node + + # Modify the AST + modifier = PluginListModifier() + modifiedTree = modifier.visit(tree) + + # # If PLUGINS was not found, add it at the end of the module + if not pluginsListFound: + modifiedTree.body.append( + ast.Assign( + targets=[ast.Name(id='PLUGINS', ctx=ast.Store())], + value=ast.List(elts=[ast.Constant(value=plugin) for plugin in pluginNames], ctx=ast.Load()), + ) + ) + + # Unparse the modified AST back into code + modifiedCode = ast.unparse(ast.fix_missing_locations(modifiedTree)) + + # Write the modified code back to the file + with open(os.path.join(args.netboxConfigDir, 'plugins.py'), 'w') as pluginFile: + pluginFile.write(modifiedCode) + + # END CUSTOM PLUGIN INSTALLATION ############################################################################# + # if there is a database backup .gz in the preload directory, load it up (preferring the newest) # if there are multiple) instead of populating via API preloadDatabaseFile = args.preloadBackupFile diff --git a/netbox/supervisord.conf b/netbox/supervisord.conf index be80d8736..0764e5865 100644 --- a/netbox/supervisord.conf +++ b/netbox/supervisord.conf @@ -39,6 +39,7 @@ command=/opt/netbox/venv/bin/python /usr/local/bin/netbox_init.py --token "%(ENV_SUPERUSER_API_TOKEN)s" --library "%(ENV_NETBOX_DEVICETYPE_LIBRARY_IMPORT_PATH)s" --preload "%(ENV_NETBOX_PRELOAD_PATH)s" + --custom-plugins "%(ENV_NETBOX_CUSTOM_PLUGINS_PATH)s" --postgres-host "%(ENV_DB_HOST)s" --postgres-db "%(ENV_DB_NAME)s" --postgres-user "%(ENV_DB_USER)s" diff --git a/scripts/malcolm_appliance_packager.sh b/scripts/malcolm_appliance_packager.sh index 2c824c099..bd4634b96 100755 --- a/scripts/malcolm_appliance_packager.sh +++ b/scripts/malcolm_appliance_packager.sh @@ -66,6 +66,7 @@ if mkdir "$DESTDIR"; then mkdir $VERBOSE -p "$DESTDIR/htadmin/" mkdir $VERBOSE -p "$DESTDIR/logstash/certs/" mkdir $VERBOSE -p "$DESTDIR/logstash/maps/" + mkdir $VERBOSE -p "$DESTDIR/netbox/custom-plugins/requirements/" mkdir $VERBOSE -p "$DESTDIR/netbox/media/" mkdir $VERBOSE -p "$DESTDIR/netbox/postgres/" mkdir $VERBOSE -p "$DESTDIR/netbox/redis/" diff --git a/scripts/malcolm_utils.py b/scripts/malcolm_utils.py index 208ef2d2e..80d6ccf68 100644 --- a/scripts/malcolm_utils.py +++ b/scripts/malcolm_utils.py @@ -176,9 +176,9 @@ def decapitalize(s): # # Example: # d = {'meta': {'status': 'OK', 'status_code': 200}} -# DeepGet(d, ['meta', 'status_code']) # => 200 -# DeepGet(d, ['garbage', 'status_code']) # => None -# DeepGet(d, ['meta', 'garbage'], default='-') # => '-' +# deep_get(d, ['meta', 'status_code']) # => 200 +# deep_get(d, ['garbage', 'status_code']) # => None +# deep_get(d, ['meta', 'garbage'], default='-') # => '-' def deep_get(d, keys, default=None): k = get_iterable(keys) if d is None: