From 75d9cafacaf8cdcac2f457e45290caa4ebfc5448 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Thu, 18 Jul 2024 15:32:12 -0600 Subject: [PATCH] remove long-deprecated net-map.json file support, idaholab/Malcolm#517 --- _config.yml | 1 - docker-compose-dev.yml | 6 - docker-compose.yml | 6 - docs/contributing-local-modifications.md | 643 +++++++++++++++++++---- docs/kubernetes.md | 1 - malcolm-iso/build.sh | 1 - net-map.json | 1 - netbox/scripts/netbox_init.py | 141 ----- netbox/supervisord.conf | 1 - scripts/malcolm_appliance_packager.sh | 1 - scripts/malcolm_kubernetes.py | 6 - 11 files changed, 538 insertions(+), 270 deletions(-) delete mode 100644 net-map.json diff --git a/_config.yml b/_config.yml index 3ba54be6f..e93e6c7af 100644 --- a/_config.yml +++ b/_config.yml @@ -78,7 +78,6 @@ exclude: - htadmin - logstash - malcolm-iso - - net-map.json - netbox - nginx - opensearch diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index d771b3b04..2147df912 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -1112,12 +1112,6 @@ services: create_host_path: false source: ./netbox/media target: /opt/netbox/netbox/media - - type: bind - bind: - create_host_path: false - source: ./net-map.json - target: /usr/local/share/net-map.json - read_only: true - type: bind bind: create_host_path: false diff --git a/docker-compose.yml b/docker-compose.yml index 7f02b9f76..8b5389060 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -986,12 +986,6 @@ services: create_host_path: false source: ./netbox/media target: /opt/netbox/netbox/media - - type: bind - bind: - create_host_path: false - source: ./net-map.json - target: /usr/local/share/net-map.json - read_only: true - type: bind bind: create_host_path: false diff --git a/docs/contributing-local-modifications.md b/docs/contributing-local-modifications.md index 6183410f6..c58be6d16 100644 --- a/docs/contributing-local-modifications.md +++ b/docs/contributing-local-modifications.md @@ -6,111 +6,544 @@ There are several ways to customize Malcolm's runtime behavior via local changes Some configuration changes can be put in place by modifying local copies of configuration files and then using a [Docker bind mount](https://docs.docker.com/storage/bind-mounts/) to overlay the modified file onto the running Malcolm container. This is already done for many files and directories used to persist Malcolm configuration and data. For example, the default list of bind mounted files and directories for each Malcolm service is as follows: -``` -$ grep -P "^( - ./| [\w-]+:)" docker-compose.yml -opensearch: - - ./nginx/ca-trust:/var/local/ca-trust:ro - - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro - - ./.opensearch.secondary.curlrc:/var/local/curlrc/.opensearch.secondary.curlrc:ro - - ./opensearch:/usr/share/opensearch/data - - ./opensearch-backup:/opt/opensearch/backup - - ./opensearch/opensearch.keystore:/usr/share/opensearch/config/persist/opensearch.keystore:rw -dashboards-helper: - - ./nginx/ca-trust:/var/local/ca-trust:ro - - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro - - ./.opensearch.secondary.curlrc:/var/local/curlrc/.opensearch.secondary.curlrc:ro -dashboards: - - ./nginx/ca-trust:/var/local/ca-trust:ro - - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro -logstash: - - ./nginx/ca-trust:/var/local/ca-trust:ro - - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro - - ./.opensearch.secondary.curlrc:/var/local/curlrc/.opensearch.secondary.curlrc:ro - - ./logstash/maps/malcolm_severity.yaml:/etc/malcolm_severity.yaml:ro - - ./logstash/certs/ca.crt:/certs/ca.crt:ro - - ./logstash/certs/server.crt:/certs/server.crt:ro - - ./logstash/certs/server.key:/certs/server.key:ro -filebeat: - - ./nginx/ca-trust:/var/local/ca-trust:ro - - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro - - ./zeek-logs:/zeek - - ./suricata-logs:/suricata - - ./filebeat/certs/ca.crt:/certs/ca.crt:ro - - ./filebeat/certs/client.crt:/certs/client.crt:ro - - ./filebeat/certs/client.key:/certs/client.key:ro -arkime: - - ./nginx/ca-trust:/var/local/ca-trust:ro - - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro - - ./pcap:/data/pcap -zeek: - - ./nginx/ca-trust:/var/local/ca-trust:ro - - ./pcap:/pcap - - ./zeek-logs/upload:/zeek/upload - - ./zeek-logs/extract_files:/zeek/extract_files - - ./zeek/intel:/opt/zeek/share/zeek/site/intel - - ./zeek/custom:/opt/zeek/share/zeek/site/custom:ro -zeek-live: - - ./nginx/ca-trust:/var/local/ca-trust:ro - - ./zeek-logs/live:/zeek/live - - ./zeek-logs/extract_files:/zeek/extract_files - - ./zeek/intel:/opt/zeek/share/zeek/site/intel - - ./zeek/custom:/opt/zeek/share/zeek/site/custom:ro -suricata: - - ./nginx/ca-trust:/var/local/ca-trust:ro - - ./suricata-logs:/var/log/suricata - - ./pcap:/data/pcap - - ./suricata/rules:/opt/suricata/rules:ro - - ./suricata/include-configs:/opt/suricata/include-configs:ro -suricata-live: - - ./nginx/ca-trust:/var/local/ca-trust:ro - - ./suricata-logs:/var/log/suricata - - ./suricata/rules:/opt/suricata/rules:ro - - ./suricata/include-configs:/opt/suricata/include-configs:ro -file-monitor: - - ./nginx/ca-trust:/var/local/ca-trust:ro - - ./zeek-logs/extract_files:/zeek/extract_files - - ./zeek-logs/current:/zeek/logs - - ./yara/rules:/yara-rules/custom:ro -pcap-capture: - - ./nginx/ca-trust:/var/local/ca-trust:ro - - ./pcap/upload:/pcap -pcap-monitor: - - ./nginx/ca-trust:/var/local/ca-trust:ro - - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro - - ./zeek-logs:/zeek - - ./pcap:/pcap -upload: - - ./nginx/ca-trust:/var/local/ca-trust:ro - - ./pcap/upload:/var/www/upload/server/php/chroot/files -htadmin: - - ./nginx/ca-trust:/var/local/ca-trust:ro - - ./htadmin/config.ini:/var/www/htadmin/config/config.ini:rw - - ./htadmin/metadata:/var/www/htadmin/config/metadata:rw - - ./nginx/htpasswd:/var/www/htadmin/auth/htpasswd:rw -freq: - - ./nginx/ca-trust:/var/local/ca-trust:ro -netbox: - - ./nginx/ca-trust:/var/local/ca-trust:ro - - ./netbox/config:/etc/netbox/config:ro - - ./netbox/media:/opt/netbox/netbox/media:rw - - ./net-map.json:/usr/local/share/net-map.json:ro -netbox-postgres: - - ./nginx/ca-trust:/var/local/ca-trust:ro - - ./netbox/postgres:/var/lib/postgresql/data:rw -netbox-redis: - - ./nginx/ca-trust:/var/local/ca-trust:ro - - ./netbox/redis:/data -netbox-redis-cache: - - ./nginx/ca-trust:/var/local/ca-trust:ro -api: - - ./nginx/ca-trust:/var/local/ca-trust:ro - - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro -nginx-proxy: - - ./nginx/ca-trust:/var/local/ca-trust:ro - - ./nginx/nginx_ldap.conf:/etc/nginx/nginx_ldap.conf:ro - - ./nginx/htpasswd:/etc/nginx/auth/htpasswd:ro - - ./nginx/certs:/etc/nginx/certs:ro - - ./nginx/certs/dhparam.pem:/etc/nginx/dhparam/dhparam.pem:ro +`$ yq eval '.services = (.services | with_entries(.value = {"volumes": .value.volumes}))' docker-compose.yml` +```yaml +services: + opensearch: + volumes: + - type: bind + bind: + create_host_path: false + source: ./nginx/ca-trust + target: /var/local/ca-trust + read_only: true + - type: bind + bind: + create_host_path: false + source: ./.opensearch.primary.curlrc + target: /var/local/curlrc/.opensearch.primary.curlrc + read_only: true + - type: bind + bind: + create_host_path: false + source: ./.opensearch.secondary.curlrc + target: /var/local/curlrc/.opensearch.secondary.curlrc + read_only: true + - type: bind + bind: + create_host_path: false + source: ./opensearch + target: /usr/share/opensearch/data + - type: bind + bind: + create_host_path: false + source: ./opensearch-backup + target: /opt/opensearch/backup + - type: bind + bind: + create_host_path: false + source: ./opensearch/opensearch.keystore + target: /usr/share/opensearch/config/persist/opensearch.keystore + dashboards-helper: + volumes: + - type: bind + bind: + create_host_path: false + source: ./nginx/ca-trust + target: /var/local/ca-trust + read_only: true + - type: bind + bind: + create_host_path: false + source: ./.opensearch.primary.curlrc + target: /var/local/curlrc/.opensearch.primary.curlrc + read_only: true + - type: bind + bind: + create_host_path: false + source: ./.opensearch.secondary.curlrc + target: /var/local/curlrc/.opensearch.secondary.curlrc + read_only: true + dashboards: + volumes: + - type: bind + bind: + create_host_path: false + source: ./nginx/ca-trust + target: /var/local/ca-trust + read_only: true + - type: bind + bind: + create_host_path: false + source: ./.opensearch.primary.curlrc + target: /var/local/curlrc/.opensearch.primary.curlrc + read_only: true + logstash: + volumes: + - type: bind + bind: + create_host_path: false + source: ./nginx/ca-trust + target: /var/local/ca-trust + read_only: true + - type: bind + bind: + create_host_path: false + source: ./.opensearch.primary.curlrc + target: /var/local/curlrc/.opensearch.primary.curlrc + read_only: true + - type: bind + bind: + create_host_path: false + source: ./.opensearch.secondary.curlrc + target: /var/local/curlrc/.opensearch.secondary.curlrc + read_only: true + - type: bind + bind: + create_host_path: false + source: ./logstash/maps/malcolm_severity.yaml + target: /etc/malcolm_severity.yaml + read_only: true + - type: bind + bind: + create_host_path: false + source: ./logstash/certs/ca.crt + target: /certs/ca.crt + read_only: true + - type: bind + bind: + create_host_path: false + source: ./logstash/certs/server.crt + target: /certs/server.crt + read_only: true + - type: bind + bind: + create_host_path: false + source: ./logstash/certs/server.key + target: /certs/server.key + read_only: true + filebeat: + volumes: + - nginx-log-path:/nginx:ro + - type: bind + bind: + create_host_path: false + source: ./nginx/ca-trust + target: /var/local/ca-trust + read_only: true + - type: bind + bind: + create_host_path: false + source: ./.opensearch.primary.curlrc + target: /var/local/curlrc/.opensearch.primary.curlrc + read_only: true + - type: bind + bind: + create_host_path: false + source: ./zeek-logs + target: /zeek + - type: bind + bind: + create_host_path: false + source: ./suricata-logs + target: /suricata + - type: bind + bind: + create_host_path: false + source: ./filebeat/certs/ca.crt + target: /certs/ca.crt + read_only: true + - type: bind + bind: + create_host_path: false + source: ./filebeat/certs/client.crt + target: /certs/client.crt + read_only: true + - type: bind + bind: + create_host_path: false + source: ./filebeat/certs/client.key + target: /certs/client.key + read_only: true + arkime: + volumes: + - type: bind + bind: + create_host_path: false + source: ./nginx/ca-trust + target: /var/local/ca-trust + read_only: true + - type: bind + bind: + create_host_path: false + source: ./.opensearch.primary.curlrc + target: /var/local/curlrc/.opensearch.primary.curlrc + read_only: true + - type: bind + bind: + create_host_path: false + source: ./arkime/rules + target: /opt/arkime/rules + read_only: true + - type: bind + bind: + create_host_path: false + source: ./pcap + target: /data/pcap + arkime-live: + volumes: + - type: bind + bind: + create_host_path: false + source: ./nginx/ca-trust + target: /var/local/ca-trust + read_only: true + - type: bind + bind: + create_host_path: false + source: ./.opensearch.primary.curlrc + target: /var/local/curlrc/.opensearch.primary.curlrc + read_only: true + - type: bind + bind: + create_host_path: false + source: ./arkime/rules + target: /opt/arkime/rules + read_only: true + - type: bind + bind: + create_host_path: false + source: ./pcap + target: /data/pcap + zeek: + volumes: + - type: bind + bind: + create_host_path: false + source: ./nginx/ca-trust + target: /var/local/ca-trust + read_only: true + - type: bind + bind: + create_host_path: false + source: ./pcap + target: /pcap + - type: bind + bind: + create_host_path: false + source: ./zeek-logs/upload + target: /zeek/upload + - type: bind + bind: + create_host_path: false + source: ./zeek-logs/extract_files + target: /zeek/extract_files + - type: bind + bind: + create_host_path: false + source: ./zeek/intel + target: /opt/zeek/share/zeek/site/intel + - type: bind + bind: + create_host_path: false + source: ./zeek/custom + target: /opt/zeek/share/zeek/site/custom + read_only: true + zeek-live: + volumes: + - type: bind + bind: + create_host_path: false + source: ./nginx/ca-trust + target: /var/local/ca-trust + read_only: true + - type: bind + bind: + create_host_path: false + source: ./zeek-logs/live + target: /zeek/live + - type: bind + bind: + create_host_path: false + source: ./zeek-logs/extract_files + target: /zeek/extract_files + - type: bind + bind: + create_host_path: false + source: ./zeek/intel + target: /opt/zeek/share/zeek/site/intel + - type: bind + bind: + create_host_path: false + source: ./zeek/custom + target: /opt/zeek/share/zeek/site/custom + read_only: true + suricata: + volumes: + - type: bind + bind: + create_host_path: false + source: ./nginx/ca-trust + target: /var/local/ca-trust + read_only: true + - type: bind + bind: + create_host_path: false + source: ./suricata-logs + target: /var/log/suricata + - type: bind + bind: + create_host_path: false + source: ./pcap + target: /data/pcap + - type: bind + bind: + create_host_path: false + source: ./suricata/rules + target: /opt/suricata/rules + read_only: true + - type: bind + bind: + create_host_path: false + source: ./suricata/include-configs + target: /opt/suricata/include-configs + read_only: true + suricata-live: + volumes: + - type: bind + bind: + create_host_path: false + source: ./nginx/ca-trust + target: /var/local/ca-trust + read_only: true + - type: bind + bind: + create_host_path: false + source: ./suricata-logs + target: /var/log/suricata + - type: bind + bind: + create_host_path: false + source: ./suricata/rules + target: /opt/suricata/rules + read_only: true + - type: bind + bind: + create_host_path: false + source: ./suricata/include-configs + target: /opt/suricata/include-configs + read_only: true + file-monitor: + volumes: + - type: bind + bind: + create_host_path: false + source: ./nginx/ca-trust + target: /var/local/ca-trust + read_only: true + - type: bind + bind: + create_host_path: false + source: ./zeek-logs/extract_files + target: /zeek/extract_files + - type: bind + bind: + create_host_path: false + source: ./zeek-logs/current + target: /zeek/logs + - type: bind + bind: + create_host_path: false + source: ./yara/rules + target: /yara-rules/custom + read_only: true + pcap-capture: + volumes: + - type: bind + bind: + create_host_path: false + source: ./nginx/ca-trust + target: /var/local/ca-trust + read_only: true + - type: bind + bind: + create_host_path: false + source: ./pcap/upload + target: /pcap + pcap-monitor: + volumes: + - type: bind + bind: + create_host_path: false + source: ./nginx/ca-trust + target: /var/local/ca-trust + read_only: true + - type: bind + bind: + create_host_path: false + source: ./.opensearch.primary.curlrc + target: /var/local/curlrc/.opensearch.primary.curlrc + read_only: true + - type: bind + bind: + create_host_path: false + source: ./zeek-logs + target: /zeek + - type: bind + bind: + create_host_path: false + source: ./pcap + target: /pcap + upload: + volumes: + - type: bind + bind: + create_host_path: false + source: ./nginx/ca-trust + target: /var/local/ca-trust + read_only: true + - type: bind + bind: + create_host_path: false + source: ./pcap/upload + target: /var/www/upload/server/php/chroot/files + htadmin: + volumes: + - type: bind + bind: + create_host_path: false + source: ./nginx/ca-trust + target: /var/local/ca-trust + read_only: true + - type: bind + bind: + create_host_path: false + source: ./htadmin/config.ini + target: /var/www/htadmin/config/config.ini + - type: bind + bind: + create_host_path: false + source: ./htadmin/metadata + target: /var/www/htadmin/config/metadata + - type: bind + bind: + create_host_path: false + source: ./nginx/htpasswd + target: /var/www/htadmin/auth/htpasswd + freq: + volumes: + - type: bind + bind: + create_host_path: false + source: ./nginx/ca-trust + target: /var/local/ca-trust + read_only: true + netbox: + volumes: + - type: bind + bind: + create_host_path: false + source: ./nginx/ca-trust + target: /var/local/ca-trust + read_only: true + - type: bind + bind: + create_host_path: false + source: ./netbox/config + target: /etc/netbox/config + read_only: true + - type: bind + bind: + create_host_path: false + source: ./netbox/media + target: /opt/netbox/netbox/media + - type: bind + bind: + create_host_path: false + source: ./netbox/preload + target: /opt/netbox-preload/configmap + read_only: true + netbox-postgres: + volumes: + - type: bind + bind: + create_host_path: false + source: ./nginx/ca-trust + target: /var/local/ca-trust + read_only: true + - type: bind + bind: + create_host_path: false + source: ./netbox/postgres + target: /var/lib/postgresql/data + netbox-redis: + volumes: + - type: bind + bind: + create_host_path: false + source: ./nginx/ca-trust + target: /var/local/ca-trust + read_only: true + - type: bind + bind: + create_host_path: false + source: ./netbox/redis + target: /data + netbox-redis-cache: + volumes: + - type: bind + bind: + create_host_path: false + source: ./nginx/ca-trust + target: /var/local/ca-trust + read_only: true + api: + volumes: + - type: bind + bind: + create_host_path: false + source: ./nginx/ca-trust + target: /var/local/ca-trust + read_only: true + - type: bind + bind: + create_host_path: false + source: ./.opensearch.primary.curlrc + target: /var/local/curlrc/.opensearch.primary.curlrc + read_only: true + nginx-proxy: + volumes: + - nginx-log-path:/var/log/nginx + - type: bind + bind: + create_host_path: false + source: ./nginx/ca-trust + target: /var/local/ca-trust + read_only: true + - type: bind + bind: + create_host_path: false + source: ./nginx/nginx_ldap.conf + target: /etc/nginx/nginx_ldap.conf + read_only: true + - type: bind + bind: + create_host_path: false + source: ./nginx/htpasswd + target: /etc/nginx/auth/htpasswd + read_only: true + - type: bind + bind: + create_host_path: false + source: ./nginx/certs + target: /etc/nginx/certs + read_only: true + - type: bind + bind: + create_host_path: false + source: ./nginx/certs/dhparam.pem + target: /etc/nginx/dhparam/dhparam.pem + read_only: true ``` So, for example, if a user wanted to make a change to the `nginx-proxy` container's `nginx.conf` file, they could add the following line to the `volumes:` section of the `nginx-proxy` service in the `docker-compose.yml` file: diff --git a/docs/kubernetes.md b/docs/kubernetes.md index 665482cdb..71049e9e5 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -324,7 +324,6 @@ drwxr-xr-x 3 user user 19 Apr 24 14:35 yara drwxr-xr-x 3 user user 19 Apr 24 14:35 zeek drwxr-xr-x 7 user user 85 Apr 24 14:35 zeek-logs -rw-r--r-- 1 user user 18,761 Apr 24 14:35 docker-compose.yml --rw-r--r-- 1 user user 2 Apr 24 14:35 net-map.json -rw-r--r-- 1 user user 3,453 Apr 24 14:35 README.md ``` diff --git a/malcolm-iso/build.sh b/malcolm-iso/build.sh index 838f72248..dc325fcb2 100755 --- a/malcolm-iso/build.sh +++ b/malcolm-iso/build.sh @@ -128,7 +128,6 @@ if [ -d "$WORKDIR" ]; then mkdir -p "$MALCOLM_DEST_DIR/zeek/intel/MISP/" mkdir -p "$MALCOLM_DEST_DIR/zeek/intel/STIX/" cp ./docker-compose.yml "$MALCOLM_DEST_DIR/docker-compose.yml" - cp ./net-map.json "$MALCOLM_DEST_DIR/" cp ./scripts/install.py "$MALCOLM_DEST_DIR/scripts/" cp ./scripts/control.py "$MALCOLM_DEST_DIR/scripts/" pushd "$MALCOLM_DEST_DIR/scripts/" >/dev/null 2>&1 diff --git a/net-map.json b/net-map.json deleted file mode 100644 index 0637a088a..000000000 --- a/net-map.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/netbox/scripts/netbox_init.py b/netbox/scripts/netbox_init.py index 5cc502a5a..f1337ca90 100755 --- a/netbox/scripts/netbox_init.py +++ b/netbox/scripts/netbox_init.py @@ -164,14 +164,6 @@ def main(): required=False, help="Site(s) to create", ) - parser.add_argument( - '--net-map', - dest='netMapFileName', - type=str, - default=None, - required=False, - help="Filename of JSON file containing network subnet/host name mapping", - ) parser.add_argument( '--default-group', dest='defaultGroupName', @@ -710,139 +702,6 @@ def main(): except Exception as e: logging.error(f"{type(e).__name__} processing sites: {e}") - # ###### Net Map ############################################################################################### - try: - # load net-map.json from file - netMapJson = None - if args.netMapFileName is not None and os.path.isfile(args.netMapFileName): - with open(args.netMapFileName) as f: - netMapJson = json.load(f) - if netMapJson is not None: - # create IP prefixes - - prefixesPreExisting = {x.prefix: x for x in nb.ipam.prefixes.all()} - logging.debug(f"prefixes (before): { {k:v.id for k, v in prefixesPreExisting.items()} }") - - for segment in [ - x - for x in get_iterable(netMapJson) - if isinstance(x, dict) - and (x.get('type', '') == "segment") - and x.get('name', None) - and is_ip_network(x.get('address', None)) - ]: - try: - site = min_hash_value_by_value(sites) - nb.ipam.prefixes.create( - { - "prefix": segment['address'], - "site": site.id if site else None, - "description": segment['name'], - }, - ) - except pynetbox.RequestError as nbe: - logging.warning( - f"{type(nbe).__name__} processing prefix \"{segment['address']}\" (\"{segment['name']}\"): {nbe}" - ) - - prefixes = {x.prefix: x for x in nb.ipam.prefixes.all()} - logging.debug(f"prefixes (after): { {k:v.id for k, v in prefixes.items()} }") - - # create hosts as devices - devicesPreExisting = {x.name: x for x in nb.dcim.devices.all()} - logging.debug(f"devices (before): { {k:v.id for k, v in devicesPreExisting.items()} }") - - for host in [ - x - for x in get_iterable(netMapJson) - if isinstance(x, dict) - and (x.get('type', '') == "host") - and x.get('name', None) - and x.get('address', None) - and x['name'] not in devicesPreExisting - ]: - try: - site = min_hash_value_by_value(sites) - dType = min_hash_value_by_value(deviceTypes) - role = min_hash_value_by_value(roles) - deviceCreated = nb.dcim.devices.create( - { - "name": host['name'], - "site": site.id if site else None, - "device_type": dType.id if dType else None, - "role": role.id if role else None, - }, - ) - if deviceCreated is not None: - # create interface for the device - if is_ip_address(host['address']): - nb.dcim.interfaces.create( - { - "device": deviceCreated.id, - "name": "default", - "type": "other", - }, - ) - elif re.match(r'^([0-9a-f]{2}[:-]){5}([0-9a-f]{2})$', host['address'].lower()): - nb.dcim.interfaces.create( - { - "device": deviceCreated.id, - "name": "default", - "type": "other", - "mac_address": host['address'].lower(), - }, - ) - - except pynetbox.RequestError as nbe: - logging.warning(f"{type(nbe).__name__} processing device \"{host['name']}\": {nbe}") - - devices = {x.name: x for x in nb.dcim.devices.all()} - logging.debug(f"devices (after): { {k:v.id for k, v in devices.items()} }") - interfaces = {x.device.id: x for x in nb.dcim.interfaces.all()} - logging.debug(f"interfaces (after): { {k:v.id for k, v in interfaces.items()} }") - - # and associate IP addresses with them - ipAddressesPreExisting = {x.address: x for x in nb.ipam.ip_addresses.all()} - logging.debug(f"IP addresses (before): { {k:v.id for k, v in ipAddressesPreExisting.items()} }") - - for host in [ - x - for x in get_iterable(netMapJson) - if isinstance(x, dict) - and (x.get('type', '') == "host") - and x.get('name', None) - and is_ip_address(x.get('address', None)) - and x['name'] in devices - ]: - try: - hostKey = f"{host['address']}/{'32' if is_ip_v4_address(host['address']) else '128'}" - if hostKey not in ipAddressesPreExisting: - ipCreated = nb.ipam.ip_addresses.create( - { - "address": host['address'], - "assigned_object_type": "dcim.interface", - "assigned_object_id": interfaces[devices[host['name']].id].id, - }, - ) - if ipCreated is not None: - # update device to set this as its primary IPv4 address - deviceForIp = nb.dcim.devices.get(id=devices[host['name']].id) - if deviceForIp is not None: - if is_ip_v4_address(host['address']): - deviceForIp.primary_ip4 = ipCreated - elif is_ip_v6_address(host['address']): - deviceForIp.primary_ip = ipCreated - deviceForIp.save() - - except pynetbox.RequestError as nbe: - logging.warning(f"{type(nbe).__name__} processing address \"{host['address']}\": {nbe}") - - ipAddresses = {x.address: x for x in nb.ipam.ip_addresses.all()} - logging.debug(f"IP addresses (after): { {k:v.id for k, v in ipAddresses.items()} }") - - except Exception as e: - logging.error(f"{type(e).__name__} processing net map JSON \"{args.netMapFileName}\": {e}") - # ###### Missing prefix descriptions from VRF names (see idaholab/Malcolm#280) ################################## try: for prefix in [x for x in nb.ipam.prefixes.filter(description__empty=True) if x.vrf]: diff --git a/netbox/supervisord.conf b/netbox/supervisord.conf index f83ca0647..be80d8736 100644 --- a/netbox/supervisord.conf +++ b/netbox/supervisord.conf @@ -37,7 +37,6 @@ command=/opt/netbox/venv/bin/python /usr/local/bin/netbox_init.py --wait --url "http://localhost:8080/netbox" --token "%(ENV_SUPERUSER_API_TOKEN)s" - --net-map /usr/local/share/net-map.json --library "%(ENV_NETBOX_DEVICETYPE_LIBRARY_IMPORT_PATH)s" --preload "%(ENV_NETBOX_PRELOAD_PATH)s" --postgres-host "%(ENV_DB_HOST)s" diff --git a/scripts/malcolm_appliance_packager.sh b/scripts/malcolm_appliance_packager.sh index 77be27b6b..84134eb33 100755 --- a/scripts/malcolm_appliance_packager.sh +++ b/scripts/malcolm_appliance_packager.sh @@ -96,7 +96,6 @@ if mkdir "$DESTDIR"; then cp $VERBOSE ./config/*.example "$DESTDIR/config/" cp $VERBOSE ./docker-compose.yml "$DESTDIR/docker-compose.yml" - cp $VERBOSE ./net-map.json "$DESTDIR/" cp $VERBOSE ./scripts/install.py "$DESTDIR/scripts/" cp $VERBOSE ./scripts/control.py "$DESTDIR/scripts/" cp $VERBOSE ./scripts/malcolm_common.py "$DESTDIR/scripts/" diff --git a/scripts/malcolm_kubernetes.py b/scripts/malcolm_kubernetes.py index f12e9d009..43e9d38a5 100644 --- a/scripts/malcolm_kubernetes.py +++ b/scripts/malcolm_kubernetes.py @@ -138,12 +138,6 @@ 'path': os.path.join(MalcolmPath, os.path.join('filebeat', 'certs')), }, ], - 'netbox-netmap-json': [ - { - 'secret': False, - 'path': os.path.join(MalcolmPath, 'net-map.json'), - }, - ], 'netbox-config': [ { 'secret': False,