From 0042c0c844f4cfc2dc8f4fc9087323a9fb6ccc99 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 1 May 2023 16:21:44 -0600 Subject: [PATCH 01/47] Bump flask from 2.0.2 to 2.3.2 in /api Bumps [flask](https://github.com/pallets/flask) from 2.0.2 to 2.3.2. - [Release notes](https://github.com/pallets/flask/releases) - [Changelog](https://github.com/pallets/flask/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/flask/compare/2.0.2...2.3.2) --- updated-dependencies: - dependency-name: flask dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- api/requirements.txt | 2 +- sensor-iso/interface/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/requirements.txt b/api/requirements.txt index 77c4ee103..abe59525b 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -1,5 +1,5 @@ pytz==2021.3 -Flask==2.0.2 +Flask==2.3.2 gunicorn==20.1.0 opensearch-py==2.2.0 requests==2.26.0 diff --git a/sensor-iso/interface/requirements.txt b/sensor-iso/interface/requirements.txt index fd639a82d..4d072296a 100644 --- a/sensor-iso/interface/requirements.txt +++ b/sensor-iso/interface/requirements.txt @@ -1,7 +1,7 @@ certifi==2022.12.7 chardet==5.1.0 click==8.1.3 -Flask==2.2.3 +Flask==2.3.2 Flask-Cors==3.0.10 gunicorn==20.1.0 idna==3.4 From 98eafdb149f95ac5364b655de31c367b901d5d6c Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 1 May 2023 17:33:53 -0600 Subject: [PATCH 02/47] bump werkzeug to 2.3.3 --- sensor-iso/interface/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sensor-iso/interface/requirements.txt b/sensor-iso/interface/requirements.txt index 4d072296a..0c55b80f8 100644 --- a/sensor-iso/interface/requirements.txt +++ b/sensor-iso/interface/requirements.txt @@ -13,4 +13,4 @@ python-dotenv==1.0.0 requests==2.28.2 six==1.16.0 urllib3==1.26.14 -Werkzeug==2.2.3 +Werkzeug==2.3.3 From d8a595db49c187b0e459f24e9d0e1689c4e3bab6 Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 4 May 2023 15:18:27 -0600 Subject: [PATCH 03/47] bump build version to v23.05.1 --- docker-compose-standalone.yml | 44 +++++++------- docker-compose.yml | 44 +++++++------- docs/download.md | 4 +- docs/hedgehog-iso-build.md | 2 +- docs/kubernetes.md | 88 ++++++++++++++-------------- docs/malcolm-iso.md | 2 +- docs/quickstart.md | 38 ++++++------ docs/ubuntu-install-example.md | 38 ++++++------ kubernetes/02-opensearch.yml | 2 +- kubernetes/03-dashboards.yml | 2 +- kubernetes/04-upload.yml | 2 +- kubernetes/05-pcap-monitor.yml | 2 +- kubernetes/06-arkime.yml | 2 +- kubernetes/07-api.yml | 2 +- kubernetes/08-dashboards-helper.yml | 2 +- kubernetes/09-zeek.yml | 2 +- kubernetes/10-suricata.yml | 2 +- kubernetes/11-file-monitor.yml | 2 +- kubernetes/12-filebeat.yml | 2 +- kubernetes/13-logstash.yml | 2 +- kubernetes/15-netbox-redis.yml | 2 +- kubernetes/16-netbox-redis-cache.yml | 2 +- kubernetes/17-netbox-postgres.yml | 2 +- kubernetes/18-netbox.yml | 2 +- kubernetes/19-htadmin.yml | 2 +- kubernetes/20-pcap-capture.yml | 2 +- kubernetes/21-zeek-live.yml | 2 +- kubernetes/22-suricata-live.yml | 2 +- kubernetes/23-freq.yml | 2 +- kubernetes/99-nginx-proxy.yml | 2 +- 30 files changed, 152 insertions(+), 152 deletions(-) diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml index f1efc89c0..f72ca6903 100644 --- a/docker-compose-standalone.yml +++ b/docker-compose-standalone.yml @@ -4,7 +4,7 @@ version: '3.7' services: opensearch: - image: ghcr.io/idaholab/malcolm/opensearch:23.05.0 + image: ghcr.io/idaholab/malcolm/opensearch:23.05.1 restart: "no" stdin_open: false tty: true @@ -37,7 +37,7 @@ services: retries: 3 start_period: 180s dashboards-helper: - image: ghcr.io/idaholab/malcolm/dashboards-helper:23.05.0 + image: ghcr.io/idaholab/malcolm/dashboards-helper:23.05.1 restart: "no" stdin_open: false tty: true @@ -64,7 +64,7 @@ services: retries: 3 start_period: 30s dashboards: - image: ghcr.io/idaholab/malcolm/dashboards:23.05.0 + image: ghcr.io/idaholab/malcolm/dashboards:23.05.1 restart: "no" stdin_open: false tty: true @@ -90,7 +90,7 @@ services: retries: 3 start_period: 210s logstash: - image: ghcr.io/idaholab/malcolm/logstash-oss:23.05.0 + image: ghcr.io/idaholab/malcolm/logstash-oss:23.05.1 restart: "no" stdin_open: false tty: true @@ -132,7 +132,7 @@ services: retries: 3 start_period: 600s filebeat: - image: ghcr.io/idaholab/malcolm/filebeat-oss:23.05.0 + image: ghcr.io/idaholab/malcolm/filebeat-oss:23.05.1 restart: "no" stdin_open: false tty: true @@ -167,7 +167,7 @@ services: retries: 3 start_period: 60s arkime: - image: ghcr.io/idaholab/malcolm/arkime:23.05.0 + image: ghcr.io/idaholab/malcolm/arkime:23.05.1 restart: "no" stdin_open: false tty: true @@ -203,7 +203,7 @@ services: retries: 3 start_period: 210s zeek: - image: ghcr.io/idaholab/malcolm/zeek:23.05.0 + image: ghcr.io/idaholab/malcolm/zeek:23.05.1 restart: "no" stdin_open: false tty: true @@ -241,7 +241,7 @@ services: retries: 3 start_period: 60s zeek-live: - image: ghcr.io/idaholab/malcolm/zeek:23.05.0 + image: ghcr.io/idaholab/malcolm/zeek:23.05.1 restart: "no" stdin_open: false tty: true @@ -269,7 +269,7 @@ services: - ./zeek-logs/extract_files:/zeek/extract_files - ./zeek/intel:/opt/zeek/share/zeek/site/intel suricata: - image: ghcr.io/idaholab/malcolm/suricata:23.05.0 + image: ghcr.io/idaholab/malcolm/suricata:23.05.1 restart: "no" stdin_open: false tty: true @@ -305,7 +305,7 @@ services: retries: 3 start_period: 120s suricata-live: - image: ghcr.io/idaholab/malcolm/suricata:23.05.0 + image: ghcr.io/idaholab/malcolm/suricata:23.05.1 restart: "no" stdin_open: false tty: true @@ -331,7 +331,7 @@ services: - ./suricata-logs:/var/log/suricata - ./suricata/rules:/opt/suricata/rules:ro file-monitor: - image: ghcr.io/idaholab/malcolm/file-monitor:23.05.0 + image: ghcr.io/idaholab/malcolm/file-monitor:23.05.1 restart: "no" stdin_open: false tty: true @@ -357,7 +357,7 @@ services: retries: 3 start_period: 60s pcap-capture: - image: ghcr.io/idaholab/malcolm/pcap-capture:23.05.0 + image: ghcr.io/idaholab/malcolm/pcap-capture:23.05.1 restart: "no" stdin_open: false tty: true @@ -379,7 +379,7 @@ services: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./pcap/upload:/pcap pcap-monitor: - image: ghcr.io/idaholab/malcolm/pcap-monitor:23.05.0 + image: ghcr.io/idaholab/malcolm/pcap-monitor:23.05.1 restart: "no" stdin_open: false tty: true @@ -405,7 +405,7 @@ services: retries: 3 start_period: 90s upload: - image: ghcr.io/idaholab/malcolm/file-upload:23.05.0 + image: ghcr.io/idaholab/malcolm/file-upload:23.05.1 restart: "no" stdin_open: false tty: true @@ -433,7 +433,7 @@ services: retries: 3 start_period: 60s htadmin: - image: ghcr.io/idaholab/malcolm/htadmin:23.05.0 + image: ghcr.io/idaholab/malcolm/htadmin:23.05.1 restart: "no" stdin_open: false tty: true @@ -458,7 +458,7 @@ services: retries: 3 start_period: 60s freq: - image: ghcr.io/idaholab/malcolm/freq:23.05.0 + image: ghcr.io/idaholab/malcolm/freq:23.05.1 restart: "no" stdin_open: false tty: true @@ -480,7 +480,7 @@ services: retries: 3 start_period: 60s netbox: - image: ghcr.io/idaholab/malcolm/netbox:23.05.0 + image: ghcr.io/idaholab/malcolm/netbox:23.05.1 restart: "no" stdin_open: false tty: true @@ -513,7 +513,7 @@ services: retries: 3 start_period: 120s netbox-postgres: - image: ghcr.io/idaholab/malcolm/postgresql:23.05.0 + image: ghcr.io/idaholab/malcolm/postgresql:23.05.1 restart: "no" stdin_open: false tty: true @@ -537,7 +537,7 @@ services: retries: 3 start_period: 45s netbox-redis: - image: ghcr.io/idaholab/malcolm/redis:23.05.0 + image: ghcr.io/idaholab/malcolm/redis:23.05.1 restart: "no" stdin_open: false tty: true @@ -565,7 +565,7 @@ services: retries: 3 start_period: 45s netbox-redis-cache: - image: ghcr.io/idaholab/malcolm/redis:23.05.0 + image: ghcr.io/idaholab/malcolm/redis:23.05.1 restart: "no" stdin_open: false tty: true @@ -592,7 +592,7 @@ services: retries: 3 start_period: 45s api: - image: ghcr.io/idaholab/malcolm/api:23.05.0 + image: ghcr.io/idaholab/malcolm/api:23.05.1 command: gunicorn --bind 0:5000 manage:app restart: "no" stdin_open: false @@ -616,7 +616,7 @@ services: retries: 3 start_period: 60s nginx-proxy: - image: ghcr.io/idaholab/malcolm/nginx-proxy:23.05.0 + image: ghcr.io/idaholab/malcolm/nginx-proxy:23.05.1 restart: "no" stdin_open: false tty: true diff --git a/docker-compose.yml b/docker-compose.yml index e2a7bbed1..1acbbe5f9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ services: build: context: . dockerfile: Dockerfiles/opensearch.Dockerfile - image: ghcr.io/idaholab/malcolm/opensearch:23.05.0 + image: ghcr.io/idaholab/malcolm/opensearch:23.05.1 restart: "no" stdin_open: false tty: true @@ -43,7 +43,7 @@ services: build: context: . dockerfile: Dockerfiles/dashboards-helper.Dockerfile - image: ghcr.io/idaholab/malcolm/dashboards-helper:23.05.0 + image: ghcr.io/idaholab/malcolm/dashboards-helper:23.05.1 restart: "no" stdin_open: false tty: true @@ -73,7 +73,7 @@ services: build: context: . dockerfile: Dockerfiles/dashboards.Dockerfile - image: ghcr.io/idaholab/malcolm/dashboards:23.05.0 + image: ghcr.io/idaholab/malcolm/dashboards:23.05.1 restart: "no" stdin_open: false tty: true @@ -102,7 +102,7 @@ services: build: context: . dockerfile: Dockerfiles/logstash.Dockerfile - image: ghcr.io/idaholab/malcolm/logstash-oss:23.05.0 + image: ghcr.io/idaholab/malcolm/logstash-oss:23.05.1 restart: "no" stdin_open: false tty: true @@ -151,7 +151,7 @@ services: build: context: . dockerfile: Dockerfiles/filebeat.Dockerfile - image: ghcr.io/idaholab/malcolm/filebeat-oss:23.05.0 + image: ghcr.io/idaholab/malcolm/filebeat-oss:23.05.1 restart: "no" stdin_open: false tty: true @@ -189,7 +189,7 @@ services: build: context: . dockerfile: Dockerfiles/arkime.Dockerfile - image: ghcr.io/idaholab/malcolm/arkime:23.05.0 + image: ghcr.io/idaholab/malcolm/arkime:23.05.1 restart: "no" stdin_open: false tty: true @@ -231,7 +231,7 @@ services: build: context: . dockerfile: Dockerfiles/zeek.Dockerfile - image: ghcr.io/idaholab/malcolm/zeek:23.05.0 + image: ghcr.io/idaholab/malcolm/zeek:23.05.1 restart: "no" stdin_open: false tty: true @@ -273,7 +273,7 @@ services: build: context: . dockerfile: Dockerfiles/zeek.Dockerfile - image: ghcr.io/idaholab/malcolm/zeek:23.05.0 + image: ghcr.io/idaholab/malcolm/zeek:23.05.1 restart: "no" stdin_open: false tty: true @@ -305,7 +305,7 @@ services: build: context: . dockerfile: Dockerfiles/suricata.Dockerfile - image: ghcr.io/idaholab/malcolm/suricata:23.05.0 + image: ghcr.io/idaholab/malcolm/suricata:23.05.1 restart: "no" stdin_open: false tty: true @@ -344,7 +344,7 @@ services: build: context: . dockerfile: Dockerfiles/suricata.Dockerfile - image: ghcr.io/idaholab/malcolm/suricata:23.05.0 + image: ghcr.io/idaholab/malcolm/suricata:23.05.1 restart: "no" stdin_open: false tty: true @@ -373,7 +373,7 @@ services: build: context: . dockerfile: Dockerfiles/file-monitor.Dockerfile - image: ghcr.io/idaholab/malcolm/file-monitor:23.05.0 + image: ghcr.io/idaholab/malcolm/file-monitor:23.05.1 restart: "no" stdin_open: false tty: true @@ -402,7 +402,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-capture.Dockerfile - image: ghcr.io/idaholab/malcolm/pcap-capture:23.05.0 + image: ghcr.io/idaholab/malcolm/pcap-capture:23.05.1 restart: "no" stdin_open: false tty: true @@ -427,7 +427,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-monitor.Dockerfile - image: ghcr.io/idaholab/malcolm/pcap-monitor:23.05.0 + image: ghcr.io/idaholab/malcolm/pcap-monitor:23.05.1 restart: "no" stdin_open: false tty: true @@ -456,7 +456,7 @@ services: build: context: . dockerfile: Dockerfiles/file-upload.Dockerfile - image: ghcr.io/idaholab/malcolm/file-upload:23.05.0 + image: ghcr.io/idaholab/malcolm/file-upload:23.05.1 restart: "no" stdin_open: false tty: true @@ -484,7 +484,7 @@ services: retries: 3 start_period: 60s htadmin: - image: ghcr.io/idaholab/malcolm/htadmin:23.05.0 + image: ghcr.io/idaholab/malcolm/htadmin:23.05.1 build: context: . dockerfile: Dockerfiles/htadmin.Dockerfile @@ -512,7 +512,7 @@ services: retries: 3 start_period: 60s freq: - image: ghcr.io/idaholab/malcolm/freq:23.05.0 + image: ghcr.io/idaholab/malcolm/freq:23.05.1 build: context: . dockerfile: Dockerfiles/freq.Dockerfile @@ -537,7 +537,7 @@ services: retries: 3 start_period: 60s netbox: - image: ghcr.io/idaholab/malcolm/netbox:23.05.0 + image: ghcr.io/idaholab/malcolm/netbox:23.05.1 build: context: . dockerfile: Dockerfiles/netbox.Dockerfile @@ -574,7 +574,7 @@ services: retries: 3 start_period: 120s netbox-postgres: - image: ghcr.io/idaholab/malcolm/postgresql:23.05.0 + image: ghcr.io/idaholab/malcolm/postgresql:23.05.1 build: context: . dockerfile: Dockerfiles/postgresql.Dockerfile @@ -601,7 +601,7 @@ services: retries: 3 start_period: 45s netbox-redis: - image: ghcr.io/idaholab/malcolm/redis:23.05.0 + image: ghcr.io/idaholab/malcolm/redis:23.05.1 build: context: . dockerfile: Dockerfiles/redis.Dockerfile @@ -632,7 +632,7 @@ services: retries: 3 start_period: 45s netbox-redis-cache: - image: ghcr.io/idaholab/malcolm/redis:23.05.0 + image: ghcr.io/idaholab/malcolm/redis:23.05.1 build: context: . dockerfile: Dockerfiles/redis.Dockerfile @@ -662,7 +662,7 @@ services: retries: 3 start_period: 45s api: - image: ghcr.io/idaholab/malcolm/api:23.05.0 + image: ghcr.io/idaholab/malcolm/api:23.05.1 build: context: . dockerfile: Dockerfiles/api.Dockerfile @@ -692,7 +692,7 @@ services: build: context: . dockerfile: Dockerfiles/nginx.Dockerfile - image: ghcr.io/idaholab/malcolm/nginx-proxy:23.05.0 + image: ghcr.io/idaholab/malcolm/nginx-proxy:23.05.1 restart: "no" stdin_open: false tty: true diff --git a/docs/download.md b/docs/download.md index a9c6b154f..daf9fb7d0 100644 --- a/docs/download.md +++ b/docs/download.md @@ -16,7 +16,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | ISO | SHA256 | |---|---| -| [malcolm-23.05.0.iso](/iso/malcolm-23.05.0.iso) (5.3GiB) | [`e9e00694f25b9d0dcc286496490e184930611ddbed6c52dfab77a935d2afa850`](/iso/malcolm-23.05.0.iso.sha256.txt) | +| [malcolm-23.05.1.iso](/iso/malcolm-23.05.1.iso) (5.3GiB) | [`e9e00694f25b9d0dcc286496490e184930611ddbed6c52dfab77a935d2afa850`](/iso/malcolm-23.05.1.iso.sha256.txt) | ## Hedgehog Linux @@ -26,7 +26,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | ISO | SHA256 | |---|---| -| [hedgehog-23.05.0.iso](/iso/hedgehog-23.05.0.iso) (2.3GiB) | [`f850ecd3b62731b46ac0366bdcdd62437da30220c23f94013873c6c92cbddff7`](/iso/hedgehog-23.05.0.iso.sha256.txt) | +| [hedgehog-23.05.1.iso](/iso/hedgehog-23.05.1.iso) (2.3GiB) | [`f850ecd3b62731b46ac0366bdcdd62437da30220c23f94013873c6c92cbddff7`](/iso/hedgehog-23.05.1.iso.sha256.txt) | ## Warning diff --git a/docs/hedgehog-iso-build.md b/docs/hedgehog-iso-build.md index dcf35e219..e13bb5ffb 100644 --- a/docs/hedgehog-iso-build.md +++ b/docs/hedgehog-iso-build.md @@ -29,7 +29,7 @@ Building the ISO may take 90 minutes or more depending on your system. As the bu ``` … -Finished, created "/sensor-build/hedgehog-23.05.0.iso" +Finished, created "/sensor-build/hedgehog-23.05.1.iso" … ``` diff --git a/docs/kubernetes.md b/docs/kubernetes.md index e743c10b5..8ceaa0868 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -261,28 +261,28 @@ agent2 | agent2 | 192.168.56.12 | agent2 | k3s | 6000m | agent1 | agent1 | 192.168.56.11 | agent1 | k3s | 6000m | 861.34m | 14.36% | 19.55Gi | 9.29Gi | 61.28Gi | 11 | Pod Name | State | Pod IP | Pod Kind | Worker Node | CPU Usage | Memory Usage | Container Name:Restarts | Container Image | -api-deployment-6f4686cf59-bn286 | Running | 10.42.2.14 | ReplicaSet | agent1 | 0.11m | 59.62Mi | api-container:0 | api:23.05.0 | -file-monitor-deployment-855646bd75-vk7st | Running | 10.42.2.16 | ReplicaSet | agent1 | 8.47m | 1.46Gi | file-monitor-container:0 | file-monitor:23.05.0 | -zeek-live-deployment-64b69d4b6f-947vr | Running | 10.42.2.17 | ReplicaSet | agent1 | 0.02m | 12.44Mi | zeek-live-container:0 | zeek:23.05.0 | -dashboards-helper-deployment-69dc54f6b6-ln4sq | Running | 10.42.2.15 | ReplicaSet | agent1 | 10.77m | 38.43Mi | dashboards-helper-container:0 | dashboards-helper:23.05.0 | -upload-deployment-586568844b-4jnk9 | Running | 10.42.2.18 | ReplicaSet | agent1 | 0.15m | 29.78Mi | upload-container:0 | file-upload:23.05.0 | -filebeat-deployment-6ff8bc444f-t7h49 | Running | 10.42.2.20 | ReplicaSet | agent1 | 2.84m | 70.71Mi | filebeat-container:0 | filebeat-oss:23.05.0 | -zeek-offline-deployment-844f4865bd-g2sdm | Running | 10.42.2.21 | ReplicaSet | agent1 | 0.17m | 41.92Mi | zeek-offline-container:0 | zeek:23.05.0 | -logstash-deployment-6fbc9fdcd5-hwx8s | Running | 10.42.2.22 | ReplicaSet | agent1 | 85.55m | 2.91Gi | logstash-container:0 | logstash-oss:23.05.0 | -netbox-deployment-cdcff4977-hbbw5 | Running | 10.42.2.23 | ReplicaSet | agent1 | 807.64m | 702.86Mi | netbox-container:0 | netbox:23.05.0 | -suricata-offline-deployment-6ccdb89478-z5696 | Running | 10.42.2.19 | ReplicaSet | agent1 | 0.22m | 34.88Mi | suricata-offline-container:0 | suricata:23.05.0 | -dashboards-deployment-69b5465db-vz88g | Running | 10.42.1.14 | ReplicaSet | agent2 | 0.94m | 100.12Mi | dashboards-container:0 | dashboards:23.05.0 | -netbox-redis-cache-deployment-5f77d47b8b-z7t2z | Running | 10.42.1.15 | ReplicaSet | agent2 | 3.57m | 7.36Mi | netbox-redis-cache-container:0 | redis:23.05.0 | -suricata-live-deployment-6494c77759-9rlnt | Running | 10.42.1.16 | ReplicaSet | agent2 | 0.02m | 9.69Mi | suricata-live-container:0 | suricata:23.05.0 | -freq-deployment-cfd84fd97-dnngf | Running | 10.42.1.17 | ReplicaSet | agent2 | 0.2m | 26.36Mi | freq-container:0 | freq:23.05.0 | -arkime-deployment-56999cdd66-s98pp | Running | 10.42.1.18 | ReplicaSet | agent2 | 4.15m | 113.07Mi | arkime-container:0 | arkime:23.05.0 | -pcap-monitor-deployment-594ff674c4-fsm7m | Running | 10.42.1.19 | ReplicaSet | agent2 | 1.24m | 48.44Mi | pcap-monitor-container:0 | pcap-monitor:23.05.0 | -pcap-capture-deployment-7c8bf6957-jzpzn | Running | 10.42.1.20 | ReplicaSet | agent2 | 0.02m | 9.64Mi | pcap-capture-container:0 | pcap-capture:23.05.0 | -netbox-postgres-deployment-5879b8dffc-kkt56 | Running | 10.42.1.21 | ReplicaSet | agent2 | 70.91m | 33.02Mi | netbox-postgres-container:0 | postgresql:23.05.0 | -htadmin-deployment-6fc46888b9-sq6ln | Running | 10.42.1.23 | ReplicaSet | agent2 | 0.14m | 30.53Mi | htadmin-container:0 | htadmin:23.05.0 | -netbox-redis-deployment-5bcd8f6c96-j5xpf | Running | 10.42.1.24 | ReplicaSet | agent2 | 1.46m | 7.34Mi | netbox-redis-container:0 | redis:23.05.0 | -nginx-proxy-deployment-69fcc4968d-f68tq | Running | 10.42.1.22 | ReplicaSet | agent2 | 0.31m | 22.63Mi | nginx-proxy-container:0 | nginx-proxy:23.05.0 | -opensearch-deployment-75498799f6-4zmwd | Running | 10.42.1.25 | ReplicaSet | agent2 | 89.8m | 11.03Gi | opensearch-container:0 | opensearch:23.05.0 | +api-deployment-6f4686cf59-bn286 | Running | 10.42.2.14 | ReplicaSet | agent1 | 0.11m | 59.62Mi | api-container:0 | api:23.05.1 | +file-monitor-deployment-855646bd75-vk7st | Running | 10.42.2.16 | ReplicaSet | agent1 | 8.47m | 1.46Gi | file-monitor-container:0 | file-monitor:23.05.1 | +zeek-live-deployment-64b69d4b6f-947vr | Running | 10.42.2.17 | ReplicaSet | agent1 | 0.02m | 12.44Mi | zeek-live-container:0 | zeek:23.05.1 | +dashboards-helper-deployment-69dc54f6b6-ln4sq | Running | 10.42.2.15 | ReplicaSet | agent1 | 10.77m | 38.43Mi | dashboards-helper-container:0 | dashboards-helper:23.05.1 | +upload-deployment-586568844b-4jnk9 | Running | 10.42.2.18 | ReplicaSet | agent1 | 0.15m | 29.78Mi | upload-container:0 | file-upload:23.05.1 | +filebeat-deployment-6ff8bc444f-t7h49 | Running | 10.42.2.20 | ReplicaSet | agent1 | 2.84m | 70.71Mi | filebeat-container:0 | filebeat-oss:23.05.1 | +zeek-offline-deployment-844f4865bd-g2sdm | Running | 10.42.2.21 | ReplicaSet | agent1 | 0.17m | 41.92Mi | zeek-offline-container:0 | zeek:23.05.1 | +logstash-deployment-6fbc9fdcd5-hwx8s | Running | 10.42.2.22 | ReplicaSet | agent1 | 85.55m | 2.91Gi | logstash-container:0 | logstash-oss:23.05.1 | +netbox-deployment-cdcff4977-hbbw5 | Running | 10.42.2.23 | ReplicaSet | agent1 | 807.64m | 702.86Mi | netbox-container:0 | netbox:23.05.1 | +suricata-offline-deployment-6ccdb89478-z5696 | Running | 10.42.2.19 | ReplicaSet | agent1 | 0.22m | 34.88Mi | suricata-offline-container:0 | suricata:23.05.1 | +dashboards-deployment-69b5465db-vz88g | Running | 10.42.1.14 | ReplicaSet | agent2 | 0.94m | 100.12Mi | dashboards-container:0 | dashboards:23.05.1 | +netbox-redis-cache-deployment-5f77d47b8b-z7t2z | Running | 10.42.1.15 | ReplicaSet | agent2 | 3.57m | 7.36Mi | netbox-redis-cache-container:0 | redis:23.05.1 | +suricata-live-deployment-6494c77759-9rlnt | Running | 10.42.1.16 | ReplicaSet | agent2 | 0.02m | 9.69Mi | suricata-live-container:0 | suricata:23.05.1 | +freq-deployment-cfd84fd97-dnngf | Running | 10.42.1.17 | ReplicaSet | agent2 | 0.2m | 26.36Mi | freq-container:0 | freq:23.05.1 | +arkime-deployment-56999cdd66-s98pp | Running | 10.42.1.18 | ReplicaSet | agent2 | 4.15m | 113.07Mi | arkime-container:0 | arkime:23.05.1 | +pcap-monitor-deployment-594ff674c4-fsm7m | Running | 10.42.1.19 | ReplicaSet | agent2 | 1.24m | 48.44Mi | pcap-monitor-container:0 | pcap-monitor:23.05.1 | +pcap-capture-deployment-7c8bf6957-jzpzn | Running | 10.42.1.20 | ReplicaSet | agent2 | 0.02m | 9.64Mi | pcap-capture-container:0 | pcap-capture:23.05.1 | +netbox-postgres-deployment-5879b8dffc-kkt56 | Running | 10.42.1.21 | ReplicaSet | agent2 | 70.91m | 33.02Mi | netbox-postgres-container:0 | postgresql:23.05.1 | +htadmin-deployment-6fc46888b9-sq6ln | Running | 10.42.1.23 | ReplicaSet | agent2 | 0.14m | 30.53Mi | htadmin-container:0 | htadmin:23.05.1 | +netbox-redis-deployment-5bcd8f6c96-j5xpf | Running | 10.42.1.24 | ReplicaSet | agent2 | 1.46m | 7.34Mi | netbox-redis-container:0 | redis:23.05.1 | +nginx-proxy-deployment-69fcc4968d-f68tq | Running | 10.42.1.22 | ReplicaSet | agent2 | 0.31m | 22.63Mi | nginx-proxy-container:0 | nginx-proxy:23.05.1 | +opensearch-deployment-75498799f6-4zmwd | Running | 10.42.1.25 | ReplicaSet | agent2 | 89.8m | 11.03Gi | opensearch-container:0 | opensearch:23.05.1 | ``` The other control scripts (`stop`, `restart`, `logs`, etc.) work in a similar manner as in a Docker-based deployment. One notable difference is the `wipe` script: data on PersistentVolume storage cannot be deleted by `wipe`. It must be deleted manually on the storage media underlying the PersistentVolumes. @@ -536,28 +536,28 @@ agent1 | agent1 | 192.168.56.11 | agent1 | k3s | 6000m | agent2 | agent2 | 192.168.56.12 | agent2 | k3s | 6000m | 552.71m | 9.21% | 19.55Gi | 13.27Gi | 61.28Gi | 12 | Pod Name | State | Pod IP | Pod Kind | Worker Node | CPU Usage | Memory Usage | Container Name:Restarts | Container Image | -netbox-redis-cache-deployment-5f77d47b8b-jr9nt | Running | 10.42.2.6 | ReplicaSet | agent2 | 1.89m | 7.24Mi | netbox-redis-cache-container:0 | redis:23.05.0 | -netbox-redis-deployment-5bcd8f6c96-bkzmh | Running | 10.42.2.5 | ReplicaSet | agent2 | 1.62m | 7.52Mi | netbox-redis-container:0 | redis:23.05.0 | -dashboards-helper-deployment-69dc54f6b6-ks7ps | Running | 10.42.2.4 | ReplicaSet | agent2 | 12.95m | 40.75Mi | dashboards-helper-container:0 | dashboards-helper:23.05.0 | -freq-deployment-cfd84fd97-5bwp6 | Running | 10.42.2.8 | ReplicaSet | agent2 | 0.11m | 26.33Mi | freq-container:0 | freq:23.05.0 | -pcap-capture-deployment-7c8bf6957-hkvkn | Running | 10.42.2.12 | ReplicaSet | agent2 | 0.02m | 9.21Mi | pcap-capture-container:0 | pcap-capture:23.05.0 | -nginx-proxy-deployment-69fcc4968d-m57rz | Running | 10.42.2.10 | ReplicaSet | agent2 | 0.91m | 22.72Mi | nginx-proxy-container:0 | nginx-proxy:23.05.0 | -htadmin-deployment-6fc46888b9-vpt7l | Running | 10.42.2.7 | ReplicaSet | agent2 | 0.16m | 30.21Mi | htadmin-container:0 | htadmin:23.05.0 | -opensearch-deployment-75498799f6-5v92w | Running | 10.42.2.13 | ReplicaSet | agent2 | 139.2m | 10.86Gi | opensearch-container:0 | opensearch:23.05.0 | -zeek-live-deployment-64b69d4b6f-fcb6n | Running | 10.42.2.9 | ReplicaSet | agent2 | 0.02m | 109.55Mi | zeek-live-container:0 | zeek:23.05.0 | -dashboards-deployment-69b5465db-kgsqk | Running | 10.42.2.3 | ReplicaSet | agent2 | 14.98m | 108.85Mi | dashboards-container:0 | dashboards:23.05.0 | -arkime-deployment-56999cdd66-xxpw9 | Running | 10.42.2.11 | ReplicaSet | agent2 | 208.95m | 78.42Mi | arkime-container:0 | arkime:23.05.0 | -api-deployment-6f4686cf59-xt9md | Running | 10.42.1.3 | ReplicaSet | agent1 | 0.14m | 56.88Mi | api-container:0 | api:23.05.0 | -netbox-postgres-deployment-5879b8dffc-lb4qm | Running | 10.42.1.6 | ReplicaSet | agent1 | 141.2m | 48.02Mi | netbox-postgres-container:0 | postgresql:23.05.0 | -pcap-monitor-deployment-594ff674c4-fwq7g | Running | 10.42.1.12 | ReplicaSet | agent1 | 3.93m | 46.44Mi | pcap-monitor-container:0 | pcap-monitor:23.05.0 | -suricata-offline-deployment-6ccdb89478-j5fgj | Running | 10.42.1.10 | ReplicaSet | agent1 | 10.42m | 35.12Mi | suricata-offline-container:0 | suricata:23.05.0 | -suricata-live-deployment-6494c77759-rpt48 | Running | 10.42.1.8 | ReplicaSet | agent1 | 0.01m | 9.62Mi | suricata-live-container:0 | suricata:23.05.0 | -netbox-deployment-cdcff4977-7ns2q | Running | 10.42.1.7 | ReplicaSet | agent1 | 830.47m | 530.7Mi | netbox-container:0 | netbox:23.05.0 | -zeek-offline-deployment-844f4865bd-7x68b | Running | 10.42.1.9 | ReplicaSet | agent1 | 1.44m | 43.66Mi | zeek-offline-container:0 | zeek:23.05.0 | -filebeat-deployment-6ff8bc444f-pdgzj | Running | 10.42.1.11 | ReplicaSet | agent1 | 0.78m | 75.25Mi | filebeat-container:0 | filebeat-oss:23.05.0 | -file-monitor-deployment-855646bd75-nbngq | Running | 10.42.1.4 | ReplicaSet | agent1 | 1.69m | 1.46Gi | file-monitor-container:0 | file-monitor:23.05.0 | -upload-deployment-586568844b-9s7f5 | Running | 10.42.1.13 | ReplicaSet | agent1 | 0.14m | 29.62Mi | upload-container:0 | file-upload:23.05.0 | -logstash-deployment-6fbc9fdcd5-2hhx8 | Running | 10.42.1.5 | ReplicaSet | agent1 | 3236.29m | 357.36Mi | logstash-container:0 | logstash-oss:23.05.0 | +netbox-redis-cache-deployment-5f77d47b8b-jr9nt | Running | 10.42.2.6 | ReplicaSet | agent2 | 1.89m | 7.24Mi | netbox-redis-cache-container:0 | redis:23.05.1 | +netbox-redis-deployment-5bcd8f6c96-bkzmh | Running | 10.42.2.5 | ReplicaSet | agent2 | 1.62m | 7.52Mi | netbox-redis-container:0 | redis:23.05.1 | +dashboards-helper-deployment-69dc54f6b6-ks7ps | Running | 10.42.2.4 | ReplicaSet | agent2 | 12.95m | 40.75Mi | dashboards-helper-container:0 | dashboards-helper:23.05.1 | +freq-deployment-cfd84fd97-5bwp6 | Running | 10.42.2.8 | ReplicaSet | agent2 | 0.11m | 26.33Mi | freq-container:0 | freq:23.05.1 | +pcap-capture-deployment-7c8bf6957-hkvkn | Running | 10.42.2.12 | ReplicaSet | agent2 | 0.02m | 9.21Mi | pcap-capture-container:0 | pcap-capture:23.05.1 | +nginx-proxy-deployment-69fcc4968d-m57rz | Running | 10.42.2.10 | ReplicaSet | agent2 | 0.91m | 22.72Mi | nginx-proxy-container:0 | nginx-proxy:23.05.1 | +htadmin-deployment-6fc46888b9-vpt7l | Running | 10.42.2.7 | ReplicaSet | agent2 | 0.16m | 30.21Mi | htadmin-container:0 | htadmin:23.05.1 | +opensearch-deployment-75498799f6-5v92w | Running | 10.42.2.13 | ReplicaSet | agent2 | 139.2m | 10.86Gi | opensearch-container:0 | opensearch:23.05.1 | +zeek-live-deployment-64b69d4b6f-fcb6n | Running | 10.42.2.9 | ReplicaSet | agent2 | 0.02m | 109.55Mi | zeek-live-container:0 | zeek:23.05.1 | +dashboards-deployment-69b5465db-kgsqk | Running | 10.42.2.3 | ReplicaSet | agent2 | 14.98m | 108.85Mi | dashboards-container:0 | dashboards:23.05.1 | +arkime-deployment-56999cdd66-xxpw9 | Running | 10.42.2.11 | ReplicaSet | agent2 | 208.95m | 78.42Mi | arkime-container:0 | arkime:23.05.1 | +api-deployment-6f4686cf59-xt9md | Running | 10.42.1.3 | ReplicaSet | agent1 | 0.14m | 56.88Mi | api-container:0 | api:23.05.1 | +netbox-postgres-deployment-5879b8dffc-lb4qm | Running | 10.42.1.6 | ReplicaSet | agent1 | 141.2m | 48.02Mi | netbox-postgres-container:0 | postgresql:23.05.1 | +pcap-monitor-deployment-594ff674c4-fwq7g | Running | 10.42.1.12 | ReplicaSet | agent1 | 3.93m | 46.44Mi | pcap-monitor-container:0 | pcap-monitor:23.05.1 | +suricata-offline-deployment-6ccdb89478-j5fgj | Running | 10.42.1.10 | ReplicaSet | agent1 | 10.42m | 35.12Mi | suricata-offline-container:0 | suricata:23.05.1 | +suricata-live-deployment-6494c77759-rpt48 | Running | 10.42.1.8 | ReplicaSet | agent1 | 0.01m | 9.62Mi | suricata-live-container:0 | suricata:23.05.1 | +netbox-deployment-cdcff4977-7ns2q | Running | 10.42.1.7 | ReplicaSet | agent1 | 830.47m | 530.7Mi | netbox-container:0 | netbox:23.05.1 | +zeek-offline-deployment-844f4865bd-7x68b | Running | 10.42.1.9 | ReplicaSet | agent1 | 1.44m | 43.66Mi | zeek-offline-container:0 | zeek:23.05.1 | +filebeat-deployment-6ff8bc444f-pdgzj | Running | 10.42.1.11 | ReplicaSet | agent1 | 0.78m | 75.25Mi | filebeat-container:0 | filebeat-oss:23.05.1 | +file-monitor-deployment-855646bd75-nbngq | Running | 10.42.1.4 | ReplicaSet | agent1 | 1.69m | 1.46Gi | file-monitor-container:0 | file-monitor:23.05.1 | +upload-deployment-586568844b-9s7f5 | Running | 10.42.1.13 | ReplicaSet | agent1 | 0.14m | 29.62Mi | upload-container:0 | file-upload:23.05.1 | +logstash-deployment-6fbc9fdcd5-2hhx8 | Running | 10.42.1.5 | ReplicaSet | agent1 | 3236.29m | 357.36Mi | logstash-container:0 | logstash-oss:23.05.1 | ``` View container logs for the Malcolm deployment with `./scripts/logs` (if **[stern](https://github.com/stern/stern)** present in `$PATH`): diff --git a/docs/malcolm-iso.md b/docs/malcolm-iso.md index f2946d534..7b0f0a39a 100644 --- a/docs/malcolm-iso.md +++ b/docs/malcolm-iso.md @@ -41,7 +41,7 @@ Building the ISO may take 30 minutes or more depending on your system. As the bu ``` … -Finished, created "/malcolm-build/malcolm-iso/malcolm-23.05.0.iso" +Finished, created "/malcolm-build/malcolm-iso/malcolm-23.05.1.iso" … ``` diff --git a/docs/quickstart.md b/docs/quickstart.md index 40e5d49dd..f721f9f5e 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -54,25 +54,25 @@ You can then observe that the images have been retrieved by running `docker imag ``` $ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -ghcr.io/idaholab/malcolm/api 23.05.0 xxxxxxxxxxxx 3 days ago 158MB -ghcr.io/idaholab/malcolm/arkime 23.05.0 xxxxxxxxxxxx 3 days ago 816MB -ghcr.io/idaholab/malcolm/dashboards 23.05.0 xxxxxxxxxxxx 3 days ago 1.02GB -ghcr.io/idaholab/malcolm/dashboards-helper 23.05.0 xxxxxxxxxxxx 3 days ago 184MB -ghcr.io/idaholab/malcolm/file-monitor 23.05.0 xxxxxxxxxxxx 3 days ago 588MB -ghcr.io/idaholab/malcolm/file-upload 23.05.0 xxxxxxxxxxxx 3 days ago 259MB -ghcr.io/idaholab/malcolm/filebeat-oss 23.05.0 xxxxxxxxxxxx 3 days ago 624MB -ghcr.io/idaholab/malcolm/freq 23.05.0 xxxxxxxxxxxx 3 days ago 132MB -ghcr.io/idaholab/malcolm/htadmin 23.05.0 xxxxxxxxxxxx 3 days ago 242MB -ghcr.io/idaholab/malcolm/logstash-oss 23.05.0 xxxxxxxxxxxx 3 days ago 1.35GB -ghcr.io/idaholab/malcolm/netbox 23.05.0 xxxxxxxxxxxx 3 days ago 1.01GB -ghcr.io/idaholab/malcolm/nginx-proxy 23.05.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/opensearch 23.05.0 xxxxxxxxxxxx 3 days ago 1.17GB -ghcr.io/idaholab/malcolm/pcap-capture 23.05.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/pcap-monitor 23.05.0 xxxxxxxxxxxx 3 days ago 213MB -ghcr.io/idaholab/malcolm/postgresql 23.05.0 xxxxxxxxxxxx 3 days ago 268MB -ghcr.io/idaholab/malcolm/redis 23.05.0 xxxxxxxxxxxx 3 days ago 34.2MB -ghcr.io/idaholab/malcolm/suricata 23.05.0 xxxxxxxxxxxx 3 days ago 278MB -ghcr.io/idaholab/malcolm/zeek 23.05.0 xxxxxxxxxxxx 3 days ago 1GB +ghcr.io/idaholab/malcolm/api 23.05.1 xxxxxxxxxxxx 3 days ago 158MB +ghcr.io/idaholab/malcolm/arkime 23.05.1 xxxxxxxxxxxx 3 days ago 816MB +ghcr.io/idaholab/malcolm/dashboards 23.05.1 xxxxxxxxxxxx 3 days ago 1.02GB +ghcr.io/idaholab/malcolm/dashboards-helper 23.05.1 xxxxxxxxxxxx 3 days ago 184MB +ghcr.io/idaholab/malcolm/file-monitor 23.05.1 xxxxxxxxxxxx 3 days ago 588MB +ghcr.io/idaholab/malcolm/file-upload 23.05.1 xxxxxxxxxxxx 3 days ago 259MB +ghcr.io/idaholab/malcolm/filebeat-oss 23.05.1 xxxxxxxxxxxx 3 days ago 624MB +ghcr.io/idaholab/malcolm/freq 23.05.1 xxxxxxxxxxxx 3 days ago 132MB +ghcr.io/idaholab/malcolm/htadmin 23.05.1 xxxxxxxxxxxx 3 days ago 242MB +ghcr.io/idaholab/malcolm/logstash-oss 23.05.1 xxxxxxxxxxxx 3 days ago 1.35GB +ghcr.io/idaholab/malcolm/netbox 23.05.1 xxxxxxxxxxxx 3 days ago 1.01GB +ghcr.io/idaholab/malcolm/nginx-proxy 23.05.1 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/opensearch 23.05.1 xxxxxxxxxxxx 3 days ago 1.17GB +ghcr.io/idaholab/malcolm/pcap-capture 23.05.1 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/pcap-monitor 23.05.1 xxxxxxxxxxxx 3 days ago 213MB +ghcr.io/idaholab/malcolm/postgresql 23.05.1 xxxxxxxxxxxx 3 days ago 268MB +ghcr.io/idaholab/malcolm/redis 23.05.1 xxxxxxxxxxxx 3 days ago 34.2MB +ghcr.io/idaholab/malcolm/suricata 23.05.1 xxxxxxxxxxxx 3 days ago 278MB +ghcr.io/idaholab/malcolm/zeek 23.05.1 xxxxxxxxxxxx 3 days ago 1GB ``` ### Import from pre-packaged tarballs diff --git a/docs/ubuntu-install-example.md b/docs/ubuntu-install-example.md index 0e4e8c315..88285f207 100644 --- a/docs/ubuntu-install-example.md +++ b/docs/ubuntu-install-example.md @@ -256,25 +256,25 @@ Pulling zeek ... done user@host:~/Malcolm$ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -ghcr.io/idaholab/malcolm/api 23.05.0 xxxxxxxxxxxx 3 days ago 158MB -ghcr.io/idaholab/malcolm/arkime 23.05.0 xxxxxxxxxxxx 3 days ago 816MB -ghcr.io/idaholab/malcolm/dashboards 23.05.0 xxxxxxxxxxxx 3 days ago 1.02GB -ghcr.io/idaholab/malcolm/dashboards-helper 23.05.0 xxxxxxxxxxxx 3 days ago 184MB -ghcr.io/idaholab/malcolm/file-monitor 23.05.0 xxxxxxxxxxxx 3 days ago 588MB -ghcr.io/idaholab/malcolm/file-upload 23.05.0 xxxxxxxxxxxx 3 days ago 259MB -ghcr.io/idaholab/malcolm/filebeat-oss 23.05.0 xxxxxxxxxxxx 3 days ago 624MB -ghcr.io/idaholab/malcolm/freq 23.05.0 xxxxxxxxxxxx 3 days ago 132MB -ghcr.io/idaholab/malcolm/htadmin 23.05.0 xxxxxxxxxxxx 3 days ago 242MB -ghcr.io/idaholab/malcolm/logstash-oss 23.05.0 xxxxxxxxxxxx 3 days ago 1.35GB -ghcr.io/idaholab/malcolm/netbox 23.05.0 xxxxxxxxxxxx 3 days ago 1.01GB -ghcr.io/idaholab/malcolm/nginx-proxy 23.05.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/opensearch 23.05.0 xxxxxxxxxxxx 3 days ago 1.17GB -ghcr.io/idaholab/malcolm/pcap-capture 23.05.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/pcap-monitor 23.05.0 xxxxxxxxxxxx 3 days ago 213MB -ghcr.io/idaholab/malcolm/postgresql 23.05.0 xxxxxxxxxxxx 3 days ago 268MB -ghcr.io/idaholab/malcolm/redis 23.05.0 xxxxxxxxxxxx 3 days ago 34.2MB -ghcr.io/idaholab/malcolm/suricata 23.05.0 xxxxxxxxxxxx 3 days ago 278MB -ghcr.io/idaholab/malcolm/zeek 23.05.0 xxxxxxxxxxxx 3 days ago 1GB +ghcr.io/idaholab/malcolm/api 23.05.1 xxxxxxxxxxxx 3 days ago 158MB +ghcr.io/idaholab/malcolm/arkime 23.05.1 xxxxxxxxxxxx 3 days ago 816MB +ghcr.io/idaholab/malcolm/dashboards 23.05.1 xxxxxxxxxxxx 3 days ago 1.02GB +ghcr.io/idaholab/malcolm/dashboards-helper 23.05.1 xxxxxxxxxxxx 3 days ago 184MB +ghcr.io/idaholab/malcolm/file-monitor 23.05.1 xxxxxxxxxxxx 3 days ago 588MB +ghcr.io/idaholab/malcolm/file-upload 23.05.1 xxxxxxxxxxxx 3 days ago 259MB +ghcr.io/idaholab/malcolm/filebeat-oss 23.05.1 xxxxxxxxxxxx 3 days ago 624MB +ghcr.io/idaholab/malcolm/freq 23.05.1 xxxxxxxxxxxx 3 days ago 132MB +ghcr.io/idaholab/malcolm/htadmin 23.05.1 xxxxxxxxxxxx 3 days ago 242MB +ghcr.io/idaholab/malcolm/logstash-oss 23.05.1 xxxxxxxxxxxx 3 days ago 1.35GB +ghcr.io/idaholab/malcolm/netbox 23.05.1 xxxxxxxxxxxx 3 days ago 1.01GB +ghcr.io/idaholab/malcolm/nginx-proxy 23.05.1 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/opensearch 23.05.1 xxxxxxxxxxxx 3 days ago 1.17GB +ghcr.io/idaholab/malcolm/pcap-capture 23.05.1 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/pcap-monitor 23.05.1 xxxxxxxxxxxx 3 days ago 213MB +ghcr.io/idaholab/malcolm/postgresql 23.05.1 xxxxxxxxxxxx 3 days ago 268MB +ghcr.io/idaholab/malcolm/redis 23.05.1 xxxxxxxxxxxx 3 days ago 34.2MB +ghcr.io/idaholab/malcolm/suricata 23.05.1 xxxxxxxxxxxx 3 days ago 278MB +ghcr.io/idaholab/malcolm/zeek 23.05.1 xxxxxxxxxxxx 3 days ago 1GB ``` Finally, we can start Malcolm. When Malcolm starts it will stream informational and debug messages to the console. If you wish, you can safely close the console or use `Ctrl+C` to stop these messages; Malcolm will continue running in the background. diff --git a/kubernetes/02-opensearch.yml b/kubernetes/02-opensearch.yml index 6cf5af14e..bac0e641f 100644 --- a/kubernetes/02-opensearch.yml +++ b/kubernetes/02-opensearch.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: opensearch-container - image: ghcr.io/idaholab/malcolm/opensearch:23.05.0 + image: ghcr.io/idaholab/malcolm/opensearch:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/03-dashboards.yml b/kubernetes/03-dashboards.yml index 8db23880b..2ab9d7ad6 100644 --- a/kubernetes/03-dashboards.yml +++ b/kubernetes/03-dashboards.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: dashboards-container - image: ghcr.io/idaholab/malcolm/dashboards:23.05.0 + image: ghcr.io/idaholab/malcolm/dashboards:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/04-upload.yml b/kubernetes/04-upload.yml index bb978dbd2..7a0233e38 100644 --- a/kubernetes/04-upload.yml +++ b/kubernetes/04-upload.yml @@ -34,7 +34,7 @@ spec: spec: containers: - name: upload-container - image: ghcr.io/idaholab/malcolm/file-upload:23.05.0 + image: ghcr.io/idaholab/malcolm/file-upload:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/05-pcap-monitor.yml b/kubernetes/05-pcap-monitor.yml index 04be6b978..8cb55fb9a 100644 --- a/kubernetes/05-pcap-monitor.yml +++ b/kubernetes/05-pcap-monitor.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: pcap-monitor-container - image: ghcr.io/idaholab/malcolm/pcap-monitor:23.05.0 + image: ghcr.io/idaholab/malcolm/pcap-monitor:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/06-arkime.yml b/kubernetes/06-arkime.yml index 786961836..f74812290 100644 --- a/kubernetes/06-arkime.yml +++ b/kubernetes/06-arkime.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: arkime-container - image: ghcr.io/idaholab/malcolm/arkime:23.05.0 + image: ghcr.io/idaholab/malcolm/arkime:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/07-api.yml b/kubernetes/07-api.yml index d318a8c46..33fa6d1e0 100644 --- a/kubernetes/07-api.yml +++ b/kubernetes/07-api.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: api-container - image: ghcr.io/idaholab/malcolm/api:23.05.0 + image: ghcr.io/idaholab/malcolm/api:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/08-dashboards-helper.yml b/kubernetes/08-dashboards-helper.yml index bdf8cb767..4ae09aaed 100644 --- a/kubernetes/08-dashboards-helper.yml +++ b/kubernetes/08-dashboards-helper.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: dashboards-helper-container - image: ghcr.io/idaholab/malcolm/dashboards-helper:23.05.0 + image: ghcr.io/idaholab/malcolm/dashboards-helper:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/09-zeek.yml b/kubernetes/09-zeek.yml index edfc8a64d..ab83065ca 100644 --- a/kubernetes/09-zeek.yml +++ b/kubernetes/09-zeek.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: zeek-offline-container - image: ghcr.io/idaholab/malcolm/zeek:23.05.0 + image: ghcr.io/idaholab/malcolm/zeek:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/10-suricata.yml b/kubernetes/10-suricata.yml index d2208ab26..d89e987a7 100644 --- a/kubernetes/10-suricata.yml +++ b/kubernetes/10-suricata.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: suricata-offline-container - image: ghcr.io/idaholab/malcolm/suricata:23.05.0 + image: ghcr.io/idaholab/malcolm/suricata:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/11-file-monitor.yml b/kubernetes/11-file-monitor.yml index 05b4227f1..4d65038a5 100644 --- a/kubernetes/11-file-monitor.yml +++ b/kubernetes/11-file-monitor.yml @@ -33,7 +33,7 @@ spec: spec: containers: - name: file-monitor-container - image: ghcr.io/idaholab/malcolm/file-monitor:23.05.0 + image: ghcr.io/idaholab/malcolm/file-monitor:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/12-filebeat.yml b/kubernetes/12-filebeat.yml index aa073b19d..86e0e1c83 100644 --- a/kubernetes/12-filebeat.yml +++ b/kubernetes/12-filebeat.yml @@ -31,7 +31,7 @@ spec: spec: containers: - name: filebeat-container - image: ghcr.io/idaholab/malcolm/filebeat-oss:23.05.0 + image: ghcr.io/idaholab/malcolm/filebeat-oss:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/13-logstash.yml b/kubernetes/13-logstash.yml index 56b9c256c..2a3920b40 100644 --- a/kubernetes/13-logstash.yml +++ b/kubernetes/13-logstash.yml @@ -47,7 +47,7 @@ spec: # topologyKey: "kubernetes.io/hostname" containers: - name: logstash-container - image: ghcr.io/idaholab/malcolm/logstash-oss:23.05.0 + image: ghcr.io/idaholab/malcolm/logstash-oss:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/15-netbox-redis.yml b/kubernetes/15-netbox-redis.yml index 9fceac45d..dcada13b0 100644 --- a/kubernetes/15-netbox-redis.yml +++ b/kubernetes/15-netbox-redis.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-redis-container - image: ghcr.io/idaholab/malcolm/redis:23.05.0 + image: ghcr.io/idaholab/malcolm/redis:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/16-netbox-redis-cache.yml b/kubernetes/16-netbox-redis-cache.yml index 1096ca615..5d8ff37e9 100644 --- a/kubernetes/16-netbox-redis-cache.yml +++ b/kubernetes/16-netbox-redis-cache.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-redis-cache-container - image: ghcr.io/idaholab/malcolm/redis:23.05.0 + image: ghcr.io/idaholab/malcolm/redis:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/17-netbox-postgres.yml b/kubernetes/17-netbox-postgres.yml index 5d5ad21a0..70f70002f 100644 --- a/kubernetes/17-netbox-postgres.yml +++ b/kubernetes/17-netbox-postgres.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-postgres-container - image: ghcr.io/idaholab/malcolm/postgresql:23.05.0 + image: ghcr.io/idaholab/malcolm/postgresql:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/18-netbox.yml b/kubernetes/18-netbox.yml index d22b3f7ac..ac53304b5 100644 --- a/kubernetes/18-netbox.yml +++ b/kubernetes/18-netbox.yml @@ -36,7 +36,7 @@ spec: spec: containers: - name: netbox-container - image: ghcr.io/idaholab/malcolm/netbox:23.05.0 + image: ghcr.io/idaholab/malcolm/netbox:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/19-htadmin.yml b/kubernetes/19-htadmin.yml index 0bfc8348a..918d3a8fb 100644 --- a/kubernetes/19-htadmin.yml +++ b/kubernetes/19-htadmin.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: htadmin-container - image: ghcr.io/idaholab/malcolm/htadmin:23.05.0 + image: ghcr.io/idaholab/malcolm/htadmin:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/20-pcap-capture.yml b/kubernetes/20-pcap-capture.yml index 5c9b21f3f..816c491dc 100644 --- a/kubernetes/20-pcap-capture.yml +++ b/kubernetes/20-pcap-capture.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: pcap-capture-container - image: ghcr.io/idaholab/malcolm/pcap-capture:23.05.0 + image: ghcr.io/idaholab/malcolm/pcap-capture:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/21-zeek-live.yml b/kubernetes/21-zeek-live.yml index f67b32625..835654692 100644 --- a/kubernetes/21-zeek-live.yml +++ b/kubernetes/21-zeek-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: zeek-live-container - image: ghcr.io/idaholab/malcolm/zeek:23.05.0 + image: ghcr.io/idaholab/malcolm/zeek:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/22-suricata-live.yml b/kubernetes/22-suricata-live.yml index d0fa77305..2b48ed2d5 100644 --- a/kubernetes/22-suricata-live.yml +++ b/kubernetes/22-suricata-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: suricata-live-container - image: ghcr.io/idaholab/malcolm/suricata:23.05.0 + image: ghcr.io/idaholab/malcolm/suricata:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/23-freq.yml b/kubernetes/23-freq.yml index c515bd917..86f316139 100644 --- a/kubernetes/23-freq.yml +++ b/kubernetes/23-freq.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: freq-container - image: ghcr.io/idaholab/malcolm/freq:23.05.0 + image: ghcr.io/idaholab/malcolm/freq:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/99-nginx-proxy.yml b/kubernetes/99-nginx-proxy.yml index ccd1d5124..ef3f0e74a 100644 --- a/kubernetes/99-nginx-proxy.yml +++ b/kubernetes/99-nginx-proxy.yml @@ -37,7 +37,7 @@ spec: spec: containers: - name: nginx-proxy-container - image: ghcr.io/idaholab/malcolm/nginx-proxy:23.05.0 + image: ghcr.io/idaholab/malcolm/nginx-proxy:23.05.1 imagePullPolicy: Always stdin: false tty: true From 329c6a1a9482c10a5743c792129b1b79111800b8 Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 4 May 2023 15:20:03 -0600 Subject: [PATCH 04/47] switch kubernetes image names for development --- kubernetes/02-opensearch.yml | 2 +- kubernetes/03-dashboards.yml | 2 +- kubernetes/04-upload.yml | 2 +- kubernetes/05-pcap-monitor.yml | 2 +- kubernetes/06-arkime.yml | 2 +- kubernetes/07-api.yml | 2 +- kubernetes/08-dashboards-helper.yml | 2 +- kubernetes/09-zeek.yml | 2 +- kubernetes/10-suricata.yml | 2 +- kubernetes/11-file-monitor.yml | 2 +- kubernetes/12-filebeat.yml | 2 +- kubernetes/13-logstash.yml | 2 +- kubernetes/15-netbox-redis.yml | 2 +- kubernetes/16-netbox-redis-cache.yml | 2 +- kubernetes/17-netbox-postgres.yml | 2 +- kubernetes/18-netbox.yml | 2 +- kubernetes/19-htadmin.yml | 2 +- kubernetes/20-pcap-capture.yml | 2 +- kubernetes/21-zeek-live.yml | 2 +- kubernetes/22-suricata-live.yml | 2 +- kubernetes/23-freq.yml | 2 +- kubernetes/99-nginx-proxy.yml | 2 +- 22 files changed, 22 insertions(+), 22 deletions(-) diff --git a/kubernetes/02-opensearch.yml b/kubernetes/02-opensearch.yml index bac0e641f..e2f70e54c 100644 --- a/kubernetes/02-opensearch.yml +++ b/kubernetes/02-opensearch.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: opensearch-container - image: ghcr.io/idaholab/malcolm/opensearch:23.05.1 + image: ghcr.io/mmguero-dev/malcolm/opensearch:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/03-dashboards.yml b/kubernetes/03-dashboards.yml index 2ab9d7ad6..cfbb8b422 100644 --- a/kubernetes/03-dashboards.yml +++ b/kubernetes/03-dashboards.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: dashboards-container - image: ghcr.io/idaholab/malcolm/dashboards:23.05.1 + image: ghcr.io/mmguero-dev/malcolm/dashboards:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/04-upload.yml b/kubernetes/04-upload.yml index 7a0233e38..19915597c 100644 --- a/kubernetes/04-upload.yml +++ b/kubernetes/04-upload.yml @@ -34,7 +34,7 @@ spec: spec: containers: - name: upload-container - image: ghcr.io/idaholab/malcolm/file-upload:23.05.1 + image: ghcr.io/mmguero-dev/malcolm/file-upload:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/05-pcap-monitor.yml b/kubernetes/05-pcap-monitor.yml index 8cb55fb9a..b8b76084b 100644 --- a/kubernetes/05-pcap-monitor.yml +++ b/kubernetes/05-pcap-monitor.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: pcap-monitor-container - image: ghcr.io/idaholab/malcolm/pcap-monitor:23.05.1 + image: ghcr.io/mmguero-dev/malcolm/pcap-monitor:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/06-arkime.yml b/kubernetes/06-arkime.yml index f74812290..ef24eb44b 100644 --- a/kubernetes/06-arkime.yml +++ b/kubernetes/06-arkime.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: arkime-container - image: ghcr.io/idaholab/malcolm/arkime:23.05.1 + image: ghcr.io/mmguero-dev/malcolm/arkime:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/07-api.yml b/kubernetes/07-api.yml index 33fa6d1e0..3f611da69 100644 --- a/kubernetes/07-api.yml +++ b/kubernetes/07-api.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: api-container - image: ghcr.io/idaholab/malcolm/api:23.05.1 + image: ghcr.io/mmguero-dev/malcolm/api:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/08-dashboards-helper.yml b/kubernetes/08-dashboards-helper.yml index 4ae09aaed..529647210 100644 --- a/kubernetes/08-dashboards-helper.yml +++ b/kubernetes/08-dashboards-helper.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: dashboards-helper-container - image: ghcr.io/idaholab/malcolm/dashboards-helper:23.05.1 + image: ghcr.io/mmguero-dev/malcolm/dashboards-helper:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/09-zeek.yml b/kubernetes/09-zeek.yml index ab83065ca..f7fd93b7f 100644 --- a/kubernetes/09-zeek.yml +++ b/kubernetes/09-zeek.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: zeek-offline-container - image: ghcr.io/idaholab/malcolm/zeek:23.05.1 + image: ghcr.io/mmguero-dev/malcolm/zeek:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/10-suricata.yml b/kubernetes/10-suricata.yml index d89e987a7..da525b557 100644 --- a/kubernetes/10-suricata.yml +++ b/kubernetes/10-suricata.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: suricata-offline-container - image: ghcr.io/idaholab/malcolm/suricata:23.05.1 + image: ghcr.io/mmguero-dev/malcolm/suricata:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/11-file-monitor.yml b/kubernetes/11-file-monitor.yml index 4d65038a5..39a29b65d 100644 --- a/kubernetes/11-file-monitor.yml +++ b/kubernetes/11-file-monitor.yml @@ -33,7 +33,7 @@ spec: spec: containers: - name: file-monitor-container - image: ghcr.io/idaholab/malcolm/file-monitor:23.05.1 + image: ghcr.io/mmguero-dev/malcolm/file-monitor:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/12-filebeat.yml b/kubernetes/12-filebeat.yml index 86e0e1c83..7676193c0 100644 --- a/kubernetes/12-filebeat.yml +++ b/kubernetes/12-filebeat.yml @@ -31,7 +31,7 @@ spec: spec: containers: - name: filebeat-container - image: ghcr.io/idaholab/malcolm/filebeat-oss:23.05.1 + image: ghcr.io/mmguero-dev/malcolm/filebeat-oss:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/13-logstash.yml b/kubernetes/13-logstash.yml index 2a3920b40..081c47ed3 100644 --- a/kubernetes/13-logstash.yml +++ b/kubernetes/13-logstash.yml @@ -47,7 +47,7 @@ spec: # topologyKey: "kubernetes.io/hostname" containers: - name: logstash-container - image: ghcr.io/idaholab/malcolm/logstash-oss:23.05.1 + image: ghcr.io/mmguero-dev/malcolm/logstash-oss:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/15-netbox-redis.yml b/kubernetes/15-netbox-redis.yml index dcada13b0..984110449 100644 --- a/kubernetes/15-netbox-redis.yml +++ b/kubernetes/15-netbox-redis.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-redis-container - image: ghcr.io/idaholab/malcolm/redis:23.05.1 + image: ghcr.io/mmguero-dev/malcolm/redis:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/16-netbox-redis-cache.yml b/kubernetes/16-netbox-redis-cache.yml index 5d8ff37e9..22f1be5a4 100644 --- a/kubernetes/16-netbox-redis-cache.yml +++ b/kubernetes/16-netbox-redis-cache.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-redis-cache-container - image: ghcr.io/idaholab/malcolm/redis:23.05.1 + image: ghcr.io/mmguero-dev/malcolm/redis:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/17-netbox-postgres.yml b/kubernetes/17-netbox-postgres.yml index 70f70002f..1497c1bea 100644 --- a/kubernetes/17-netbox-postgres.yml +++ b/kubernetes/17-netbox-postgres.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-postgres-container - image: ghcr.io/idaholab/malcolm/postgresql:23.05.1 + image: ghcr.io/mmguero-dev/malcolm/postgresql:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/18-netbox.yml b/kubernetes/18-netbox.yml index ac53304b5..5a613fc8e 100644 --- a/kubernetes/18-netbox.yml +++ b/kubernetes/18-netbox.yml @@ -36,7 +36,7 @@ spec: spec: containers: - name: netbox-container - image: ghcr.io/idaholab/malcolm/netbox:23.05.1 + image: ghcr.io/mmguero-dev/malcolm/netbox:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/19-htadmin.yml b/kubernetes/19-htadmin.yml index 918d3a8fb..716a46363 100644 --- a/kubernetes/19-htadmin.yml +++ b/kubernetes/19-htadmin.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: htadmin-container - image: ghcr.io/idaholab/malcolm/htadmin:23.05.1 + image: ghcr.io/mmguero-dev/malcolm/htadmin:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/20-pcap-capture.yml b/kubernetes/20-pcap-capture.yml index 816c491dc..63360e5a4 100644 --- a/kubernetes/20-pcap-capture.yml +++ b/kubernetes/20-pcap-capture.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: pcap-capture-container - image: ghcr.io/idaholab/malcolm/pcap-capture:23.05.1 + image: ghcr.io/mmguero-dev/malcolm/pcap-capture:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/21-zeek-live.yml b/kubernetes/21-zeek-live.yml index 835654692..3eb2cf92f 100644 --- a/kubernetes/21-zeek-live.yml +++ b/kubernetes/21-zeek-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: zeek-live-container - image: ghcr.io/idaholab/malcolm/zeek:23.05.1 + image: ghcr.io/mmguero-dev/malcolm/zeek:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/22-suricata-live.yml b/kubernetes/22-suricata-live.yml index 2b48ed2d5..fe0ee1d21 100644 --- a/kubernetes/22-suricata-live.yml +++ b/kubernetes/22-suricata-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: suricata-live-container - image: ghcr.io/idaholab/malcolm/suricata:23.05.1 + image: ghcr.io/mmguero-dev/malcolm/suricata:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/23-freq.yml b/kubernetes/23-freq.yml index 86f316139..b9dc580df 100644 --- a/kubernetes/23-freq.yml +++ b/kubernetes/23-freq.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: freq-container - image: ghcr.io/idaholab/malcolm/freq:23.05.1 + image: ghcr.io/mmguero-dev/malcolm/freq:development imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/99-nginx-proxy.yml b/kubernetes/99-nginx-proxy.yml index ef3f0e74a..5c2f49b4c 100644 --- a/kubernetes/99-nginx-proxy.yml +++ b/kubernetes/99-nginx-proxy.yml @@ -37,7 +37,7 @@ spec: spec: containers: - name: nginx-proxy-container - image: ghcr.io/idaholab/malcolm/nginx-proxy:23.05.1 + image: ghcr.io/mmguero-dev/malcolm/nginx-proxy:development imagePullPolicy: Always stdin: false tty: true From 97d8cae52f7846bc56512ae78a2df6144a392f0e Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 4 May 2023 15:20:23 -0600 Subject: [PATCH 05/47] define types for some arkime fields --- dashboards/templates/composable/component/arkime.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dashboards/templates/composable/component/arkime.json b/dashboards/templates/composable/component/arkime.json index b1b975fd5..678585380 100644 --- a/dashboards/templates/composable/component/arkime.json +++ b/dashboards/templates/composable/component/arkime.json @@ -11,6 +11,10 @@ "destination.geo.longitude": { "type": "float" }, "dns.host": { "type": "keyword" }, "firstPacket": { "type": "date" }, + "http.xffASN": { "type": "keyword" }, + "http.xffGEO": { "type": "keyword" }, + "http.xffIp": { "type": "ip" }, + "http.xffRIR": { "type": "keyword" }, "lastPacket": { "type": "date" }, "node": { "type": "keyword" }, "protocol": { "type": "keyword" }, From 6887895c41bf73165b3de082d4bf22001de93743 Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 4 May 2023 15:53:43 -0600 Subject: [PATCH 06/47] work in progress version bumps, probably broken --- Dockerfiles/dashboards.Dockerfile | 16 +++-- Dockerfiles/filebeat.Dockerfile | 2 +- Dockerfiles/opensearch.Dockerfile | 2 +- sensor-iso/beats/Dockerfile | 51 -------------- sensor-iso/beats/beat-build.sh | 26 ------- sensor-iso/beats/build-docker-image.sh | 11 --- sensor-iso/beats/build.sh | 69 ------------------- .../normal/0910-sensor-build.hook.chroot | 2 +- 8 files changed, 15 insertions(+), 164 deletions(-) delete mode 100644 sensor-iso/beats/Dockerfile delete mode 100755 sensor-iso/beats/beat-build.sh delete mode 100755 sensor-iso/beats/build-docker-image.sh delete mode 100755 sensor-iso/beats/build.sh diff --git a/Dockerfiles/dashboards.Dockerfile b/Dockerfiles/dashboards.Dockerfile index c6ccfcb93..1811ae826 100644 --- a/Dockerfiles/dashboards.Dockerfile +++ b/Dockerfiles/dashboards.Dockerfile @@ -14,10 +14,10 @@ ENV PGROUP "dashboarder" ENV TERM xterm -ARG OPENSEARCH_VERSION="2.6.0" +ARG OPENSEARCH_VERSION="2.7.0" ENV OPENSEARCH_VERSION $OPENSEARCH_VERSION -ARG OPENSEARCH_DASHBOARDS_VERSION="2.6.0" +ARG OPENSEARCH_DASHBOARDS_VERSION="2.7.0" ENV OPENSEARCH_DASHBOARDS_VERSION $OPENSEARCH_DASHBOARDS_VERSION # base system dependencies for checking out and building plugins @@ -68,7 +68,7 @@ RUN eval "$(nodenv init -)" && \ # runtime ################################################################## -FROM opensearchproject/opensearch-dashboards:2.6.0 +FROM opensearchproject/opensearch-dashboards:2.7.0 LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' @@ -114,6 +114,7 @@ USER root COPY --from=build /usr/share/opensearch-dashboards/plugins/sankey_vis/build/kbnSankeyVis.zip /tmp/kbnSankeyVis.zip ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /usr/bin/tini +ADD https://github.com/lguillaud/osd_transform_vis/releases/download/$OSD_TRANSFORM_VIS_VERSION/transformVis-$OSD_TRANSFORM_VIS_VERSION.zip /tmp/transformVis.zip RUN yum upgrade -y && \ yum install -y curl psmisc util-linux openssl rsync python3 zip unzip && \ @@ -122,7 +123,14 @@ RUN yum upgrade -y && \ /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove securityDashboards --allow-root && \ cd /usr/share/opensearch-dashboards/plugins && \ /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin install file:///tmp/kbnSankeyVis.zip --allow-root && \ - /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin install https://github.com/lguillaud/osd_transform_vis/releases/download/$OSD_TRANSFORM_VIS_VERSION/transformVis-$OSD_TRANSFORM_VIS_VERSION.zip --allow-root && \ + cd /tmp && \ + unzip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \ + sed -i "s/2\.6\.0/2\.7\.0/g" opensearch-dashboards/transformVis/opensearch_dashboards.json && \ + sed -i "s/2\.6\.0/2\.7\.0/g" opensearch-dashboards/transformVis/package.json && \ + zip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \ + cd /usr/share/opensearch-dashboards/plugins && \ + /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin install file:///tmp/transformVis.zip --allow-root && \ + rm -rf /tmp/transformVis /tmp/opensearch-dashboards && \ chown --silent -R root:root /usr/share/opensearch-dashboards/plugins/* \ /usr/share/opensearch-dashboards/node_modules/* \ /usr/share/opensearch-dashboards/src/* && \ diff --git a/Dockerfiles/filebeat.Dockerfile b/Dockerfiles/filebeat.Dockerfile index f3bcd19d0..799cd6a44 100644 --- a/Dockerfiles/filebeat.Dockerfile +++ b/Dockerfiles/filebeat.Dockerfile @@ -1,4 +1,4 @@ -FROM docker.elastic.co/beats/filebeat-oss:8.6.2 +FROM docker.elastic.co/beats/filebeat-oss:8.7.1 # Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" diff --git a/Dockerfiles/opensearch.Dockerfile b/Dockerfiles/opensearch.Dockerfile index a65b0a1c5..aeb6a6cee 100644 --- a/Dockerfiles/opensearch.Dockerfile +++ b/Dockerfiles/opensearch.Dockerfile @@ -1,4 +1,4 @@ -FROM opensearchproject/opensearch:2.6.0 +FROM opensearchproject/opensearch:2.7.0 # Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" diff --git a/sensor-iso/beats/Dockerfile b/sensor-iso/beats/Dockerfile deleted file mode 100644 index 5126c1bb3..000000000 --- a/sensor-iso/beats/Dockerfile +++ /dev/null @@ -1,51 +0,0 @@ -FROM debian:buster-slim - -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. - -LABEL maintainer="malcolm@inl.gov" - -ENV DEBIAN_FRONTEND noninteractive -ENV GOPATH=/go -ENV GOBIN=/go/bin -ENV GOARCH=amd64 -ENV GOVERS="2:1.15~1~bpo10+1" -ENV PATH="$GOBIN:${PATH}" -ENV PYTHON_EXE=python3 - -RUN set -x && \ - sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list && \ - echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list && \ - apt-get -q update && \ - apt-get install -y curl git vim-tiny && \ - apt-get install -t buster-backports -y \ - "golang-doc=$GOVERS" \ - "golang-go=$GOVERS" \ - "golang-src=$GOVERS" \ - "golang=$GOVERS" \ - build-essential \ - python3 \ - python3-dev \ - python3-pip \ - python3-setuptools \ - python3-virtualenv \ - python3-wheel \ - virtualenv && \ - rm -rf /var/lib/apt/lists/* && \ - update-alternatives --install /usr/bin/python python /usr/bin/python3 2 && \ - update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 2 && \ - python3 -m pip install -U pyyaml cookiecutter && \ - mkdir -p "$GOPATH/bin" && \ - bash -c "curl -sSL https://raw.githubusercontent.com/Masterminds/glide.sh/master/get | sed 's@https://glide.sh/@https://raw.githubusercontent.com/Masterminds/glide.sh/master/@g'| bash" && \ - go get -u -d github.com/magefile/mage && \ - cd $GOPATH/src/github.com/magefile/mage && \ - go run bootstrap.go - -ENV BEATS=filebeat -ENV BEATS_VERSION=8.6.2 - -ADD ./build.sh /build.sh -RUN [ "chmod", "+x", "/build.sh" ] -RUN [ "mkdir", "-p", "/go" ] -RUN [ "mkdir", "/build" ] - -CMD "/build.sh" diff --git a/sensor-iso/beats/beat-build.sh b/sensor-iso/beats/beat-build.sh deleted file mode 100755 index 63ada694c..000000000 --- a/sensor-iso/beats/beat-build.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. - -VERSION="8.6.0" -THIRD_PARTY_BRANCH="master" -while getopts b:v:t: opts; do - case ${opts} in - b) BEAT=${OPTARG} ;; - v) VERSION=${OPTARG} ;; - t) THIRD_PARTY_BRANCH=${OPTARG} ;; - esac -done - -if [[ -z $BEAT || -z $VERSION || -z $THIRD_PARTY_BRANCH ]] ; then - echo "usage:" >&2 - echo " beat-build.sh -b [-v ] [-v ]" >&2 - echo "" >&2 - echo "example:" >&2 - echo " beat-build.sh -b filebeat -v $VERSION" >&2 - exit 1 -fi - -BEAT_DIR="$(pwd)/$(echo "$BEAT" | sed "s@^https*://@@" | sed 's@/@_@g')" -mkdir -p "$BEAT_DIR" -docker run --rm -v "$BEAT_DIR":/build -e "BEATS_VERSION=$VERSION" -e "THIRD_PARTY_BRANCH=$THIRD_PARTY_BRANCH" -e "BEATS=$BEAT" beats-build:latest diff --git a/sensor-iso/beats/build-docker-image.sh b/sensor-iso/beats/build-docker-image.sh deleted file mode 100755 index ef9cb305d..000000000 --- a/sensor-iso/beats/build-docker-image.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. - -# force-navigate to script directory -SCRIPT_PATH="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -pushd "$SCRIPT_PATH" >/dev/null 2>&1 - -docker build -t beats-build:latest . - -popd >/dev/null 2>&1 diff --git a/sensor-iso/beats/build.sh b/sensor-iso/beats/build.sh deleted file mode 100755 index 87da7e31d..000000000 --- a/sensor-iso/beats/build.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. - -echo Target version: $BEATS_VERSION - -BRANCH=$(echo $BEATS_VERSION | awk -F \. {'print $1 "." $2'}) -echo Target branch: $BRANCH - -if [ ! -d "$GOPATH/src/github.com/elastic/beats" ]; then go get -v github.com/elastic/beats; fi - -cd $GOPATH/src/github.com/elastic/beats -git checkout $BRANCH - -IFS="," -BEATS_ARRAY=($BEATS) - -for BEAT in "${BEATS_ARRAY[@]}" -do - - if [[ -d "$GOPATH/src/github.com/elastic/beats/$BEAT" ]] ; then - # an official beat - cd "$GOPATH/src/github.com/elastic/beats/$BEAT" - make - cp "$BEAT" /build - - # package - DOWNLOAD="$BEAT-$BEATS_VERSION-linux-x86.tar.gz" - if [ ! -e $DOWNLOAD ]; then curl -s -O -J "https://artifacts.elastic.co/downloads/beats/$BEAT/$DOWNLOAD"; fi - tar xf "$DOWNLOAD" - - cp "$BEAT" "$BEAT-$BEATS_VERSION-linux-x86" - tar zcf "$BEAT-$BEATS_VERSION-linux-amd64.tar.gz" "$BEAT-$BEATS_VERSION-linux-x86" - cp "$BEAT-$BEATS_VERSION-linux-amd64.tar.gz" /build - - elif [[ "$BEAT" =~ ^https*://(gogs\..*|github\.com) ]] ; then - BRANCH=${THIRD_PARTY_BRANCH:-"master"} - - # clone from git manually rather than do a "go get" - mkdir -p "$GOPATH/src/$(dirname "$(echo "$BEAT" | sed "s@^https*://@@")")" - cd "$GOPATH/src/$(dirname "$(echo "$BEAT" | sed "s@^https*://@@")")" - git clone --depth=1 --single-branch --branch "$BRANCH" "$BEAT" - BEAT_EXE_NAME="$(basename "$BEAT" | sed "s/\.git$//")" - cd "$BEAT_EXE_NAME" - go get - go install - if [[ -f "$GOBIN/$BEAT_EXE_NAME" ]] ; then - cp "$GOBIN/$BEAT_EXE_NAME" /build - strip "/build/$BEAT_EXE_NAME" - fi - - else - # a community beat? - if [[ "$BEAT" =~ gogs\..* ]]; then - INSECURE_FLAG="--insecure" - else - INSECURE_FLAG="" - fi - go get $INSECURE_FLAG "$BEAT" - BEAT_EXE_NAME="$(basename "$BEAT")" - if [[ -f "$GOBIN/$BEAT_EXE_NAME" ]] ; then - cp "$GOBIN/$BEAT_EXE_NAME" /build - strip "/build/$BEAT_EXE_NAME" - fi - fi - - ls -lh /build - -done diff --git a/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot b/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot index d9fdc9deb..6af459b13 100755 --- a/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot +++ b/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot @@ -20,7 +20,7 @@ export PATH="${ZEEK_DIR}"/bin:$PATH SURICATA_RULES_DIR="/etc/suricata/rules" -BEATS_VER="8.6.2" +BEATS_VER="8.7.1" BEATS_OSS="-oss" BEATS_DEB_URL_TEMPLATE_REPLACER="XXXXX" BEATS_DEB_URL_TEMPLATE="https://artifacts.elastic.co/downloads/beats/$BEATS_DEB_URL_TEMPLATE_REPLACER/$BEATS_DEB_URL_TEMPLATE_REPLACER$BEATS_OSS-$BEATS_VER-amd64.deb" From f35ac58d369c0690e87d61f320c4ac421ba86be1 Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 8 May 2023 07:43:37 -0600 Subject: [PATCH 07/47] quiet whiny filebeat message --- scripts/malcolm_common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/malcolm_common.py b/scripts/malcolm_common.py index 8dfb84a3b..50ba6223d 100644 --- a/scripts/malcolm_common.py +++ b/scripts/malcolm_common.py @@ -678,6 +678,7 @@ def DownloadToFile(url, local_filename, debug=False): | eshealth | esindices/list | executing\s+attempt_(transition|set_replica_count)\s+for + | failed\s+to\s+get\s+tcp\s+stats\s+from\s+/proc | GET\s+/(netbox/api|_cat/health|api/status|sessions2-|arkime_\w+).+HTTP/[\d\.].+\b200\b | loaded\s+config\s+'/etc/netbox/config/ | "netbox"\s+application\s+started From c37ea91b0a7c0da1035950d7bdadf6fec953ada7 Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 8 May 2023 08:58:59 -0600 Subject: [PATCH 08/47] fix comment on dashboard --- dashboards/dashboards/03207c00-d07e-11ec-b4a7-d1b4003706b7.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboards/dashboards/03207c00-d07e-11ec-b4a7-d1b4003706b7.json b/dashboards/dashboards/03207c00-d07e-11ec-b4a7-d1b4003706b7.json index 113bae5ca..4e214d244 100644 --- a/dashboards/dashboards/03207c00-d07e-11ec-b4a7-d1b4003706b7.json +++ b/dashboards/dashboards/03207c00-d07e-11ec-b4a7-d1b4003706b7.json @@ -12,7 +12,7 @@ "attributes": { "title": "GENISYS", "hits": 0, - "description": "Dashboard for the DNP3 Protocol", + "description": "Dashboard for the GENISYS Protocol", "panelsJSON": "[{\"version\":\"1.3.1\",\"gridData\":{\"h\":28,\"i\":\"1\",\"w\":8,\"x\":0,\"y\":0},\"panelIndex\":\"1\",\"embeddableConfig\":{},\"panelRefName\":\"panel_0\"},{\"version\":\"1.3.1\",\"gridData\":{\"h\":10,\"i\":\"58856fb7-efd0-4246-9dc9-d8b0d5c3fcba\",\"w\":8,\"x\":8,\"y\":0},\"panelIndex\":\"58856fb7-efd0-4246-9dc9-d8b0d5c3fcba\",\"embeddableConfig\":{},\"panelRefName\":\"panel_1\"},{\"version\":\"1.3.1\",\"gridData\":{\"h\":10,\"i\":\"c078d6a7-456e-4fed-80c6-f36123c3ba82\",\"w\":32,\"x\":16,\"y\":0},\"panelIndex\":\"c078d6a7-456e-4fed-80c6-f36123c3ba82\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"1.3.1\",\"gridData\":{\"h\":18,\"i\":\"c04b22a5-6b7e-4c18-8172-d39ec8549e4a\",\"w\":8,\"x\":8,\"y\":10},\"panelIndex\":\"c04b22a5-6b7e-4c18-8172-d39ec8549e4a\",\"embeddableConfig\":{},\"panelRefName\":\"panel_3\"},{\"version\":\"1.3.1\",\"gridData\":{\"h\":18,\"i\":\"4da40cc7-ad85-4dd1-88cf-8b207995c932\",\"w\":12,\"x\":16,\"y\":10},\"panelIndex\":\"4da40cc7-ad85-4dd1-88cf-8b207995c932\",\"embeddableConfig\":{\"table\":null,\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}},\"panelRefName\":\"panel_4\"},{\"version\":\"1.3.1\",\"gridData\":{\"h\":18,\"i\":\"74347ef4-7a00-4d8f-a172-120339fd5e30\",\"w\":20,\"x\":28,\"y\":10},\"panelIndex\":\"74347ef4-7a00-4d8f-a172-120339fd5e30\",\"embeddableConfig\":{},\"panelRefName\":\"panel_5\"},{\"version\":\"1.3.1\",\"gridData\":{\"h\":18,\"i\":\"40ffbd38-1edc-4493-b313-6f65729cbe70\",\"w\":16,\"x\":0,\"y\":28},\"panelIndex\":\"40ffbd38-1edc-4493-b313-6f65729cbe70\",\"embeddableConfig\":{\"vis\":{\"legendOpen\":true}},\"panelRefName\":\"panel_6\"},{\"version\":\"1.3.1\",\"gridData\":{\"h\":18,\"i\":\"2cb13858-f268-4cd4-8207-3932c70dc83a\",\"w\":12,\"x\":16,\"y\":28},\"panelIndex\":\"2cb13858-f268-4cd4-8207-3932c70dc83a\",\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":2,\"direction\":\"desc\"}}},\"table\":null},\"panelRefName\":\"panel_7\"},{\"version\":\"1.3.1\",\"gridData\":{\"h\":18,\"i\":\"7aabaf8b-4a54-48df-ac8e-c732327f420e\",\"w\":20,\"x\":28,\"y\":28},\"panelIndex\":\"7aabaf8b-4a54-48df-ac8e-c732327f420e\",\"embeddableConfig\":{},\"panelRefName\":\"panel_8\"},{\"version\":\"1.3.1\",\"gridData\":{\"h\":31,\"i\":\"6b987e44-72f1-4e33-9fa3-cb21c7313829\",\"w\":48,\"x\":0,\"y\":46},\"panelIndex\":\"6b987e44-72f1-4e33-9fa3-cb21c7313829\",\"embeddableConfig\":{},\"panelRefName\":\"panel_9\"}]", "optionsJSON": "{\"useMargins\":true}", "version": 1, From df51ea0bb7f211ac84d2366bc4ec0a405db98e61 Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 8 May 2023 09:05:03 -0600 Subject: [PATCH 09/47] Use opensearch's new flat_object type. going to revert in a second because there's some other stuff I want to be done in opensearch with this field, but this can be reinstated when we feel ready --- Dockerfiles/dashboards-helper.Dockerfile | 2 +- api/project/__init__.py | 2 +- .../composable/component/miscbeat.json | 21 +- .../composable/component/suricata.json | 181 +----------------- .../templates/composable/component/zeek.json | 11 +- .../composable/component/zeek_ot.json | 24 +-- dashboards/templates/malcolm_template.json | 8 +- 7 files changed, 19 insertions(+), 230 deletions(-) diff --git a/Dockerfiles/dashboards-helper.Dockerfile b/Dockerfiles/dashboards-helper.Dockerfile index 73a410f0e..4429f59e9 100644 --- a/Dockerfiles/dashboards-helper.Dockerfile +++ b/Dockerfiles/dashboards-helper.Dockerfile @@ -92,7 +92,7 @@ RUN apk update --no-cache && \ find /opt/ecs-templates -name "*.json" -exec sed -i 's/\("type"[[:space:]]*:[[:space:]]*\)"match_only_text"/\1"text"/' "{}" \; && \ find /opt/ecs-templates -name "*.json" -exec sed -i 's/\("type"[[:space:]]*:[[:space:]]*\)"constant_keyword"/\1"keyword"/' "{}" \; && \ find /opt/ecs-templates -name "*.json" -exec sed -i 's/\("type"[[:space:]]*:[[:space:]]*\)"wildcard"/\1"keyword"/' "{}" \; && \ - find /opt/ecs-templates -name "*.json" -exec sed -i 's/\("type"[[:space:]]*:[[:space:]]*\)"flattened"/\1"nested"/' "{}" \; && \ + find /opt/ecs-templates -name "*.json" -exec sed -i 's/\("type"[[:space:]]*:[[:space:]]*\)"flattened"/\1"flat_object"/' "{}" \; && \ find /opt/ecs-templates -name "*.json" -exec sed -i 's/\("type"[[:space:]]*:[[:space:]]*\)"number"/\1"long"/' "{}" \; && \ rm -rf /opt/ecs && \ chown -R ${PUSER}:${PGROUP} /opt/dashboards /opt/templates /opt/ecs-templates /opt/maps /data/init /opt/anomaly_detectors && \ diff --git a/api/project/__init__.py b/api/project/__init__.py index 0036bbe9d..feac5da3d 100644 --- a/api/project/__init__.py +++ b/api/project/__init__.py @@ -682,7 +682,7 @@ def fields(): f"{dashboardsUrl}/api/index_patterns/_fields_for_wildcard", params={ 'pattern': pattern, - 'meta_fields': ["_source", "_id", "_type", "_index", "_score"], + 'meta_fields': ["_source", "_id", "_type", "_index", "_score", "_value", "_valueAndPath"], }, auth=opensearchReqHttpAuth, verify=opensearchSslVerify, diff --git a/dashboards/templates/composable/component/miscbeat.json b/dashboards/templates/composable/component/miscbeat.json index 62f272927..8ff507e6f 100644 --- a/dashboards/templates/composable/component/miscbeat.json +++ b/dashboards/templates/composable/component/miscbeat.json @@ -19,16 +19,11 @@ "changed": { "type": "integer" } } }, - "details": { - "type": "nested", - "properties": { - "path": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } } - } - }, + "details": { "type": "flat_object" }, "added": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, "changed": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, "removed": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, - "databases": { "type": "nested" } + "databases": { "type": "flat_object" } } }, "cpu": { @@ -58,15 +53,7 @@ "size": { "type": "long" }, "used": { "type": "long" }, "avail": { "type": "long" }, - "details": { - "type": "nested", - "properties": { - "target": { "type": "keyword" }, - "size": { "type": "long" }, - "used": { "type": "long" }, - "avail": { "type": "long" } - } - } + "details": { "type": "flat_object" } } } } @@ -86,7 +73,7 @@ "drops.rx": { "type": "long" }, "drops.total": { "type": "long" }, "interface": { "type": "keyword" }, - "details": { "type": "nested" } + "details": { "type": "flat_object" } } }, "syslog": { diff --git a/dashboards/templates/composable/component/suricata.json b/dashboards/templates/composable/component/suricata.json index 769713391..e6b464633 100644 --- a/dashboards/templates/composable/component/suricata.json +++ b/dashboards/templates/composable/component/suricata.json @@ -45,73 +45,7 @@ "suricata.dnp3.application.control.sequence": { "type": "integer" }, "suricata.dnp3.application.control.uns": { "type": "keyword" }, "suricata.dnp3.application.function_code": { "type": "integer" }, - "suricata.dnp3.application.objects": { - "type": "nested", - "properties": { - "count": { "type": "integer" }, - "group": { "type": "integer" }, - "prefix_code": { "type": "integer" }, - "qualifier": { "type": "integer" }, - "range_code": { "type": "integer" }, - "start": { "type": "integer" }, - "stop": { "type": "integer" }, - "variation": { "type": "integer" }, - "points": { - "type": "nested", - "properties": { - "authentication_key": { "type": "integer" }, - "block_number": { "type": "integer" }, - "challenge_data_len": { "type": "integer" }, - "chatter_filter": { "type": "integer" }, - "comm_lost": { "type": "integer" }, - "count": { "type": "integer" }, - "cr": { "type": "integer" }, - "created": { "type": "integer" }, - "data.mac_value": { "type": "keyword" }, - "data.wrapped_key_data": { "type": "keyword" }, - "delay_ms": { "type": "integer" }, - "file_data": { "type": "keyword" }, - "file_handle": { "type": "integer" }, - "file_size": { "type": "long" }, - "filename": { "type": "keyword" }, - "filename_offset": { "type": "integer" }, - "filename_size": { "type": "integer" }, - "index": { "type": "integer" }, - "key_status": { "type": "integer" }, - "key_wrap_alg": { "type": "integer" }, - "ksq": { "type": "integer" }, - "local_forced": { "type": "integer" }, - "mal": { "type": "integer" }, - "maximum_block_size": { "type": "integer" }, - "offtime": { "type": "integer" }, - "online": { "type": "integer" }, - "ontime": { "type": "integer" }, - "op_type": { "type": "integer" }, - "operational_mode": { "type": "integer" }, - "optional_text": { "type": "keyword" }, - "over_range": { "type": "integer" }, - "permissions": { "type": "integer" }, - "prefix": { "type": "integer" }, - "qu": { "type": "integer" }, - "reference_err": { "type": "integer" }, - "remote_forced": { "type": "integer" }, - "request_id": { "type": "keyword" }, - "reserved": { "type": "integer" }, - "reserved0": { "type": "integer" }, - "reserved1": { "type": "integer" }, - "restart": { "type": "integer" }, - "size": { "type": "integer" }, - "state": { "type": "integer" }, - "status_code": { "type": "integer" }, - "tcc": { "type": "integer" }, - "timestamp": { "type": "date" }, - "user_number": { "type": "integer" }, - "usr": { "type": "integer" }, - "value": { "type": "keyword" } - } - } - } - }, + "suricata.dnp3.application.objects": { "type": "flat_object" }, "suricata.dnp3.control.dir": { "type": "keyword" }, "suricata.dnp3.control.fcb": { "type": "keyword" }, "suricata.dnp3.control.fcv": { "type": "keyword" }, @@ -126,48 +60,7 @@ "suricata.dnp3.request.application.control.sequence": { "type": "integer" }, "suricata.dnp3.request.application.control.uns": { "type": "keyword" }, "suricata.dnp3.request.application.function_code": { "type": "integer" }, - "suricata.dnp3.request.application.objects": { - "type": "nested", - "properties": { - "count": { "type": "integer" }, - "group": { "type": "integer" }, - "prefix_code": { "type": "integer" }, - "qualifier": { "type": "integer" }, - "range_code": { "type": "integer" }, - "start": { "type": "integer" }, - "stop": { "type": "integer" }, - "variation": { "type": "integer" }, - "points": { - "type": "nested", - "properties": { - "authentication_key": { "type": "integer" }, - "count": { "type": "integer" }, - "cr": { "type": "integer" }, - "created": { "type": "integer" }, - "file_size": { "type": "long" }, - "filename": { "type": "keyword" }, - "filename_offset": { "type": "integer" }, - "filename_size": { "type": "integer" }, - "index": { "type": "integer" }, - "maximum_block_size": { "type": "integer" }, - "offtime": { "type": "integer" }, - "ontime": { "type": "integer" }, - "op_type": { "type": "integer" }, - "operational_mode": { "type": "integer" }, - "permissions": { "type": "integer" }, - "prefix": { "type": "integer" }, - "qu": { "type": "integer" }, - "request_id": { "type": "keyword" }, - "reserved": { "type": "integer" }, - "size": { "type": "integer" }, - "status_code": { "type": "integer" }, - "tcc": { "type": "integer" }, - "timestamp": { "type": "date" }, - "user_number": { "type": "integer" } - } - } - } - }, + "suricata.dnp3.request.application.objects": { "type": "flat_object" }, "suricata.dnp3.request.control.dir": { "type": "keyword" }, "suricata.dnp3.request.control.fcb": { "type": "keyword" }, "suricata.dnp3.request.control.fcv": { "type": "keyword" }, @@ -183,62 +76,7 @@ "suricata.dnp3.response.application.control.sequence": { "type": "integer" }, "suricata.dnp3.response.application.control.uns": { "type": "keyword" }, "suricata.dnp3.response.application.function_code": { "type": "integer" }, - "suricata.dnp3.response.application.objects": { - "type": "nested", - "properties": { - "count": { "type": "integer" }, - "group": { "type": "integer" }, - "prefix_code": { "type": "integer" }, - "qualifier": { "type": "integer" }, - "range_code": { "type": "integer" }, - "start": { "type": "integer" }, - "stop": { "type": "integer" }, - "variation": { "type": "integer" }, - "points": { - "type": "nested", - "properties": { - "challenge_data_len": { "type": "integer" }, - "chatter_filter": { "type": "integer" }, - "comm_lost": { "type": "integer" }, - "count": { "type": "integer" }, - "cr": { "type": "integer" }, - "data.challenge_data": { "type": "keyword" }, - "data.mac_value": { "type": "keyword" }, - "delay_ms": { "type": "integer" }, - "file_handle": { "type": "integer" }, - "file_size": { "type": "long" }, - "index": { "type": "integer" }, - "key_status": { "type": "integer" }, - "key_wrap_alg": { "type": "integer" }, - "ksq": { "type": "integer" }, - "local_forced": { "type": "integer" }, - "mal": { "type": "integer" }, - "maximum_block_size": { "type": "integer" }, - "offtime": { "type": "integer" }, - "online": { "type": "integer" }, - "ontime": { "type": "integer" }, - "op_type": { "type": "integer" }, - "optional_text": { "type": "keyword" }, - "over_range": { "type": "integer" }, - "prefix": { "type": "integer" }, - "qu": { "type": "integer" }, - "reference_err": { "type": "integer" }, - "remote_forced": { "type": "integer" }, - "request_id": { "type": "keyword" }, - "reserved": { "type": "integer" }, - "reserved0": { "type": "integer" }, - "reserved1": { "type": "integer" }, - "restart": { "type": "integer" }, - "size": { "type": "integer" }, - "state": { "type": "integer" }, - "status_code": { "type": "integer" }, - "tcc": { "type": "integer" }, - "user_number": { "type": "integer" }, - "value": { "type": "keyword" } - } - } - } - }, + "suricata.dnp3.response.application.objects": { "type": "flat_object" }, "suricata.dnp3.response.control.dir": { "type": "keyword" }, "suricata.dnp3.response.control.fcb": { "type": "keyword" }, "suricata.dnp3.response.control.fcv": { "type": "keyword" }, @@ -306,18 +144,7 @@ "suricata.fileinfo.start": { "type": "integer" }, "suricata.fileinfo.state": { "type": "keyword" }, "suricata.fileinfo.tx_id": { "type": "keyword" }, - "suricata.files": { - "type": "nested", - "properties": { - "end": { "type": "keyword" }, - "filename": { "type": "keyword" }, - "gaps": { "type": "keyword" }, - "size": { "type": "long" }, - "start": { "type": "keyword" }, - "state": { "type": "keyword" }, - "tx_id": { "type": "keyword" } - } - }, + "suricata.files": { "type": "flat_object" }, "suricata.flow.age": { "type": "integer" }, "suricata.flow.alerted": { "type": "keyword" }, "suricata.flow.bytes_toclient": { "type": "integer" }, diff --git a/dashboards/templates/composable/component/zeek.json b/dashboards/templates/composable/component/zeek.json index ee6a176ca..6d4a73f9d 100644 --- a/dashboards/templates/composable/component/zeek.json +++ b/dashboards/templates/composable/component/zeek.json @@ -346,14 +346,7 @@ "zeek.rfb.server_minor_version": { "type": "keyword" }, "zeek.rfb.share_flag": { "type": "keyword" }, "zeek.rfb.width": { "type": "integer" }, - "zeek.signatures.hits": { - "type": "nested", - "properties": { - "Capa": { "type": "keyword" }, - "ClamAV": { "type": "keyword" }, - "Yara": { "type": "keyword" } - } - }, + "zeek.signatures.hits": { "type": "flat_object" }, "zeek.signatures.host_count": { "type": "integer" }, "zeek.signatures.signature_count": { "type": "integer" }, "zeek.signatures.signature_id": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text", "norms": false } } }, @@ -568,7 +561,7 @@ "zeek.syslog.severity": { "type": "keyword" }, "zeek.tds.command": { "type": "keyword" }, "zeek.tds_rpc.parameter": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text", "norms": false } } }, - "zeek.tds_rpc.parameters": { "type": "nested" }, + "zeek.tds_rpc.parameters": { "type": "flat_object" }, "zeek.tds_rpc.procedure_name": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, "zeek.tds_sql_batch.header_type": { "type": "keyword" }, "zeek.tds_sql_batch.query": { "type": "keyword", "ignore_above": 16384, "fields": { "text": { "type": "text" } } }, diff --git a/dashboards/templates/composable/component/zeek_ot.json b/dashboards/templates/composable/component/zeek_ot.json index 255037c43..fd4cce450 100644 --- a/dashboards/templates/composable/component/zeek_ot.json +++ b/dashboards/templates/composable/component/zeek_ot.json @@ -33,13 +33,7 @@ "zeek.bsap_ip_rdb.res_seq": { "type": "integer" }, "zeek.bsap_ip_rdb.sequence": { "type": "integer" }, "zeek.bsap_ip_rdb.variable_count": { "type": "integer" }, - "zeek.bsap_ip_rdb.variables": { - "type": "nested", - "properties": { - "var": { "type": "keyword" }, - "val": { "type": "keyword" } - } - }, + "zeek.bsap_ip_rdb.variables": { "type": "flat_object" }, "zeek.bsap_serial_header.ctl": { "type": "integer" }, "zeek.bsap_serial_header.dadd": { "type": "integer" }, "zeek.bsap_serial_header.dfun": { "type": "keyword" }, @@ -51,13 +45,7 @@ "zeek.bsap_serial_header.type_name": { "type": "keyword" }, "zeek.bsap_serial_rdb.func_code": { "type": "keyword" }, "zeek.bsap_serial_rdb.variable_count": { "type": "integer" }, - "zeek.bsap_serial_rdb.variables": { - "type": "nested", - "properties": { - "var": { "type": "keyword" }, - "val": { "type": "keyword" } - } - }, + "zeek.bsap_serial_rdb.variables": { "type": "flat_object" }, "zeek.bsap_serial_rdb_ext.data": { "type": "keyword", "ignore_above": 128, "fields": { "text": { "type": "text", "norms": false } } }, "zeek.bsap_serial_rdb_ext.dfun": { "type": "keyword" }, "zeek.bsap_serial_rdb_ext.extfun": { "type": "keyword" }, @@ -186,13 +174,7 @@ "zeek.genisys.direction": { "type": "keyword" }, "zeek.genisys.crc_transmitted": { "type": "keyword" }, "zeek.genisys.crc_calculated": { "type": "keyword" }, - "zeek.genisys.payload": { - "type": "nested", - "properties": { - "address": { "type": "integer" }, - "data": { "type": "integer" } - } - }, + "zeek.genisys.payload": { "type": "flat_object" }, "zeek.known_modbus.device_type": { "type": "keyword" }, "zeek.modbus.exception": { "type": "keyword" }, "zeek.modbus.func": { "type": "keyword" }, diff --git a/dashboards/templates/malcolm_template.json b/dashboards/templates/malcolm_template.json index 36b8e85ea..7e58f26a9 100644 --- a/dashboards/templates/malcolm_template.json +++ b/dashboards/templates/malcolm_template.json @@ -49,7 +49,7 @@ "service": { "type": "keyword" }, "site": { "type": "keyword" }, "url": { "type": "keyword" }, - "details": { "type": "nested" } + "details": { "type": "flat_object" } } }, "destination.segment": { @@ -59,7 +59,7 @@ "site": { "type": "keyword" }, "tenant": { "type": "keyword" }, "url": { "type": "keyword" }, - "details": { "type": "nested" } + "details": { "type": "flat_object" } } }, "event.freq_score_v1": { "type": "float" }, @@ -92,7 +92,7 @@ "service": { "type": "keyword" }, "site": { "type": "keyword" }, "url": { "type": "keyword" }, - "details": { "type": "nested" } + "details": { "type": "flat_object" } } }, "source.segment": { @@ -102,7 +102,7 @@ "site": { "type": "keyword" }, "tenant": { "type": "keyword" }, "url": { "type": "keyword" }, - "details": { "type": "nested" } + "details": { "type": "flat_object" } } }, "tls.client.ja3_description": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, From 8c8503dee3f5d9326b0a60e08ea9fb101b20f852 Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 8 May 2023 09:05:14 -0600 Subject: [PATCH 10/47] Revert "Use opensearch's new flat_object type. going to revert in a second because there's some other stuff I want to be done in opensearch with this field, but this can be reinstated when we feel ready" This reverts commit df51ea0bb7f211ac84d2366bc4ec0a405db98e61. --- Dockerfiles/dashboards-helper.Dockerfile | 2 +- api/project/__init__.py | 2 +- .../composable/component/miscbeat.json | 21 +- .../composable/component/suricata.json | 181 +++++++++++++++++- .../templates/composable/component/zeek.json | 11 +- .../composable/component/zeek_ot.json | 24 ++- dashboards/templates/malcolm_template.json | 8 +- 7 files changed, 230 insertions(+), 19 deletions(-) diff --git a/Dockerfiles/dashboards-helper.Dockerfile b/Dockerfiles/dashboards-helper.Dockerfile index 4429f59e9..73a410f0e 100644 --- a/Dockerfiles/dashboards-helper.Dockerfile +++ b/Dockerfiles/dashboards-helper.Dockerfile @@ -92,7 +92,7 @@ RUN apk update --no-cache && \ find /opt/ecs-templates -name "*.json" -exec sed -i 's/\("type"[[:space:]]*:[[:space:]]*\)"match_only_text"/\1"text"/' "{}" \; && \ find /opt/ecs-templates -name "*.json" -exec sed -i 's/\("type"[[:space:]]*:[[:space:]]*\)"constant_keyword"/\1"keyword"/' "{}" \; && \ find /opt/ecs-templates -name "*.json" -exec sed -i 's/\("type"[[:space:]]*:[[:space:]]*\)"wildcard"/\1"keyword"/' "{}" \; && \ - find /opt/ecs-templates -name "*.json" -exec sed -i 's/\("type"[[:space:]]*:[[:space:]]*\)"flattened"/\1"flat_object"/' "{}" \; && \ + find /opt/ecs-templates -name "*.json" -exec sed -i 's/\("type"[[:space:]]*:[[:space:]]*\)"flattened"/\1"nested"/' "{}" \; && \ find /opt/ecs-templates -name "*.json" -exec sed -i 's/\("type"[[:space:]]*:[[:space:]]*\)"number"/\1"long"/' "{}" \; && \ rm -rf /opt/ecs && \ chown -R ${PUSER}:${PGROUP} /opt/dashboards /opt/templates /opt/ecs-templates /opt/maps /data/init /opt/anomaly_detectors && \ diff --git a/api/project/__init__.py b/api/project/__init__.py index feac5da3d..0036bbe9d 100644 --- a/api/project/__init__.py +++ b/api/project/__init__.py @@ -682,7 +682,7 @@ def fields(): f"{dashboardsUrl}/api/index_patterns/_fields_for_wildcard", params={ 'pattern': pattern, - 'meta_fields': ["_source", "_id", "_type", "_index", "_score", "_value", "_valueAndPath"], + 'meta_fields': ["_source", "_id", "_type", "_index", "_score"], }, auth=opensearchReqHttpAuth, verify=opensearchSslVerify, diff --git a/dashboards/templates/composable/component/miscbeat.json b/dashboards/templates/composable/component/miscbeat.json index 8ff507e6f..62f272927 100644 --- a/dashboards/templates/composable/component/miscbeat.json +++ b/dashboards/templates/composable/component/miscbeat.json @@ -19,11 +19,16 @@ "changed": { "type": "integer" } } }, - "details": { "type": "flat_object" }, + "details": { + "type": "nested", + "properties": { + "path": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } } + } + }, "added": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, "changed": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, "removed": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, - "databases": { "type": "flat_object" } + "databases": { "type": "nested" } } }, "cpu": { @@ -53,7 +58,15 @@ "size": { "type": "long" }, "used": { "type": "long" }, "avail": { "type": "long" }, - "details": { "type": "flat_object" } + "details": { + "type": "nested", + "properties": { + "target": { "type": "keyword" }, + "size": { "type": "long" }, + "used": { "type": "long" }, + "avail": { "type": "long" } + } + } } } } @@ -73,7 +86,7 @@ "drops.rx": { "type": "long" }, "drops.total": { "type": "long" }, "interface": { "type": "keyword" }, - "details": { "type": "flat_object" } + "details": { "type": "nested" } } }, "syslog": { diff --git a/dashboards/templates/composable/component/suricata.json b/dashboards/templates/composable/component/suricata.json index e6b464633..769713391 100644 --- a/dashboards/templates/composable/component/suricata.json +++ b/dashboards/templates/composable/component/suricata.json @@ -45,7 +45,73 @@ "suricata.dnp3.application.control.sequence": { "type": "integer" }, "suricata.dnp3.application.control.uns": { "type": "keyword" }, "suricata.dnp3.application.function_code": { "type": "integer" }, - "suricata.dnp3.application.objects": { "type": "flat_object" }, + "suricata.dnp3.application.objects": { + "type": "nested", + "properties": { + "count": { "type": "integer" }, + "group": { "type": "integer" }, + "prefix_code": { "type": "integer" }, + "qualifier": { "type": "integer" }, + "range_code": { "type": "integer" }, + "start": { "type": "integer" }, + "stop": { "type": "integer" }, + "variation": { "type": "integer" }, + "points": { + "type": "nested", + "properties": { + "authentication_key": { "type": "integer" }, + "block_number": { "type": "integer" }, + "challenge_data_len": { "type": "integer" }, + "chatter_filter": { "type": "integer" }, + "comm_lost": { "type": "integer" }, + "count": { "type": "integer" }, + "cr": { "type": "integer" }, + "created": { "type": "integer" }, + "data.mac_value": { "type": "keyword" }, + "data.wrapped_key_data": { "type": "keyword" }, + "delay_ms": { "type": "integer" }, + "file_data": { "type": "keyword" }, + "file_handle": { "type": "integer" }, + "file_size": { "type": "long" }, + "filename": { "type": "keyword" }, + "filename_offset": { "type": "integer" }, + "filename_size": { "type": "integer" }, + "index": { "type": "integer" }, + "key_status": { "type": "integer" }, + "key_wrap_alg": { "type": "integer" }, + "ksq": { "type": "integer" }, + "local_forced": { "type": "integer" }, + "mal": { "type": "integer" }, + "maximum_block_size": { "type": "integer" }, + "offtime": { "type": "integer" }, + "online": { "type": "integer" }, + "ontime": { "type": "integer" }, + "op_type": { "type": "integer" }, + "operational_mode": { "type": "integer" }, + "optional_text": { "type": "keyword" }, + "over_range": { "type": "integer" }, + "permissions": { "type": "integer" }, + "prefix": { "type": "integer" }, + "qu": { "type": "integer" }, + "reference_err": { "type": "integer" }, + "remote_forced": { "type": "integer" }, + "request_id": { "type": "keyword" }, + "reserved": { "type": "integer" }, + "reserved0": { "type": "integer" }, + "reserved1": { "type": "integer" }, + "restart": { "type": "integer" }, + "size": { "type": "integer" }, + "state": { "type": "integer" }, + "status_code": { "type": "integer" }, + "tcc": { "type": "integer" }, + "timestamp": { "type": "date" }, + "user_number": { "type": "integer" }, + "usr": { "type": "integer" }, + "value": { "type": "keyword" } + } + } + } + }, "suricata.dnp3.control.dir": { "type": "keyword" }, "suricata.dnp3.control.fcb": { "type": "keyword" }, "suricata.dnp3.control.fcv": { "type": "keyword" }, @@ -60,7 +126,48 @@ "suricata.dnp3.request.application.control.sequence": { "type": "integer" }, "suricata.dnp3.request.application.control.uns": { "type": "keyword" }, "suricata.dnp3.request.application.function_code": { "type": "integer" }, - "suricata.dnp3.request.application.objects": { "type": "flat_object" }, + "suricata.dnp3.request.application.objects": { + "type": "nested", + "properties": { + "count": { "type": "integer" }, + "group": { "type": "integer" }, + "prefix_code": { "type": "integer" }, + "qualifier": { "type": "integer" }, + "range_code": { "type": "integer" }, + "start": { "type": "integer" }, + "stop": { "type": "integer" }, + "variation": { "type": "integer" }, + "points": { + "type": "nested", + "properties": { + "authentication_key": { "type": "integer" }, + "count": { "type": "integer" }, + "cr": { "type": "integer" }, + "created": { "type": "integer" }, + "file_size": { "type": "long" }, + "filename": { "type": "keyword" }, + "filename_offset": { "type": "integer" }, + "filename_size": { "type": "integer" }, + "index": { "type": "integer" }, + "maximum_block_size": { "type": "integer" }, + "offtime": { "type": "integer" }, + "ontime": { "type": "integer" }, + "op_type": { "type": "integer" }, + "operational_mode": { "type": "integer" }, + "permissions": { "type": "integer" }, + "prefix": { "type": "integer" }, + "qu": { "type": "integer" }, + "request_id": { "type": "keyword" }, + "reserved": { "type": "integer" }, + "size": { "type": "integer" }, + "status_code": { "type": "integer" }, + "tcc": { "type": "integer" }, + "timestamp": { "type": "date" }, + "user_number": { "type": "integer" } + } + } + } + }, "suricata.dnp3.request.control.dir": { "type": "keyword" }, "suricata.dnp3.request.control.fcb": { "type": "keyword" }, "suricata.dnp3.request.control.fcv": { "type": "keyword" }, @@ -76,7 +183,62 @@ "suricata.dnp3.response.application.control.sequence": { "type": "integer" }, "suricata.dnp3.response.application.control.uns": { "type": "keyword" }, "suricata.dnp3.response.application.function_code": { "type": "integer" }, - "suricata.dnp3.response.application.objects": { "type": "flat_object" }, + "suricata.dnp3.response.application.objects": { + "type": "nested", + "properties": { + "count": { "type": "integer" }, + "group": { "type": "integer" }, + "prefix_code": { "type": "integer" }, + "qualifier": { "type": "integer" }, + "range_code": { "type": "integer" }, + "start": { "type": "integer" }, + "stop": { "type": "integer" }, + "variation": { "type": "integer" }, + "points": { + "type": "nested", + "properties": { + "challenge_data_len": { "type": "integer" }, + "chatter_filter": { "type": "integer" }, + "comm_lost": { "type": "integer" }, + "count": { "type": "integer" }, + "cr": { "type": "integer" }, + "data.challenge_data": { "type": "keyword" }, + "data.mac_value": { "type": "keyword" }, + "delay_ms": { "type": "integer" }, + "file_handle": { "type": "integer" }, + "file_size": { "type": "long" }, + "index": { "type": "integer" }, + "key_status": { "type": "integer" }, + "key_wrap_alg": { "type": "integer" }, + "ksq": { "type": "integer" }, + "local_forced": { "type": "integer" }, + "mal": { "type": "integer" }, + "maximum_block_size": { "type": "integer" }, + "offtime": { "type": "integer" }, + "online": { "type": "integer" }, + "ontime": { "type": "integer" }, + "op_type": { "type": "integer" }, + "optional_text": { "type": "keyword" }, + "over_range": { "type": "integer" }, + "prefix": { "type": "integer" }, + "qu": { "type": "integer" }, + "reference_err": { "type": "integer" }, + "remote_forced": { "type": "integer" }, + "request_id": { "type": "keyword" }, + "reserved": { "type": "integer" }, + "reserved0": { "type": "integer" }, + "reserved1": { "type": "integer" }, + "restart": { "type": "integer" }, + "size": { "type": "integer" }, + "state": { "type": "integer" }, + "status_code": { "type": "integer" }, + "tcc": { "type": "integer" }, + "user_number": { "type": "integer" }, + "value": { "type": "keyword" } + } + } + } + }, "suricata.dnp3.response.control.dir": { "type": "keyword" }, "suricata.dnp3.response.control.fcb": { "type": "keyword" }, "suricata.dnp3.response.control.fcv": { "type": "keyword" }, @@ -144,7 +306,18 @@ "suricata.fileinfo.start": { "type": "integer" }, "suricata.fileinfo.state": { "type": "keyword" }, "suricata.fileinfo.tx_id": { "type": "keyword" }, - "suricata.files": { "type": "flat_object" }, + "suricata.files": { + "type": "nested", + "properties": { + "end": { "type": "keyword" }, + "filename": { "type": "keyword" }, + "gaps": { "type": "keyword" }, + "size": { "type": "long" }, + "start": { "type": "keyword" }, + "state": { "type": "keyword" }, + "tx_id": { "type": "keyword" } + } + }, "suricata.flow.age": { "type": "integer" }, "suricata.flow.alerted": { "type": "keyword" }, "suricata.flow.bytes_toclient": { "type": "integer" }, diff --git a/dashboards/templates/composable/component/zeek.json b/dashboards/templates/composable/component/zeek.json index 6d4a73f9d..ee6a176ca 100644 --- a/dashboards/templates/composable/component/zeek.json +++ b/dashboards/templates/composable/component/zeek.json @@ -346,7 +346,14 @@ "zeek.rfb.server_minor_version": { "type": "keyword" }, "zeek.rfb.share_flag": { "type": "keyword" }, "zeek.rfb.width": { "type": "integer" }, - "zeek.signatures.hits": { "type": "flat_object" }, + "zeek.signatures.hits": { + "type": "nested", + "properties": { + "Capa": { "type": "keyword" }, + "ClamAV": { "type": "keyword" }, + "Yara": { "type": "keyword" } + } + }, "zeek.signatures.host_count": { "type": "integer" }, "zeek.signatures.signature_count": { "type": "integer" }, "zeek.signatures.signature_id": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text", "norms": false } } }, @@ -561,7 +568,7 @@ "zeek.syslog.severity": { "type": "keyword" }, "zeek.tds.command": { "type": "keyword" }, "zeek.tds_rpc.parameter": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text", "norms": false } } }, - "zeek.tds_rpc.parameters": { "type": "flat_object" }, + "zeek.tds_rpc.parameters": { "type": "nested" }, "zeek.tds_rpc.procedure_name": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, "zeek.tds_sql_batch.header_type": { "type": "keyword" }, "zeek.tds_sql_batch.query": { "type": "keyword", "ignore_above": 16384, "fields": { "text": { "type": "text" } } }, diff --git a/dashboards/templates/composable/component/zeek_ot.json b/dashboards/templates/composable/component/zeek_ot.json index fd4cce450..255037c43 100644 --- a/dashboards/templates/composable/component/zeek_ot.json +++ b/dashboards/templates/composable/component/zeek_ot.json @@ -33,7 +33,13 @@ "zeek.bsap_ip_rdb.res_seq": { "type": "integer" }, "zeek.bsap_ip_rdb.sequence": { "type": "integer" }, "zeek.bsap_ip_rdb.variable_count": { "type": "integer" }, - "zeek.bsap_ip_rdb.variables": { "type": "flat_object" }, + "zeek.bsap_ip_rdb.variables": { + "type": "nested", + "properties": { + "var": { "type": "keyword" }, + "val": { "type": "keyword" } + } + }, "zeek.bsap_serial_header.ctl": { "type": "integer" }, "zeek.bsap_serial_header.dadd": { "type": "integer" }, "zeek.bsap_serial_header.dfun": { "type": "keyword" }, @@ -45,7 +51,13 @@ "zeek.bsap_serial_header.type_name": { "type": "keyword" }, "zeek.bsap_serial_rdb.func_code": { "type": "keyword" }, "zeek.bsap_serial_rdb.variable_count": { "type": "integer" }, - "zeek.bsap_serial_rdb.variables": { "type": "flat_object" }, + "zeek.bsap_serial_rdb.variables": { + "type": "nested", + "properties": { + "var": { "type": "keyword" }, + "val": { "type": "keyword" } + } + }, "zeek.bsap_serial_rdb_ext.data": { "type": "keyword", "ignore_above": 128, "fields": { "text": { "type": "text", "norms": false } } }, "zeek.bsap_serial_rdb_ext.dfun": { "type": "keyword" }, "zeek.bsap_serial_rdb_ext.extfun": { "type": "keyword" }, @@ -174,7 +186,13 @@ "zeek.genisys.direction": { "type": "keyword" }, "zeek.genisys.crc_transmitted": { "type": "keyword" }, "zeek.genisys.crc_calculated": { "type": "keyword" }, - "zeek.genisys.payload": { "type": "flat_object" }, + "zeek.genisys.payload": { + "type": "nested", + "properties": { + "address": { "type": "integer" }, + "data": { "type": "integer" } + } + }, "zeek.known_modbus.device_type": { "type": "keyword" }, "zeek.modbus.exception": { "type": "keyword" }, "zeek.modbus.func": { "type": "keyword" }, diff --git a/dashboards/templates/malcolm_template.json b/dashboards/templates/malcolm_template.json index 7e58f26a9..36b8e85ea 100644 --- a/dashboards/templates/malcolm_template.json +++ b/dashboards/templates/malcolm_template.json @@ -49,7 +49,7 @@ "service": { "type": "keyword" }, "site": { "type": "keyword" }, "url": { "type": "keyword" }, - "details": { "type": "flat_object" } + "details": { "type": "nested" } } }, "destination.segment": { @@ -59,7 +59,7 @@ "site": { "type": "keyword" }, "tenant": { "type": "keyword" }, "url": { "type": "keyword" }, - "details": { "type": "flat_object" } + "details": { "type": "nested" } } }, "event.freq_score_v1": { "type": "float" }, @@ -92,7 +92,7 @@ "service": { "type": "keyword" }, "site": { "type": "keyword" }, "url": { "type": "keyword" }, - "details": { "type": "flat_object" } + "details": { "type": "nested" } } }, "source.segment": { @@ -102,7 +102,7 @@ "site": { "type": "keyword" }, "tenant": { "type": "keyword" }, "url": { "type": "keyword" }, - "details": { "type": "flat_object" } + "details": { "type": "nested" } } }, "tls.client.ja3_description": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, From abfcfc465c728f281e6cd22ffb4eec65a10fb178 Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 8 May 2023 12:46:16 -0600 Subject: [PATCH 11/47] change some defaults for opensearch and logstash resources --- config/logstash.env.example | 2 +- config/opensearch.env.example | 2 +- scripts/install.py | 14 +++++++------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/config/logstash.env.example b/config/logstash.env.example index f8aa43202..2e640b86a 100644 --- a/config/logstash.env.example +++ b/config/logstash.env.example @@ -12,4 +12,4 @@ LOGSTASH_REVERSE_DNS=false # Whether or not Logstash will enrich network traffic metadata via NetBox API calls LOGSTASH_NETBOX_ENRICHMENT=false # Logstash memory allowance and other Java options -LS_JAVA_OPTS=-server -Xms2g -Xmx2g -Xss1536k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true \ No newline at end of file +LS_JAVA_OPTS=-server -Xms2500m -Xmx2500m -Xss1536k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true \ No newline at end of file diff --git a/config/opensearch.env.example b/config/opensearch.env.example index 258e23233..9503d9078 100644 --- a/config/opensearch.env.example +++ b/config/opensearch.env.example @@ -37,7 +37,7 @@ OPENSEARCH_SECONDARY_CREDS_CONFIG_FILE=/var/local/curlrc/.opensearch.secondary.c # certificates). OPENSEARCH_SECONDARY_SSL_CERTIFICATE_VERIFICATION=false # OpenSearch memory allowance and other Java options -OPENSEARCH_JAVA_OPTS=-server -Xms4g -Xmx4g -Xss256k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true +OPENSEARCH_JAVA_OPTS=-server -Xms10g -Xmx10g -Xss256k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true logger.level=WARN bootstrap.memory_lock=true diff --git a/scripts/install.py b/scripts/install.py index 48e1596b8..8506a26bd 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -438,21 +438,21 @@ def tweak_malcolm_runtime( ) if self.totalMemoryGigs >= 63.0: - osMemory = '30g' - lsMemory = '6g' + osMemory = '24g' + lsMemory = '3g' elif self.totalMemoryGigs >= 31.0: osMemory = '16g' - lsMemory = '3g' + lsMemory = '2500m' elif self.totalMemoryGigs >= 15.0: osMemory = '10g' lsMemory = '2500m' elif self.totalMemoryGigs >= 11.0: osMemory = '6g' - lsMemory = '2500m' + lsMemory = '2g' elif self.totalMemoryGigs >= 7.0: eprint(f"Detected only {self.totalMemoryGigs} GiB of memory; performance will be suboptimal") osMemory = '4g' - lsMemory = '2500m' + lsMemory = '2g' elif self.totalMemoryGigs > 0.0: eprint(f"Detected only {self.totalMemoryGigs} GiB of memory; performance will be suboptimal") osMemory = '3500m' @@ -472,9 +472,9 @@ def tweak_malcolm_runtime( # we don't want it too high, as in Malcolm Logstash also competes with OpenSearch, etc. for resources if self.orchMode is OrchestrationFramework.DOCKER_COMPOSE: if self.totalCores > 16: - lsWorkers = 10 - elif self.totalCores >= 12: lsWorkers = 6 + elif self.totalCores >= 12: + lsWorkers = 4 else: lsWorkers = 3 else: From 91f78b9821f7d7d83d4340604e7f685e5e966433 Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 8 May 2023 12:55:22 -0600 Subject: [PATCH 12/47] added localhost malcolm cpu reporting services, disabled by default --- .../systemd/user/cpu-localhost-malcolm.service | 12 ++++++++++++ .../systemd/user/df-localhost-malcolm.service | 12 ++++++++++++ .../systemd/user/disk-localhost-malcolm.service | 12 ++++++++++++ .../systemd/user/mem-localhost-malcolm.service | 12 ++++++++++++ .../systemd/user/memp-localhost-malcolm.service | 12 ++++++++++++ .../systemd/user/network-localhost-malcolm.service | 12 ++++++++++++ .../systemd/user/thermal-localhost-malcolm.service | 12 ++++++++++++ shared/bin/common-init.sh | 1 + 8 files changed, 85 insertions(+) create mode 100644 malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/cpu-localhost-malcolm.service create mode 100644 malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/df-localhost-malcolm.service create mode 100644 malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/disk-localhost-malcolm.service create mode 100644 malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/mem-localhost-malcolm.service create mode 100644 malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/memp-localhost-malcolm.service create mode 100644 malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/network-localhost-malcolm.service create mode 100644 malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/thermal-localhost-malcolm.service diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/cpu-localhost-malcolm.service b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/cpu-localhost-malcolm.service new file mode 100644 index 000000000..eaba5461d --- /dev/null +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/cpu-localhost-malcolm.service @@ -0,0 +1,12 @@ +[Unit] +AssertPathExists=/usr/local/bin/fluent-bit +After=network.target + +[Service] +ExecStart=/usr/local/bin/fluent-bit -R /etc/fluent-bit/parsers.conf -i cpu -p Interval_Sec=30 -o tcp://localhost:5045 -p tls=on -p tls.verify=off -p tls.ca_file=%h/Malcolm/filebeat/certs/ca.crt -p tls.crt_file=%h/Malcolm/filebeat/certs/client.crt -p tls.key_file=%h/Malcolm/filebeat/certs/client.key -p format=json_lines -F nest -p Operation=nest -p Nested_under=cpu -p WildCard='*' -m '*' -F record_modifier -p 'Record=module cpu' -m '*' -f 1 +Restart=on-failure +PrivateTmp=false +NoNewPrivileges=false + +[Install] +WantedBy=default.target diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/df-localhost-malcolm.service b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/df-localhost-malcolm.service new file mode 100644 index 000000000..e7ab72aae --- /dev/null +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/df-localhost-malcolm.service @@ -0,0 +1,12 @@ +[Unit] +AssertPathExists=/usr/local/bin/fluent-bit +After=network.target + +[Service] +ExecStart=/usr/local/bin/fluent-bit -R /etc/fluent-bit/parsers.conf -i exec -p Parser=json -p Command=/usr/local/bin/df-json.sh -p Interval_Sec=30 -o tcp://localhost:5045 -p tls=on -p tls.verify=off -p tls.ca_file=%h/Malcolm/filebeat/certs/ca.crt -p tls.crt_file=%h/Malcolm/filebeat/certs/client.crt -p tls.key_file=%h/Malcolm/filebeat/certs/client.key -p format=json_lines -F nest -p Operation=nest -p Nested_under=disk -p WildCard='*' -m '*' -F record_modifier -p 'Record=module disk' -m '*' -f 1 +Restart=on-failure +PrivateTmp=false +NoNewPrivileges=false + +[Install] +WantedBy=default.target diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/disk-localhost-malcolm.service b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/disk-localhost-malcolm.service new file mode 100644 index 000000000..d49842c6e --- /dev/null +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/disk-localhost-malcolm.service @@ -0,0 +1,12 @@ +[Unit] +AssertPathExists=/usr/local/bin/fluent-bit +After=network.target + +[Service] +ExecStart=/usr/local/bin/fluent-bit -R /etc/fluent-bit/parsers.conf -i disk -p Interval_Sec=30 -o tcp://localhost:5045 -p tls=on -p tls.verify=off -p tls.ca_file=%h/Malcolm/filebeat/certs/ca.crt -p tls.crt_file=%h/Malcolm/filebeat/certs/client.crt -p tls.key_file=%h/Malcolm/filebeat/certs/client.key -p format=json_lines -F nest -p Operation=nest -p Nested_under=disk -p WildCard='*' -m '*' -F record_modifier -p 'Record=module disk' -m '*' -f 1 +Restart=on-failure +PrivateTmp=false +NoNewPrivileges=false + +[Install] +WantedBy=default.target diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/mem-localhost-malcolm.service b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/mem-localhost-malcolm.service new file mode 100644 index 000000000..58c558ad7 --- /dev/null +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/mem-localhost-malcolm.service @@ -0,0 +1,12 @@ +[Unit] +AssertPathExists=/usr/local/bin/fluent-bit +After=network.target + +[Service] +ExecStart=/usr/local/bin/fluent-bit -R /etc/fluent-bit/parsers.conf -i mem -p Interval_Sec=30 -o tcp://localhost:5045 -p tls=on -p tls.verify=off -p tls.ca_file=%h/Malcolm/filebeat/certs/ca.crt -p tls.crt_file=%h/Malcolm/filebeat/certs/client.crt -p tls.key_file=%h/Malcolm/filebeat/certs/client.key -p format=json_lines -F nest -p Operation=nest -p Nested_under=mem -p WildCard='*' -m '*' -F record_modifier -p 'Record=module mem' -m '*' -f 1 +Restart=on-failure +PrivateTmp=false +NoNewPrivileges=false + +[Install] +WantedBy=default.target diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/memp-localhost-malcolm.service b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/memp-localhost-malcolm.service new file mode 100644 index 000000000..d8c01cf2f --- /dev/null +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/memp-localhost-malcolm.service @@ -0,0 +1,12 @@ +[Unit] +AssertPathExists=/usr/local/bin/fluent-bit +After=network.target + +[Service] +ExecStart=/usr/local/bin/fluent-bit -R /etc/fluent-bit/parsers.conf -i exec -p Command=/usr/local/bin/memory_usage_percentage.sh -p Interval_Sec=30 -o tcp://localhost:5045 -p tls=on -p tls.verify=off -p tls.ca_file=%h/Malcolm/filebeat/certs/ca.crt -p tls.crt_file=%h/Malcolm/filebeat/certs/client.crt -p tls.key_file=%h/Malcolm/filebeat/certs/client.key -p format=json_lines -F modify -p "Hard_rename=exec Mem.used_p" -m '*' -F nest -p Operation=nest -p Nested_under=mem -p WildCard='*' -m '*' -F record_modifier -p 'Record=module mem' -m '*' -f 1 +Restart=on-failure +PrivateTmp=false +NoNewPrivileges=false + +[Install] +WantedBy=default.target diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/network-localhost-malcolm.service b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/network-localhost-malcolm.service new file mode 100644 index 000000000..885117c49 --- /dev/null +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/network-localhost-malcolm.service @@ -0,0 +1,12 @@ +[Unit] +AssertPathExists=/usr/local/bin/fluent-bit +After=network.target + +[Service] +ExecStart=/usr/local/bin/fluent-bit -R /etc/fluent-bit/parsers.conf -i exec -p Parser=json -p Command=/usr/local/bin/netdev-json.sh -p Interval_Sec=30 -o tcp://localhost:5045 -p tls=on -p tls.verify=off -p tls.ca_file=%h/Malcolm/filebeat/certs/ca.crt -p tls.crt_file=%h/Malcolm/filebeat/certs/client.crt -p tls.key_file=%h/Malcolm/filebeat/certs/client.key -p format=json_lines -F nest -p Operation=nest -p Nested_under=network -p WildCard='*' -m '*' -F record_modifier -p 'Record=module network' -m '*' -f 1 +Restart=on-failure +PrivateTmp=false +NoNewPrivileges=false + +[Install] +WantedBy=default.target diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/thermal-localhost-malcolm.service b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/thermal-localhost-malcolm.service new file mode 100644 index 000000000..bb3e63e01 --- /dev/null +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/thermal-localhost-malcolm.service @@ -0,0 +1,12 @@ +[Unit] +AssertPathExists=/usr/local/bin/fluent-bit +After=network.target + +[Service] +ExecStart=/usr/local/bin/fluent-bit -R /etc/fluent-bit/parsers.conf -i thermal -p Interval_Sec=10 -o tcp://localhost:5045 -p tls=on -p tls.verify=off -p tls.ca_file=%h/Malcolm/filebeat/certs/ca.crt -p tls.crt_file=%h/Malcolm/filebeat/certs/client.crt -p tls.key_file=%h/Malcolm/filebeat/certs/client.key -p format=json_lines -F nest -p Operation=nest -p Nested_under=thermal -p WildCard='*' -m '*' -F record_modifier -p 'Record=module thermal' -m '*' -f 1 +Restart=on-failure +PrivateTmp=false +NoNewPrivileges=false + +[Install] +WantedBy=default.target diff --git a/shared/bin/common-init.sh b/shared/bin/common-init.sh index 838cc52c9..ddc0bbd26 100755 --- a/shared/bin/common-init.sh +++ b/shared/bin/common-init.sh @@ -105,6 +105,7 @@ function FixPermissions() { echo "$USER_TO_FIX" >> /etc/at.allow fi chmod 644 /etc/cron.allow /etc/at.allow + loginctl enable-linger "$USER_TO_FIX" 2>/dev/null || true fi } From dc6a29d54fb896a41bbd6495163abc57fe5d1180 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 8 May 2023 15:01:16 -0600 Subject: [PATCH 13/47] fluent-bit bin location for malcolm --- malcolm-iso/config/includes.chroot/etc/skel/.bashrc | 4 ++++ .../skel/.config/systemd/user/cpu-localhost-malcolm.service | 3 +-- .../skel/.config/systemd/user/df-localhost-malcolm.service | 3 +-- .../skel/.config/systemd/user/disk-localhost-malcolm.service | 3 +-- .../skel/.config/systemd/user/mem-localhost-malcolm.service | 3 +-- .../skel/.config/systemd/user/memp-localhost-malcolm.service | 3 +-- .../.config/systemd/user/network-localhost-malcolm.service | 3 +-- .../.config/systemd/user/thermal-localhost-malcolm.service | 3 +-- 8 files changed, 11 insertions(+), 14 deletions(-) diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.bashrc b/malcolm-iso/config/includes.chroot/etc/skel/.bashrc index a4b80d247..a18e760e7 100644 --- a/malcolm-iso/config/includes.chroot/etc/skel/.bashrc +++ b/malcolm-iso/config/includes.chroot/etc/skel/.bashrc @@ -40,6 +40,10 @@ fi ############################################################################### # PATH ############################################################################### +if [ -d /opt/fluent-bit/bin ]; then + PATH=/opt/fluent-bit/bin:$PATH +fi + if [ -d ~/bin ]; then PATH=~/bin:$PATH fi diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/cpu-localhost-malcolm.service b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/cpu-localhost-malcolm.service index eaba5461d..683e0ecf6 100644 --- a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/cpu-localhost-malcolm.service +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/cpu-localhost-malcolm.service @@ -1,9 +1,8 @@ [Unit] -AssertPathExists=/usr/local/bin/fluent-bit After=network.target [Service] -ExecStart=/usr/local/bin/fluent-bit -R /etc/fluent-bit/parsers.conf -i cpu -p Interval_Sec=30 -o tcp://localhost:5045 -p tls=on -p tls.verify=off -p tls.ca_file=%h/Malcolm/filebeat/certs/ca.crt -p tls.crt_file=%h/Malcolm/filebeat/certs/client.crt -p tls.key_file=%h/Malcolm/filebeat/certs/client.key -p format=json_lines -F nest -p Operation=nest -p Nested_under=cpu -p WildCard='*' -m '*' -F record_modifier -p 'Record=module cpu' -m '*' -f 1 +ExecStart=/opt/fluent-bit/bin/fluent-bit -R /etc/fluent-bit/parsers.conf -i cpu -p Interval_Sec=30 -o tcp://localhost:5045 -p tls=on -p tls.verify=off -p tls.ca_file=%h/Malcolm/filebeat/certs/ca.crt -p tls.crt_file=%h/Malcolm/filebeat/certs/client.crt -p tls.key_file=%h/Malcolm/filebeat/certs/client.key -p format=json_lines -F nest -p Operation=nest -p Nested_under=cpu -p WildCard='*' -m '*' -F record_modifier -p 'Record=module cpu' -m '*' -f 1 Restart=on-failure PrivateTmp=false NoNewPrivileges=false diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/df-localhost-malcolm.service b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/df-localhost-malcolm.service index e7ab72aae..c7fbc3c75 100644 --- a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/df-localhost-malcolm.service +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/df-localhost-malcolm.service @@ -1,9 +1,8 @@ [Unit] -AssertPathExists=/usr/local/bin/fluent-bit After=network.target [Service] -ExecStart=/usr/local/bin/fluent-bit -R /etc/fluent-bit/parsers.conf -i exec -p Parser=json -p Command=/usr/local/bin/df-json.sh -p Interval_Sec=30 -o tcp://localhost:5045 -p tls=on -p tls.verify=off -p tls.ca_file=%h/Malcolm/filebeat/certs/ca.crt -p tls.crt_file=%h/Malcolm/filebeat/certs/client.crt -p tls.key_file=%h/Malcolm/filebeat/certs/client.key -p format=json_lines -F nest -p Operation=nest -p Nested_under=disk -p WildCard='*' -m '*' -F record_modifier -p 'Record=module disk' -m '*' -f 1 +ExecStart=/opt/fluent-bit/bin/fluent-bit -R /etc/fluent-bit/parsers.conf -i exec -p Parser=json -p Command=/usr/local/bin/df-json.sh -p Interval_Sec=30 -o tcp://localhost:5045 -p tls=on -p tls.verify=off -p tls.ca_file=%h/Malcolm/filebeat/certs/ca.crt -p tls.crt_file=%h/Malcolm/filebeat/certs/client.crt -p tls.key_file=%h/Malcolm/filebeat/certs/client.key -p format=json_lines -F nest -p Operation=nest -p Nested_under=disk -p WildCard='*' -m '*' -F record_modifier -p 'Record=module disk' -m '*' -f 1 Restart=on-failure PrivateTmp=false NoNewPrivileges=false diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/disk-localhost-malcolm.service b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/disk-localhost-malcolm.service index d49842c6e..dc4f4dff2 100644 --- a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/disk-localhost-malcolm.service +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/disk-localhost-malcolm.service @@ -1,9 +1,8 @@ [Unit] -AssertPathExists=/usr/local/bin/fluent-bit After=network.target [Service] -ExecStart=/usr/local/bin/fluent-bit -R /etc/fluent-bit/parsers.conf -i disk -p Interval_Sec=30 -o tcp://localhost:5045 -p tls=on -p tls.verify=off -p tls.ca_file=%h/Malcolm/filebeat/certs/ca.crt -p tls.crt_file=%h/Malcolm/filebeat/certs/client.crt -p tls.key_file=%h/Malcolm/filebeat/certs/client.key -p format=json_lines -F nest -p Operation=nest -p Nested_under=disk -p WildCard='*' -m '*' -F record_modifier -p 'Record=module disk' -m '*' -f 1 +ExecStart=/opt/fluent-bit/bin/fluent-bit -R /etc/fluent-bit/parsers.conf -i disk -p Interval_Sec=30 -o tcp://localhost:5045 -p tls=on -p tls.verify=off -p tls.ca_file=%h/Malcolm/filebeat/certs/ca.crt -p tls.crt_file=%h/Malcolm/filebeat/certs/client.crt -p tls.key_file=%h/Malcolm/filebeat/certs/client.key -p format=json_lines -F nest -p Operation=nest -p Nested_under=disk -p WildCard='*' -m '*' -F record_modifier -p 'Record=module disk' -m '*' -f 1 Restart=on-failure PrivateTmp=false NoNewPrivileges=false diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/mem-localhost-malcolm.service b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/mem-localhost-malcolm.service index 58c558ad7..64eff358d 100644 --- a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/mem-localhost-malcolm.service +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/mem-localhost-malcolm.service @@ -1,9 +1,8 @@ [Unit] -AssertPathExists=/usr/local/bin/fluent-bit After=network.target [Service] -ExecStart=/usr/local/bin/fluent-bit -R /etc/fluent-bit/parsers.conf -i mem -p Interval_Sec=30 -o tcp://localhost:5045 -p tls=on -p tls.verify=off -p tls.ca_file=%h/Malcolm/filebeat/certs/ca.crt -p tls.crt_file=%h/Malcolm/filebeat/certs/client.crt -p tls.key_file=%h/Malcolm/filebeat/certs/client.key -p format=json_lines -F nest -p Operation=nest -p Nested_under=mem -p WildCard='*' -m '*' -F record_modifier -p 'Record=module mem' -m '*' -f 1 +ExecStart=/opt/fluent-bit/bin/fluent-bit -R /etc/fluent-bit/parsers.conf -i mem -p Interval_Sec=30 -o tcp://localhost:5045 -p tls=on -p tls.verify=off -p tls.ca_file=%h/Malcolm/filebeat/certs/ca.crt -p tls.crt_file=%h/Malcolm/filebeat/certs/client.crt -p tls.key_file=%h/Malcolm/filebeat/certs/client.key -p format=json_lines -F nest -p Operation=nest -p Nested_under=mem -p WildCard='*' -m '*' -F record_modifier -p 'Record=module mem' -m '*' -f 1 Restart=on-failure PrivateTmp=false NoNewPrivileges=false diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/memp-localhost-malcolm.service b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/memp-localhost-malcolm.service index d8c01cf2f..7418701c1 100644 --- a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/memp-localhost-malcolm.service +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/memp-localhost-malcolm.service @@ -1,9 +1,8 @@ [Unit] -AssertPathExists=/usr/local/bin/fluent-bit After=network.target [Service] -ExecStart=/usr/local/bin/fluent-bit -R /etc/fluent-bit/parsers.conf -i exec -p Command=/usr/local/bin/memory_usage_percentage.sh -p Interval_Sec=30 -o tcp://localhost:5045 -p tls=on -p tls.verify=off -p tls.ca_file=%h/Malcolm/filebeat/certs/ca.crt -p tls.crt_file=%h/Malcolm/filebeat/certs/client.crt -p tls.key_file=%h/Malcolm/filebeat/certs/client.key -p format=json_lines -F modify -p "Hard_rename=exec Mem.used_p" -m '*' -F nest -p Operation=nest -p Nested_under=mem -p WildCard='*' -m '*' -F record_modifier -p 'Record=module mem' -m '*' -f 1 +ExecStart=/opt/fluent-bit/bin/fluent-bit -R /etc/fluent-bit/parsers.conf -i exec -p Command=/usr/local/bin/memory_usage_percentage.sh -p Interval_Sec=30 -o tcp://localhost:5045 -p tls=on -p tls.verify=off -p tls.ca_file=%h/Malcolm/filebeat/certs/ca.crt -p tls.crt_file=%h/Malcolm/filebeat/certs/client.crt -p tls.key_file=%h/Malcolm/filebeat/certs/client.key -p format=json_lines -F modify -p "Hard_rename=exec Mem.used_p" -m '*' -F nest -p Operation=nest -p Nested_under=mem -p WildCard='*' -m '*' -F record_modifier -p 'Record=module mem' -m '*' -f 1 Restart=on-failure PrivateTmp=false NoNewPrivileges=false diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/network-localhost-malcolm.service b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/network-localhost-malcolm.service index 885117c49..5b1fc9a31 100644 --- a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/network-localhost-malcolm.service +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/network-localhost-malcolm.service @@ -1,9 +1,8 @@ [Unit] -AssertPathExists=/usr/local/bin/fluent-bit After=network.target [Service] -ExecStart=/usr/local/bin/fluent-bit -R /etc/fluent-bit/parsers.conf -i exec -p Parser=json -p Command=/usr/local/bin/netdev-json.sh -p Interval_Sec=30 -o tcp://localhost:5045 -p tls=on -p tls.verify=off -p tls.ca_file=%h/Malcolm/filebeat/certs/ca.crt -p tls.crt_file=%h/Malcolm/filebeat/certs/client.crt -p tls.key_file=%h/Malcolm/filebeat/certs/client.key -p format=json_lines -F nest -p Operation=nest -p Nested_under=network -p WildCard='*' -m '*' -F record_modifier -p 'Record=module network' -m '*' -f 1 +ExecStart=/opt/fluent-bit/bin/fluent-bit -R /etc/fluent-bit/parsers.conf -i exec -p Parser=json -p Command=/usr/local/bin/netdev-json.sh -p Interval_Sec=30 -o tcp://localhost:5045 -p tls=on -p tls.verify=off -p tls.ca_file=%h/Malcolm/filebeat/certs/ca.crt -p tls.crt_file=%h/Malcolm/filebeat/certs/client.crt -p tls.key_file=%h/Malcolm/filebeat/certs/client.key -p format=json_lines -F nest -p Operation=nest -p Nested_under=network -p WildCard='*' -m '*' -F record_modifier -p 'Record=module network' -m '*' -f 1 Restart=on-failure PrivateTmp=false NoNewPrivileges=false diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/thermal-localhost-malcolm.service b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/thermal-localhost-malcolm.service index bb3e63e01..cbcb1e8f6 100644 --- a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/thermal-localhost-malcolm.service +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/thermal-localhost-malcolm.service @@ -1,9 +1,8 @@ [Unit] -AssertPathExists=/usr/local/bin/fluent-bit After=network.target [Service] -ExecStart=/usr/local/bin/fluent-bit -R /etc/fluent-bit/parsers.conf -i thermal -p Interval_Sec=10 -o tcp://localhost:5045 -p tls=on -p tls.verify=off -p tls.ca_file=%h/Malcolm/filebeat/certs/ca.crt -p tls.crt_file=%h/Malcolm/filebeat/certs/client.crt -p tls.key_file=%h/Malcolm/filebeat/certs/client.key -p format=json_lines -F nest -p Operation=nest -p Nested_under=thermal -p WildCard='*' -m '*' -F record_modifier -p 'Record=module thermal' -m '*' -f 1 +ExecStart=/opt/fluent-bit/bin/fluent-bit -R /etc/fluent-bit/parsers.conf -i thermal -p Interval_Sec=10 -o tcp://localhost:5045 -p tls=on -p tls.verify=off -p tls.ca_file=%h/Malcolm/filebeat/certs/ca.crt -p tls.crt_file=%h/Malcolm/filebeat/certs/client.crt -p tls.key_file=%h/Malcolm/filebeat/certs/client.key -p format=json_lines -F nest -p Operation=nest -p Nested_under=thermal -p WildCard='*' -m '*' -F record_modifier -p 'Record=module thermal' -m '*' -f 1 Restart=on-failure PrivateTmp=false NoNewPrivileges=false From ede482ab7a0762c9fdde49ca8deb53ca7f02a599 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 8 May 2023 15:13:17 -0600 Subject: [PATCH 14/47] bump Arkime to v4.3.1 (https://github.com/arkime/arkime/blob/14b61feb9e85b962dae0180c3726cd7c020e5ba0/CHANGELOG#L33-L44) --- Dockerfiles/arkime.Dockerfile | 2 +- sensor-iso/arkime/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index 99a1fd88f..cc4a1b851 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -4,7 +4,7 @@ FROM debian:11-slim AS build ENV DEBIAN_FRONTEND noninteractive -ENV ARKIME_VERSION "v4.3.0" +ENV ARKIME_VERSION "v4.3.1" ENV ARKIME_DIR "/opt/arkime" ENV ARKIME_URL "https://github.com/arkime/arkime.git" ENV ARKIME_LOCALELASTICSEARCH no diff --git a/sensor-iso/arkime/Dockerfile b/sensor-iso/arkime/Dockerfile index eaec223e1..1884a9d56 100644 --- a/sensor-iso/arkime/Dockerfile +++ b/sensor-iso/arkime/Dockerfile @@ -6,7 +6,7 @@ LABEL maintainer="malcolm@inl.gov" ENV DEBIAN_FRONTEND noninteractive -ENV ARKIME_VERSION "4.3.0" +ENV ARKIME_VERSION "4.3.1" ENV ARKIME_DIR "/opt/arkime" RUN sed -i "s/bullseye main/bullseye main contrib non-free/g" /etc/apt/sources.list && \ From 3bd39aa5f7382434ccc191cb239b3f4efb25e0c6 Mon Sep 17 00:00:00 2001 From: SG Date: Tue, 9 May 2023 08:11:02 -0600 Subject: [PATCH 15/47] assert client key for local fluent-bit in malcolm ISO --- .../etc/skel/.config/systemd/user/cpu-localhost-malcolm.service | 1 + .../etc/skel/.config/systemd/user/df-localhost-malcolm.service | 1 + .../etc/skel/.config/systemd/user/disk-localhost-malcolm.service | 1 + .../etc/skel/.config/systemd/user/mem-localhost-malcolm.service | 1 + .../etc/skel/.config/systemd/user/memp-localhost-malcolm.service | 1 + .../skel/.config/systemd/user/network-localhost-malcolm.service | 1 + .../skel/.config/systemd/user/thermal-localhost-malcolm.service | 1 + 7 files changed, 7 insertions(+) diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/cpu-localhost-malcolm.service b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/cpu-localhost-malcolm.service index 683e0ecf6..bf7bc40c6 100644 --- a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/cpu-localhost-malcolm.service +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/cpu-localhost-malcolm.service @@ -1,4 +1,5 @@ [Unit] +AssertPathExists=%h/Malcolm/filebeat/certs/client.key After=network.target [Service] diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/df-localhost-malcolm.service b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/df-localhost-malcolm.service index c7fbc3c75..3ece47d60 100644 --- a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/df-localhost-malcolm.service +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/df-localhost-malcolm.service @@ -1,4 +1,5 @@ [Unit] +AssertPathExists=%h/Malcolm/filebeat/certs/client.key After=network.target [Service] diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/disk-localhost-malcolm.service b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/disk-localhost-malcolm.service index dc4f4dff2..8fb2fee1b 100644 --- a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/disk-localhost-malcolm.service +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/disk-localhost-malcolm.service @@ -1,4 +1,5 @@ [Unit] +AssertPathExists=%h/Malcolm/filebeat/certs/client.key After=network.target [Service] diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/mem-localhost-malcolm.service b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/mem-localhost-malcolm.service index 64eff358d..f73368f78 100644 --- a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/mem-localhost-malcolm.service +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/mem-localhost-malcolm.service @@ -1,4 +1,5 @@ [Unit] +AssertPathExists=%h/Malcolm/filebeat/certs/client.key After=network.target [Service] diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/memp-localhost-malcolm.service b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/memp-localhost-malcolm.service index 7418701c1..f9d8e9135 100644 --- a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/memp-localhost-malcolm.service +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/memp-localhost-malcolm.service @@ -1,4 +1,5 @@ [Unit] +AssertPathExists=%h/Malcolm/filebeat/certs/client.key After=network.target [Service] diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/network-localhost-malcolm.service b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/network-localhost-malcolm.service index 5b1fc9a31..e0a1cf718 100644 --- a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/network-localhost-malcolm.service +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/network-localhost-malcolm.service @@ -1,4 +1,5 @@ [Unit] +AssertPathExists=%h/Malcolm/filebeat/certs/client.key After=network.target [Service] diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/thermal-localhost-malcolm.service b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/thermal-localhost-malcolm.service index cbcb1e8f6..6ea73ba54 100644 --- a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/thermal-localhost-malcolm.service +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/thermal-localhost-malcolm.service @@ -1,4 +1,5 @@ [Unit] +AssertPathExists=%h/Malcolm/filebeat/certs/client.key After=network.target [Service] From 80a29bfc41ea9ced1bfc6ab3274af1c6189d49aa Mon Sep 17 00:00:00 2001 From: SG Date: Tue, 9 May 2023 09:22:08 -0600 Subject: [PATCH 16/47] improvements to convenience functions --- .../includes.chroot/etc/bash.bash_functions | 78 ++++++++++++++++--- .../etc/skel/.config/procps/toprc | 16 ++++ .../includes.chroot/etc/bash.bash_functions | 62 ++++++++++++++- 3 files changed, 142 insertions(+), 14 deletions(-) create mode 100644 malcolm-iso/config/includes.chroot/etc/skel/.config/procps/toprc diff --git a/malcolm-iso/config/includes.chroot/etc/bash.bash_functions b/malcolm-iso/config/includes.chroot/etc/bash.bash_functions index f28514ab8..7705973e3 100644 --- a/malcolm-iso/config/includes.chroot/etc/bash.bash_functions +++ b/malcolm-iso/config/includes.chroot/etc/bash.bash_functions @@ -223,6 +223,30 @@ function h() { if [ -z "$1" ]; then history; else history | grep -i "$@"; fi; } ######################################################################## function fname() { find . -iname "*$@*"; } +function findupes() { + find . -not -empty -type f -printf "%s\n" 2>/dev/null | \ + sort -rn | \ + uniq -d | \ + xargs -I{} -n1 find -type f -size {}c -print0 | \ + xargs -0 md5sum | \ + sort | \ + uniq -w32 --all-repeated=separate +} + +function sfind() { + if [ "$1" ]; then + FIND_FOLDER="$1" + else + FIND_FOLDER="$(pwd)" + fi + if [ "$2" ]; then + FIND_PATTERN="$2" + else + FIND_PATTERN="*" + fi + find "$FIND_FOLDER" -type f -iname "$FIND_PATTERN" -print0 | xargs -r -0 ls -la | awk '{system("numfmt -z --to=iec-i --suffix=B --padding=7 "$5) ; out=""; for(i=9;i<=NF;i++){out=out" "$i}; print " KB\t"out}' | sort -h +} + ######################################################################## # examine running processes ######################################################################## @@ -291,7 +315,36 @@ function arps() function portping() { - python <<<"import socket; socket.setdefaulttimeout(1); socket.socket().connect(('$1', $2))" 2> /dev/null && echo OPEN || echo CLOSED; + CONN_TIMEOUT=5 + if [[ -n "$BASH_VERSION" ]] && [[ $LINUX ]]; then + # use /dev/tcp + timeout $CONN_TIMEOUT bash -c "cat /dev/null > /dev/tcp/$1/$2" && echo OPEN || echo CLOSED + elif command -v python3 >/dev/null 2>&1; then + # use python socket library + python3 <<<"import socket; socket.setdefaulttimeout($CONN_TIMEOUT); socket.socket().connect(('$1', $2))" 2> /dev/null && echo OPEN || echo CLOSED + elif command -v socat >/dev/null 2>&1; then + # use socat + socat /dev/null TCP4:"$1":"$2",connect-timeout="$CONN_TIMEOUT" >/dev/null 2>&1 && echo OPEN || echo CLOSED + elif command -v nc >/dev/null 2>&1; then + # use some flavor of netcat + if ( nc -h 2>&1 | grep -q 'to somewhere' ); then + # traditional + ( timeout $((CONN_TIMEOUT+1)) bash -c "cat /dev/null | nc -v -w "$CONN_TIMEOUT" "$1" "$2" 2>&1" || true ) | grep -q 'open$' && echo OPEN || echo CLOSED + elif ( nc 2>&1 | grep -q '46CDdFhklNnrStUuvZz' ); then + # openbsd + timeout $((CONN_TIMEOUT+1)) bash -c "cat /dev/null | nc -w "$CONN_TIMEOUT" "$1" "$2" >/dev/null 2>&1" && echo OPEN || echo CLOSED + elif ( nc --help 2>&1 | grep -q 'Ncat' ); then + # ncat + timeout $CONN_TIMEOUT bash -c "cat /dev/null | nc -v --send-only "$1" "$2" >/dev/null 2>&1" && echo OPEN || echo CLOSED + else + echo UNKNOWN + fi + elif command -v telnet >/dev/null 2>&1; then + # use telnet + timeout $CONN_TIMEOUT bash -c "echo -e '\x1dclose\x0d' | telnet "$1" "$2" >/dev/null 2>&1" && echo OPEN || echo CLOSED + else + echo UNKNOWN + fi } ######################################################################## @@ -443,7 +496,7 @@ alias dis="docker images | tail -n +2 | cols 1 2 | sed \"s/ /:/\"" alias dip="docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}'" # a slimmed-down stats -alias dstats="docker stats --format 'table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.NetIO}}\t{{.BlockIO}}'" +alias dstats="docker stats --format 'table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}'" # Execute in existing interactive container, e.g., $dex base /bin/bash alias dex="docker exec -i -t" @@ -500,14 +553,14 @@ function malcolmmonitor () { select-pane -t 1 \; \ send-keys '~/Malcolm/scripts/logs' C-m \; \ select-pane -t 2 \; \ - send-keys 'dstats' C-m \; \ + send-keys "docker stats --format 'table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}'" C-m \; \ select-pane -t 3 \; \ - send-keys 'while true; do clear; df -h ~/Malcolm/; sleep 60; done' C-m \; \ + send-keys 'while true; do clear; df -h ~/Malcolm/ | tail -n +2; sleep 60; done' C-m \; \ select-pane -t 4 \; \ send-keys 'top' C-m \; \ split-window -v \; \ select-pane -t 5 \; \ - send-keys 'while true; do clear; free -m | head -n 2; sleep 60; done' C-m \; \ + send-keys 'while true; do clear; free -m | grep ^Mem: | cut -d" " -f2- | sed "s/[[:space:]]\+/,/g" | sed "s/^,//" ; sleep 60; done' C-m \; \ select-pane -t 6 \; \ send-keys "while true; do clear; pushd ~/Malcolm >/dev/null 2>&1; docker-compose exec -u $(id -u) api curl -sSL 'http://localhost:5000/mapi/agg/event.dataset?from=1970' | python3 -m json.tool | grep -P '\b(doc_count|key)\b' | tr -d '\", ' | cut -d: -f2 | paste - - -d'\t\t' | head -n $(( (MAX_HEIGHT / 2) - 1 )) ; popd >/dev/null 2>&1; sleep 60; done" C-m \; \ select-pane -t 7 \; \ @@ -519,14 +572,15 @@ function malcolmmonitor () { send-keys "while true; do clear; find ~/Malcolm/zeek-logs/extract_files -type f | sed 's@.*/@@' | sed 's/.*\.//' | sort | uniq -c | sort -nr | head -n $(( (MAX_HEIGHT / 3) - 1 )) ; sleep 60; done" C-m \; \ select-pane -t 9 \; \ resize-pane -R $(( ($MAX_WIDTH / 2) - 30 )) \; \ + select-pane -t 1 \; \ + resize-pane -D 999 \; \ + resize-pane -U 24 \; \ select-pane -t 3 \; \ - resize-pane -D $(( ($MAX_HEIGHT / 4) - 4 )) \; \ + resize-pane -D 999 \; \ + resize-pane -U 1 \; \ select-pane -t 5 \; \ - resize-pane -D $(( ($MAX_HEIGHT / 4) - 4 )) \; \ - select-pane -t 7 \; \ - resize-pane -U $(( ($MAX_HEIGHT / 8) - 4 )) \; \ - select-pane -t 8 \; \ - resize-pane -U $(( ($MAX_HEIGHT / 8) - 1 )) \; \ - select-pane -t 4 \; + resize-pane -D 999 \; \ + resize-pane -U 1 \; fi } + diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/procps/toprc b/malcolm-iso/config/includes.chroot/etc/skel/.config/procps/toprc new file mode 100644 index 000000000..481147027 --- /dev/null +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/procps/toprc @@ -0,0 +1,16 @@ +top's Config File (Linux processes with windows) +Id:j, Mode_altscr=0, Mode_irixps=1, Delay_time=1.0, Curwin=0 +Def fieldscur=¥(34»½@Ä·º¹Å&')*+,-./012568<>?ABCFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz + winflags=193844, sortindx=18, maxtasks=0, graph_cpus=0, graph_mems=0, double_up=0, combine_cpus=0 + summclr=1, msgsclr=1, headclr=3, taskclr=1 +Job fieldscur=¥¦¹·º(³´Ä»½@<§Å)*+,-./012568>?ABCFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz + winflags=193844, sortindx=0, maxtasks=0, graph_cpus=0, graph_mems=0, double_up=0, combine_cpus=0 + summclr=6, msgsclr=6, headclr=7, taskclr=6 +Mem fieldscur=¥º»<½¾¿ÀÁMBNÃD34·Å&'()*+,-./0125689FGHIJKLOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz + winflags=193844, sortindx=21, maxtasks=0, graph_cpus=0, graph_mems=0, double_up=0, combine_cpus=0 + summclr=5, msgsclr=5, headclr=4, taskclr=5 +Usr fieldscur=¥¦§¨ª°¹·ºÄÅ)+,-./1234568;<=>?@ABCFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz + winflags=193844, sortindx=3, maxtasks=0, graph_cpus=0, graph_mems=0, double_up=0, combine_cpus=0 + summclr=3, msgsclr=3, headclr=2, taskclr=3 +Fixed_widest=0, Summ_mscale=1, Task_mscale=0, Zero_suppress=0 + diff --git a/sensor-iso/config/includes.chroot/etc/bash.bash_functions b/sensor-iso/config/includes.chroot/etc/bash.bash_functions index d555810cb..f53ac6309 100644 --- a/sensor-iso/config/includes.chroot/etc/bash.bash_functions +++ b/sensor-iso/config/includes.chroot/etc/bash.bash_functions @@ -223,6 +223,30 @@ function h() { if [ -z "$1" ]; then history; else history | grep -i "$@"; fi; } ######################################################################## function fname() { find . -iname "*$@*"; } +function findupes() { + find . -not -empty -type f -printf "%s\n" 2>/dev/null | \ + sort -rn | \ + uniq -d | \ + xargs -I{} -n1 find -type f -size {}c -print0 | \ + xargs -0 md5sum | \ + sort | \ + uniq -w32 --all-repeated=separate +} + +function sfind() { + if [ "$1" ]; then + FIND_FOLDER="$1" + else + FIND_FOLDER="$(pwd)" + fi + if [ "$2" ]; then + FIND_PATTERN="$2" + else + FIND_PATTERN="*" + fi + find "$FIND_FOLDER" -type f -iname "$FIND_PATTERN" -print0 | xargs -r -0 ls -la | awk '{system("numfmt -z --to=iec-i --suffix=B --padding=7 "$5) ; out=""; for(i=9;i<=NF;i++){out=out" "$i}; print " KB\t"out}' | sort -h +} + ######################################################################## # examine running processes ######################################################################## @@ -291,7 +315,36 @@ function arps() function portping() { - python <<<"import socket; socket.setdefaulttimeout(1); socket.socket().connect(('$1', $2))" 2> /dev/null && echo OPEN || echo CLOSED; + CONN_TIMEOUT=5 + if [[ -n "$BASH_VERSION" ]] && [[ $LINUX ]]; then + # use /dev/tcp + timeout $CONN_TIMEOUT bash -c "cat /dev/null > /dev/tcp/$1/$2" && echo OPEN || echo CLOSED + elif command -v python3 >/dev/null 2>&1; then + # use python socket library + python3 <<<"import socket; socket.setdefaulttimeout($CONN_TIMEOUT); socket.socket().connect(('$1', $2))" 2> /dev/null && echo OPEN || echo CLOSED + elif command -v socat >/dev/null 2>&1; then + # use socat + socat /dev/null TCP4:"$1":"$2",connect-timeout="$CONN_TIMEOUT" >/dev/null 2>&1 && echo OPEN || echo CLOSED + elif command -v nc >/dev/null 2>&1; then + # use some flavor of netcat + if ( nc -h 2>&1 | grep -q 'to somewhere' ); then + # traditional + ( timeout $((CONN_TIMEOUT+1)) bash -c "cat /dev/null | nc -v -w "$CONN_TIMEOUT" "$1" "$2" 2>&1" || true ) | grep -q 'open$' && echo OPEN || echo CLOSED + elif ( nc 2>&1 | grep -q '46CDdFhklNnrStUuvZz' ); then + # openbsd + timeout $((CONN_TIMEOUT+1)) bash -c "cat /dev/null | nc -w "$CONN_TIMEOUT" "$1" "$2" >/dev/null 2>&1" && echo OPEN || echo CLOSED + elif ( nc --help 2>&1 | grep -q 'Ncat' ); then + # ncat + timeout $CONN_TIMEOUT bash -c "cat /dev/null | nc -v --send-only "$1" "$2" >/dev/null 2>&1" && echo OPEN || echo CLOSED + else + echo UNKNOWN + fi + elif command -v telnet >/dev/null 2>&1; then + # use telnet + timeout $CONN_TIMEOUT bash -c "echo -e '\x1dclose\x0d' | telnet "$1" "$2" >/dev/null 2>&1" && echo OPEN || echo CLOSED + else + echo UNKNOWN + fi } ######################################################################## @@ -466,7 +519,12 @@ function sensormonitor () { select-pane -t 2 \; \ send-keys 'while true; do clear; /opt/sensor/sensor_ctl/status | grep -v "Not started" | sed "s/pid.* //"; sleep 60; done' C-m \; \ select-pane -t 3 \; \ - send-keys 'tail -F /opt/sensor/sensor_ctl/log/*' C-m + send-keys 'tail -F /opt/sensor/sensor_ctl/log/*' C-m \; \ + select-pane -t 2 \; \ + resize-pane -U 999 \; \ + resize-pane -D 27 \; \ + resize-pane -R 999 \; \ + resize-pane -L 58 } function suricata-update () { From 3f01c325c77d74ed62d720042d58b3cd524d0a6a Mon Sep 17 00:00:00 2001 From: SG Date: Tue, 9 May 2023 10:31:43 -0600 Subject: [PATCH 17/47] bump transform plugin version --- Dockerfiles/dashboards.Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfiles/dashboards.Dockerfile b/Dockerfiles/dashboards.Dockerfile index 1811ae826..cdbc9850e 100644 --- a/Dockerfiles/dashboards.Dockerfile +++ b/Dockerfiles/dashboards.Dockerfile @@ -90,7 +90,7 @@ ENV PUSER_PRIV_DROP true ENV TERM xterm ENV TINI_VERSION v0.19.0 -ENV OSD_TRANSFORM_VIS_VERSION 2.6.0 +ENV OSD_TRANSFORM_VIS_VERSION 2.7.0 ARG OPENSEARCH_URL="http://opensearch:9200" ARG OPENSEARCH_LOCAL="true" @@ -124,10 +124,10 @@ RUN yum upgrade -y && \ cd /usr/share/opensearch-dashboards/plugins && \ /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin install file:///tmp/kbnSankeyVis.zip --allow-root && \ cd /tmp && \ - unzip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \ - sed -i "s/2\.6\.0/2\.7\.0/g" opensearch-dashboards/transformVis/opensearch_dashboards.json && \ - sed -i "s/2\.6\.0/2\.7\.0/g" opensearch-dashboards/transformVis/package.json && \ - zip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \ + # unzip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \ + # sed -i "s/2\.6\.0/2\.7\.0/g" opensearch-dashboards/transformVis/opensearch_dashboards.json && \ + # sed -i "s/2\.6\.0/2\.7\.0/g" opensearch-dashboards/transformVis/package.json && \ + # zip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \ cd /usr/share/opensearch-dashboards/plugins && \ /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin install file:///tmp/transformVis.zip --allow-root && \ rm -rf /tmp/transformVis /tmp/opensearch-dashboards && \ From f7e7a4d37a9a21868124d6614aece3925475fd5a Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 9 May 2023 14:02:11 -0600 Subject: [PATCH 18/47] documentation tweak --- docs/quickstart.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/quickstart.md b/docs/quickstart.md index f721f9f5e..1d554f85b 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -86,10 +86,10 @@ instance, wipe the database and restore Malcolm to a fresh state, etc. ## User interface -A few minutes after starting Malcolm (probably 5 to 10 minutes for Logstash to be completely up, depending on the system), the following services will be accessible: +A few minutes after starting Malcolm (probably 5 or so for Logstash to be completely up, depending on the system), the following services will be accessible: -* [Arkime](https://arkime.com/): [https://localhost:443](https://localhost:443) -* [OpenSearch Dashboards](https://opensearch.org/docs/latest/dashboards/index/): [https://localhost/dashboards/](https://localhost/dashboards/) or [https://localhost:5601](https://localhost:5601) +* [Arkime](https://arkime.com/): [https://localhost](https://localhost) +* [OpenSearch Dashboards](https://opensearch.org/docs/latest/dashboards/index/): [https://localhost/dashboards/](https://localhost/dashboards/) * [Capture File and Log Archive Upload (Web)](upload.md#Upload): [https://localhost/upload/](https://localhost/upload/) * [Capture File and Log Archive Upload (SFTP)](upload.md#Upload): `sftp://@127.0.0.1:8022/files` * [NetBox](asset-interaction-analysis.md#AssetInteractionAnalysis): [https://localhost/netbox/](https://localhost/netbox/) From 77427cd7dc7a3aae98417c88ab7621283b0ab254 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 10 May 2023 07:15:07 -0600 Subject: [PATCH 19/47] move deploy ingress-nginx logic to bash script --- docs/kubernetes.md | 2 +- kubernetes/vagrant/Vagrantfile | 10 +-- kubernetes/vagrant/deploy_ingress_nginx.sh | 94 ++++++++++++++++++++++ 3 files changed, 98 insertions(+), 8 deletions(-) create mode 100755 kubernetes/vagrant/deploy_ingress_nginx.sh diff --git a/docs/kubernetes.md b/docs/kubernetes.md index 8ceaa0868..0f8411462 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -18,7 +18,7 @@ ### Ingress Controller -Malcolm's [ingress controller manifest]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/kubernetes/00-ingress.yml) uses the [Ingress-NGINX controller for Kubernetes](https://github.com/kubernetes/ingress-nginx). A few Malcolm features require some customization when installing and configuring the Ingress-NGINX controller: +Malcolm's [ingress controller manifest]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/kubernetes/00-ingress.yml) uses the [Ingress-NGINX controller for Kubernetes](https://github.com/kubernetes/ingress-nginx). A few Malcolm features require some customization when installing and configuring the Ingress-NGINX controller. As well as being listed below, see [kubernetes/vagrant/deploy_ingress_nginx.sh]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/kubernetes/vagrant/deploy_ingress_nginx.sh) for an example of how to configure and apply the Ingress-NGINX controller for Kubernetes. * To [forward](malcolm-hedgehog-e2e-iso-install.md#HedgehogConfigForwarding) logs from a remote instance of [Hedgehog Linux](hedgehog.md): - See ["Exposing TCP and UDP services"](https://kubernetes.github.io/ingress-nginx/user-guide/exposing-tcp-udp-services/) in the Ingress-NGINX documentation. diff --git a/kubernetes/vagrant/Vagrantfile b/kubernetes/vagrant/Vagrantfile index 77e2e2b61..d76a5780b 100644 --- a/kubernetes/vagrant/Vagrantfile +++ b/kubernetes/vagrant/Vagrantfile @@ -13,8 +13,6 @@ end server_ip = "192.168.56.10" server_hostname = "server.k3s.internal" -load_balancer_additional_ports = "{\\\"appProtocol\\\": \\\"tcp\\\", \\\"name\\\": \\\"lumberjack\\\", \\\"port\\\": 5044, \\\"targetPort\\\": 5044, \\\"protocol\\\": \\\"TCP\\\"}, {\\\"appProtocol\\\": \\\"tcp\\\", \\\"name\\\": \\\"tcpjson\\\", \\\"port\\\": 5045, \\\"targetPort\\\": 5045, \\\"protocol\\\": \\\"TCP\\\"}, {\\\"appProtocol\\\": \\\"tcp\\\", \\\"name\\\": \\\"sftp\\\", \\\"port\\\": 8022, \\\"targetPort\\\": 8022, \\\"protocol\\\": \\\"TCP\\\"}, {\\\"appProtocol\\\": \\\"tcp\\\", \\\"name\\\": \\\"opensearch\\\", \\\"port\\\": 9200, \\\"targetPort\\\": 9200, \\\"protocol\\\": \\\"TCP\\\"}" -deployment_additional_ports = "{\\\"name\\\": \\\"lumberjack\\\", \\\"containerPort\\\": 5044, \\\"protocol\\\": \\\"TCP\\\"}, {\\\"name\\\": \\\"tcpjson\\\", \\\"containerPort\\\": 5045, \\\"protocol\\\": \\\"TCP\\\"}, {\\\"name\\\": \\\"sftp\\\", \\\"containerPort\\\": 8022, \\\"protocol\\\": \\\"TCP\\\"}, {\\\"name\\\": \\\"opensearch\\\", \\\"containerPort\\\": 9200, \\\"protocol\\\": \\\"TCP\\\"}" agents = { "agent1" => "192.168.56.11", "agent2" => "192.168.56.12" } @@ -113,14 +111,11 @@ server_script_1 = <<-SHELL curl -sfL https://get.k3s.io | sh - echo "Waiting for k3s to start..." sleep 30 - curl -sSL -o /tmp/deploy_nginx.yaml https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.7.0/deploy/static/provider/cloud/deploy.yaml - yq -i '( select(.kind == "Deployment").spec.template.spec.containers[].args[] | select(contains("/nginx-ingress-controller")) | parent ) += ["--enable-ssl-passthrough", "--tcp-services-configmap=ingress-nginx/tcp-services"]' /tmp/deploy_nginx.yaml - yq -i "( select(.kind == \\"Deployment\\").spec.template.spec.containers[].args[] | select(contains(\\"/nginx-ingress-controller\\")) | parent | parent | .ports ) += [#{deployment_additional_ports}]" /tmp/deploy_nginx.yaml - yq -i "( select(.kind == \\"Service\\" and .spec.type == \\"LoadBalancer\\").spec.ports ) += [#{load_balancer_additional_ports}]" /tmp/deploy_nginx.yaml - kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml apply -f /tmp/deploy_nginx.yaml + bash /tmp/deploy_ingress_nginx.sh -k /etc/rancher/k3s/k3s.yaml until [ -f /var/lib/rancher/k3s/server/token ] && [ -f /etc/rancher/k3s/k3s.yaml ]; do sleep 5; done cp -v /var/lib/rancher/k3s/server/token /vagrant_shared cp -v /etc/rancher/k3s/k3s.yaml /vagrant_shared + rm -f /tmp/deploy_ingress_nginx.sh SHELL agent_script_1 = <<-SHELL @@ -168,6 +163,7 @@ Vagrant.configure("2") do |config| server.vm.provision "shell", inline: server_script_0 server.vm.provision "shell", inline: common_script_0 server.vm.provision :reload + server.vm.provision "file", source: "./deploy_ingress_nginx.sh", destination: "/tmp/deploy_ingress_nginx.sh" server.vm.provision "shell", inline: server_script_1 end diff --git a/kubernetes/vagrant/deploy_ingress_nginx.sh b/kubernetes/vagrant/deploy_ingress_nginx.sh new file mode 100755 index 000000000..2dfbbbaf7 --- /dev/null +++ b/kubernetes/vagrant/deploy_ingress_nginx.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash + +if [ -z "$BASH_VERSION" ]; then + echo "Wrong interpreter, please run \"$0\" with bash" >&2 + exit 1 +fi + +############################################################################### +# script options +set -o pipefail +set -e +shopt -s nocasematch +ENCODING="utf-8" + +############################################################################### +# script variables +LOAD_BALANCER_ADDITIONAL_PORTS="{\"appProtocol\": \"tcp\", \"name\": \"lumberjack\", \"port\": 5044, \"targetPort\": 5044, \"protocol\": \"TCP\"}, {\"appProtocol\": \"tcp\", \"name\": \"tcpjson\", \"port\": 5045, \"targetPort\": 5045, \"protocol\": \"TCP\"}, {\"appProtocol\": \"tcp\", \"name\": \"sftp\", \"port\": 8022, \"targetPort\": 8022, \"protocol\": \"TCP\"}, {\"appProtocol\": \"tcp\", \"name\": \"opensearch\", \"port\": 9200, \"targetPort\": 9200, \"protocol\": \"TCP\"}" +DEPLOYMENT_ADDITIONAL_PORTS="{\"name\": \"lumberjack\", \"containerPort\": 5044, \"protocol\": \"TCP\"}, {\"name\": \"tcpjson\", \"containerPort\": 5045, \"protocol\": \"TCP\"}, {\"name\": \"sftp\", \"containerPort\": 8022, \"protocol\": \"TCP\"}, {\"name\": \"opensearch\", \"containerPort\": 9200, \"protocol\": \"TCP\"}" +INGRESS_NGINX_CONTROLLER_VERSION=1.7.0 +KUBECONFIG= +DEPLOY_YAML_FILE= +DRY_RUN=none + +############################################################################### +# show script usage +function help() { + echo -e "$(basename $0)\n" >&2 + echo "-v enable bash verbosity" >&2 + echo "-h display help" >&2 + echo "-k kubeconfig kubeconfig file" >&2 + echo "-d dryrunval --dry-run=dryrunval for kubectl (dryrunval=none|server|client)" >&2 + echo "-i version ingress-nginx controller version" >&2 + exit 1 +} + +############################################################################### +# parse command-line parameters +while getopts 'vhd:k:i:' OPTION; do + case "$OPTION" in + + v) + VERBOSE_FLAG="-v" + set -x + ;; + + d) + DRY_RUN="${OPTARG}" + ;; + + k) + KUBECONFIG="${OPTARG}" + ;; + + i) + INGRESS_NGINX_CONTROLLER_VERSION="${OPTARG}" + ;; + + ?) + help >&2 + exit 1; + ;; + + esac +done +shift "$(($OPTIND -1))" + +############################################################################### +function cleanup { + [[ -n "${DEPLOY_YAML_FILE}" ]] && [[ -f "${DEPLOY_YAML_FILE}" ]] && rm ${VERBOSE_FLAG} -f "${DEPLOY_YAML_FILE}" +} + +if ! command -v curl >/dev/null 2>&1 || ! command -v yq >/dev/null 2>&1 || ! command -v kubectl >/dev/null 2>&1; then + echo "$(basename $0) requires curl, kubectl and yq" >&2 + exit 1 + +elif [[ -z "${KUBECONFIG}" ]] || [[ ! -f "${KUBECONFIG}" ]]; then + echo "$(basename $0) requires kubeconfig specified with -k" >&2 + exit 1 +fi + +############################################################################### + +trap "cleanup" EXIT + +DEPLOY_YAML_FILE="$(mktemp --suffix=.yaml)" + +curl -fsSL -o "${DEPLOY_YAML_FILE}" "https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v${INGRESS_NGINX_CONTROLLER_VERSION}/deploy/static/provider/cloud/deploy.yaml" +yq -i '( select(.kind == "Deployment").spec.template.spec.containers[].args[] | select(contains("/nginx-ingress-controller")) | parent ) += ["--enable-ssl-passthrough", "--tcp-services-configmap=ingress-nginx/tcp-services"]' "${DEPLOY_YAML_FILE}" +yq -i "( select(.kind == \"Deployment\").spec.template.spec.containers[].args[] | select(contains(\"/nginx-ingress-controller\")) | parent | parent | .ports ) += [${DEPLOYMENT_ADDITIONAL_PORTS}]" "${DEPLOY_YAML_FILE}" +yq -i "( select(.kind == \"Service\" and .spec.type == \"LoadBalancer\").spec.ports ) += [${LOAD_BALANCER_ADDITIONAL_PORTS}]" "${DEPLOY_YAML_FILE}" + +[[ -n "${VERBOSE_FLAG}" ]] && cat "${DEPLOY_YAML_FILE}" + +kubectl --kubeconfig "${KUBECONFIG}" apply --dry-run="${DRY_RUN}" -f "${DEPLOY_YAML_FILE}" From 391a64f5f5d6def3b433e3e5fce37bd267314f4b Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 10 May 2023 08:32:18 -0600 Subject: [PATCH 20/47] Fix missing leading slash in zeek-live.env.example config files --- config/zeek-live.env.example | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/zeek-live.env.example b/config/zeek-live.env.example index ab8f03680..ec6316eb8 100644 --- a/config/zeek-live.env.example +++ b/config/zeek-live.env.example @@ -4,6 +4,6 @@ ZEEK_LIVE_CAPTURE=false ZEEK_PCAP_PROCESSOR=false ZEEK_CRON=true -ZEEK_LOG_PATH=zeek/live -ZEEK_INTEL_PATH=opt/zeek/share/zeek/site/intel -EXTRACT_FILES_PATH=zeek/extract_files \ No newline at end of file +ZEEK_LOG_PATH=/zeek/live +ZEEK_INTEL_PATH=/opt/zeek/share/zeek/site/intel +EXTRACT_FILES_PATH=/zeek/extract_files \ No newline at end of file From cd054797c67c62789553da78d6417044d4812f09 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 10 May 2023 08:41:45 -0600 Subject: [PATCH 21/47] work in progress document for EKS --- docs/kubernetes-eks.md | 319 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 319 insertions(+) create mode 100644 docs/kubernetes-eks.md diff --git a/docs/kubernetes-eks.md b/docs/kubernetes-eks.md new file mode 100644 index 000000000..4c8d9f615 --- /dev/null +++ b/docs/kubernetes-eks.md @@ -0,0 +1,319 @@ +## Notes for deploying Malcolm on AWS EKS + +This document is a rough work in progress and isn't necessarily correct (yet). -SG + +1. Create [VPC](https://us-east-1.console.aws.amazon.com/vpc/home?region=us-east-1#vpcs:) with subnets in 2 availability zones +1. Create [security group](https://us-east-1.console.aws.amazon.com/vpc/home?region=us-east-1#SecurityGroups:) for VPC +1. Create [EKS cluster](https://us-east-1.console.aws.amazon.com/eks/home?region=us-east-1#/clusters) +1. Create [node group](https://us-east-1.console.aws.amazon.com/eks/home?region=us-east-1#/clusters/cluster-name/add-node-group) +1. Create volumes (**p**cap, **z**eek, **s**uricata, **c**onfig, **r**untime-**l**ogs, **o**pensearch, **b**ackup), got volume IDs + ```bash + aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 500 --volume-type gp2 + aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 250 --volume-type gp2 + aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 100 --volume-type gp2 + aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 25 --volume-type gp2 + aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 25 --volume-type gp2 + aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 500 --volume-type gp2 + aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 500 --volume-type gp2 + ``` + ``` + p vol-0123456789c82a042 + z vol-0123456789c67edd9 + s vol-0123456789dccd75e + c vol-0123456789429a231 + r vol-0123456789dc2ea7a + o vol-01234567895ff99a1 + b vol-01234567891150804 + ``` +1. Create EC2 instance, attach volumes + ```bash + aws ec2 attach-volume --volume-id vol-0123456789c82a042 --instance-id i-0123456789abcdef0 --device /dev/xvdp + aws ec2 attach-volume --volume-id vol-0123456789c67edd9 --instance-id i-0123456789abcdef0 --device /dev/xvdz + aws ec2 attach-volume --volume-id vol-0123456789dccd75e --instance-id i-0123456789abcdef0 --device /dev/xvds + aws ec2 attach-volume --volume-id vol-0123456789429a231 --instance-id i-0123456789abcdef0 --device /dev/xvdc + aws ec2 attach-volume --volume-id vol-0123456789dc2ea7a --instance-id i-0123456789abcdef0 --device /dev/xvdr + aws ec2 attach-volume --volume-id vol-01234567895ff99a1 --instance-id i-0123456789abcdef0 --device /dev/xvdo + aws ec2 attach-volume --volume-id vol-01234567891150804 --instance-id i-0123456789abcdef0 --device /dev/xvdb + ``` +1. Format attached volumes as XFS + ```bash + for DRV in p z s c r o b; do sudo mkfs.xfs -f /dev/xvd${DRV}; done + ``` +1. Mount drives and set permissions + ```bash + for DRV in p z s c r o b; do sudo umount -f /dev/xvd${DRV} 2>/dev/null; sudo mkdir -vp /media/xvd${DRV}; sudo mount /dev/xvd${DRV} /media/xvd${DRV}; sudo chown -R $(id -u):$(id -g) /media/xvd${DRV}; df -h /media/xvd${DRV}; done + ``` +1. Create necessary subdirectories inside of some directories (config, pcap, zeek) + ```bash + mkdir -vp /media/xvdc/{auth,htadmin,opensearch,logstash,netbox/media,netbox/postgres,netbox/redis,zeek/intel/MISP,zeek/intel/STIX} + mkdir -vp /media/xvdp/{upload,proceessed} + mkdir -vp /media/xvdz/{current,upload,extract_files} + ``` +1. Unmount drives + ```bash + for DRV in p z s c r o b; do sudo umount -f /dev/xvd${DRV}; done + ``` +1. Detach volumes + ```bash + aws ec2 detach-volume --volume-id vol-0123456789c82a042 --instance-id i-0123456789abcdef0 + aws ec2 detach-volume --volume-id vol-0123456789c67edd9 --instance-id i-0123456789abcdef0 + aws ec2 detach-volume --volume-id vol-0123456789dccd75e --instance-id i-0123456789abcdef0 + aws ec2 detach-volume --volume-id vol-0123456789429a231 --instance-id i-0123456789abcdef0 + aws ec2 detach-volume --volume-id vol-0123456789dc2ea7a --instance-id i-0123456789abcdef0 + aws ec2 detach-volume --volume-id vol-01234567895ff99a1 --instance-id i-0123456789abcdef0 + aws ec2 detach-volume --volume-id vol-01234567891150804 --instance-id i-0123456789abcdef0 + ``` +1. Create YAML for persistent volumes and volume claims from the EBS Volume ID + ```yaml + --- + apiVersion: v1 + kind: PersistentVolume + metadata: + name: pcap-volume + namespace: malcolm + labels: + namespace: malcolm + spec: + capacity: + storage: 500Gi + volumeMode: Filesystem + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + storageClassName: gp2-retain + awsElasticBlockStore: + fsType: xfs + volumeID: aws://us-east-1a/vol-0123456789c82a042 + + --- + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: pcap-claim + namespace: malcolm + spec: + storageClassName: gp2-retain + accessModes: + - ReadWriteMany + volumeMode: Filesystem + resources: + requests: + storage: 500Gi + volumeName: pcap-volume + + --- + apiVersion: v1 + kind: PersistentVolume + metadata: + name: zeek-volume + namespace: malcolm + labels: + namespace: malcolm + spec: + capacity: + storage: 250Gi + volumeMode: Filesystem + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + storageClassName: gp2-retain + awsElasticBlockStore: + fsType: xfs + volumeID: aws://us-east-1a/vol-0123456789c67edd9 + + --- + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: zeek-claim + namespace: malcolm + spec: + storageClassName: gp2-retain + accessModes: + - ReadWriteMany + volumeMode: Filesystem + resources: + requests: + storage: 250Gi + volumeName: zeek-volume + + --- + apiVersion: v1 + kind: PersistentVolume + metadata: + name: suricata-volume + namespace: malcolm + labels: + namespace: malcolm + spec: + capacity: + storage: 100Gi + volumeMode: Filesystem + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + storageClassName: gp2-retain + awsElasticBlockStore: + fsType: xfs + volumeID: aws://us-east-1a/vol-0123456789dccd75e + + --- + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: suricata-claim + namespace: malcolm + spec: + storageClassName: gp2-retain + accessModes: + - ReadWriteMany + volumeMode: Filesystem + resources: + requests: + storage: 100Gi + volumeName: suricata-volume + + --- + apiVersion: v1 + kind: PersistentVolume + metadata: + name: config-volume + namespace: malcolm + labels: + namespace: malcolm + spec: + capacity: + storage: 25Gi + volumeMode: Filesystem + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + storageClassName: gp2-retain + awsElasticBlockStore: + fsType: xfs + volumeID: aws://us-east-1a/vol-0123456789429a231 + + --- + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: config-claim + namespace: malcolm + spec: + storageClassName: gp2-retain + accessModes: + - ReadWriteMany + volumeMode: Filesystem + resources: + requests: + storage: 25Gi + volumeName: config-volume + + --- + apiVersion: v1 + kind: PersistentVolume + metadata: + name: runtime-logs-volume + namespace: malcolm + labels: + namespace: malcolm + spec: + capacity: + storage: 25Gi + volumeMode: Filesystem + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + storageClassName: gp2-retain + awsElasticBlockStore: + fsType: xfs + volumeID: aws://us-east-1a/vol-0123456789dc2ea7a + + --- + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: runtime-logs-claim + namespace: malcolm + spec: + storageClassName: gp2-retain + accessModes: + - ReadWriteMany + volumeMode: Filesystem + resources: + requests: + storage: 25Gi + volumeName: runtime-logs-volume + + --- + apiVersion: v1 + kind: PersistentVolume + metadata: + name: opensearch-volume + namespace: malcolm + labels: + namespace: malcolm + spec: + capacity: + storage: 500Gi + volumeMode: Filesystem + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + storageClassName: gp2-retain + awsElasticBlockStore: + fsType: xfs + volumeID: aws://us-east-1a/vol-01234567895ff99a1 + + --- + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: opensearch-claim + namespace: malcolm + spec: + storageClassName: gp2-retain + accessModes: + - ReadWriteMany + volumeMode: Filesystem + resources: + requests: + storage: 500Gi + volumeName: opensearch-volume + + --- + apiVersion: v1 + kind: PersistentVolume + metadata: + name: opensearch-backup-volume + namespace: malcolm + labels: + namespace: malcolm + spec: + capacity: + storage: 500Gi + volumeMode: Filesystem + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + storageClassName: gp2-retain + awsElasticBlockStore: + fsType: xfs + volumeID: aws://us-east-1a/vol-01234567891150804 + + --- + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: opensearch-backup-claim + namespace: malcolm + spec: + storageClassName: gp2-retain + accessModes: + - ReadWriteMany + volumeMode: Filesystem + resources: + requests: + storage: 500Gi + volumeName: opensearch-backup-volume + ``` From 5acac3793a6e59a5ea587a60a267d2679c73bb96 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 10 May 2023 08:43:57 -0600 Subject: [PATCH 22/47] work in progress document for EKS --- docs/kubernetes-eks.md | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/kubernetes-eks.md b/docs/kubernetes-eks.md index 4c8d9f615..b1d078e92 100644 --- a/docs/kubernetes-eks.md +++ b/docs/kubernetes-eks.md @@ -7,24 +7,24 @@ This document is a rough work in progress and isn't necessarily correct (yet). - 1. Create [EKS cluster](https://us-east-1.console.aws.amazon.com/eks/home?region=us-east-1#/clusters) 1. Create [node group](https://us-east-1.console.aws.amazon.com/eks/home?region=us-east-1#/clusters/cluster-name/add-node-group) 1. Create volumes (**p**cap, **z**eek, **s**uricata, **c**onfig, **r**untime-**l**ogs, **o**pensearch, **b**ackup), got volume IDs - ```bash - aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 500 --volume-type gp2 - aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 250 --volume-type gp2 - aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 100 --volume-type gp2 - aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 25 --volume-type gp2 - aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 25 --volume-type gp2 - aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 500 --volume-type gp2 - aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 500 --volume-type gp2 - ``` - ``` - p vol-0123456789c82a042 - z vol-0123456789c67edd9 - s vol-0123456789dccd75e - c vol-0123456789429a231 - r vol-0123456789dc2ea7a - o vol-01234567895ff99a1 - b vol-01234567891150804 - ``` + ```bash + aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 500 --volume-type gp2 + aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 250 --volume-type gp2 + aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 100 --volume-type gp2 + aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 25 --volume-type gp2 + aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 25 --volume-type gp2 + aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 500 --volume-type gp2 + aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 500 --volume-type gp2 + ``` + ``` + p vol-0123456789c82a042 + z vol-0123456789c67edd9 + s vol-0123456789dccd75e + c vol-0123456789429a231 + r vol-0123456789dc2ea7a + o vol-01234567895ff99a1 + b vol-01234567891150804 + ``` 1. Create EC2 instance, attach volumes ```bash aws ec2 attach-volume --volume-id vol-0123456789c82a042 --instance-id i-0123456789abcdef0 --device /dev/xvdp From b1f35ad0872014baa97c37b99433590351165994 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 10 May 2023 10:22:54 -0600 Subject: [PATCH 23/47] documentation update --- docs/kubernetes-eks.md | 79 +++++++++++++++++++++++++----------------- scripts/control.py | 6 ++-- 2 files changed, 51 insertions(+), 34 deletions(-) diff --git a/docs/kubernetes-eks.md b/docs/kubernetes-eks.md index b1d078e92..c5d7fe36c 100644 --- a/docs/kubernetes-eks.md +++ b/docs/kubernetes-eks.md @@ -2,29 +2,44 @@ This document is a rough work in progress and isn't necessarily correct (yet). -SG +Prerequisites: + +* [aws cli](https://aws.amazon.com/cli/) +* [eksctl](https://eksctl.io/) + 1. Create [VPC](https://us-east-1.console.aws.amazon.com/vpc/home?region=us-east-1#vpcs:) with subnets in 2 availability zones 1. Create [security group](https://us-east-1.console.aws.amazon.com/vpc/home?region=us-east-1#SecurityGroups:) for VPC 1. Create [EKS cluster](https://us-east-1.console.aws.amazon.com/eks/home?region=us-east-1#/clusters) 1. Create [node group](https://us-east-1.console.aws.amazon.com/eks/home?region=us-east-1#/clusters/cluster-name/add-node-group) +1. Generate kubeconfig file if you need to + ```bash + aws eks update-kubeconfig --region us-east-1 --name cluster-name --kubeconfig malcolmeks.yaml + ``` +1. [Deploy](https://docs.aws.amazon.com/eks/latest/userguide/metrics-server.html) `metrics-server` + ```bash + kubectl --kubeconfig=malcolmeks.yaml apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml + ``` +1. [Create Amazon EBS CSI driver IAM role](https://docs.aws.amazon.com/eks/latest/userguide/csi-iam-role.html) +1. [Add the Amazon EBS CSI add-on](https://docs.aws.amazon.com/eks/latest/userguide/managing-ebs-csi.html) 1. Create volumes (**p**cap, **z**eek, **s**uricata, **c**onfig, **r**untime-**l**ogs, **o**pensearch, **b**ackup), got volume IDs - ```bash - aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 500 --volume-type gp2 - aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 250 --volume-type gp2 - aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 100 --volume-type gp2 - aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 25 --volume-type gp2 - aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 25 --volume-type gp2 - aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 500 --volume-type gp2 - aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 500 --volume-type gp2 - ``` - ``` - p vol-0123456789c82a042 - z vol-0123456789c67edd9 - s vol-0123456789dccd75e - c vol-0123456789429a231 - r vol-0123456789dc2ea7a - o vol-01234567895ff99a1 - b vol-01234567891150804 - ``` + ```bash + aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 500 --iops 16000 --volume-type io1 + aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 250 --iops 16000 --volume-type io1 + aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 100 --iops 16000 --volume-type io1 + aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 25 --iops 16000 --volume-type io1 + aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 25 --iops 16000 --volume-type io1 + aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 500 --iops 16000 --volume-type io1 + aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 500 --iops 16000 --volume-type io1 + ``` + ``` + p vol-0123456789c82a042 + z vol-0123456789c67edd9 + s vol-0123456789dccd75e + c vol-0123456789429a231 + r vol-0123456789dc2ea7a + o vol-01234567895ff99a1 + b vol-01234567891150804 + ``` 1. Create EC2 instance, attach volumes ```bash aws ec2 attach-volume --volume-id vol-0123456789c82a042 --instance-id i-0123456789abcdef0 --device /dev/xvdp @@ -80,7 +95,7 @@ This document is a rough work in progress and isn't necessarily correct (yet). - accessModes: - ReadWriteMany persistentVolumeReclaimPolicy: Retain - storageClassName: gp2-retain + storageClassName: io1 awsElasticBlockStore: fsType: xfs volumeID: aws://us-east-1a/vol-0123456789c82a042 @@ -92,7 +107,7 @@ This document is a rough work in progress and isn't necessarily correct (yet). - name: pcap-claim namespace: malcolm spec: - storageClassName: gp2-retain + storageClassName: io1 accessModes: - ReadWriteMany volumeMode: Filesystem @@ -116,7 +131,7 @@ This document is a rough work in progress and isn't necessarily correct (yet). - accessModes: - ReadWriteMany persistentVolumeReclaimPolicy: Retain - storageClassName: gp2-retain + storageClassName: io1 awsElasticBlockStore: fsType: xfs volumeID: aws://us-east-1a/vol-0123456789c67edd9 @@ -128,7 +143,7 @@ This document is a rough work in progress and isn't necessarily correct (yet). - name: zeek-claim namespace: malcolm spec: - storageClassName: gp2-retain + storageClassName: io1 accessModes: - ReadWriteMany volumeMode: Filesystem @@ -152,7 +167,7 @@ This document is a rough work in progress and isn't necessarily correct (yet). - accessModes: - ReadWriteMany persistentVolumeReclaimPolicy: Retain - storageClassName: gp2-retain + storageClassName: io1 awsElasticBlockStore: fsType: xfs volumeID: aws://us-east-1a/vol-0123456789dccd75e @@ -164,7 +179,7 @@ This document is a rough work in progress and isn't necessarily correct (yet). - name: suricata-claim namespace: malcolm spec: - storageClassName: gp2-retain + storageClassName: io1 accessModes: - ReadWriteMany volumeMode: Filesystem @@ -188,7 +203,7 @@ This document is a rough work in progress and isn't necessarily correct (yet). - accessModes: - ReadWriteMany persistentVolumeReclaimPolicy: Retain - storageClassName: gp2-retain + storageClassName: io1 awsElasticBlockStore: fsType: xfs volumeID: aws://us-east-1a/vol-0123456789429a231 @@ -200,7 +215,7 @@ This document is a rough work in progress and isn't necessarily correct (yet). - name: config-claim namespace: malcolm spec: - storageClassName: gp2-retain + storageClassName: io1 accessModes: - ReadWriteMany volumeMode: Filesystem @@ -224,7 +239,7 @@ This document is a rough work in progress and isn't necessarily correct (yet). - accessModes: - ReadWriteMany persistentVolumeReclaimPolicy: Retain - storageClassName: gp2-retain + storageClassName: io1 awsElasticBlockStore: fsType: xfs volumeID: aws://us-east-1a/vol-0123456789dc2ea7a @@ -236,7 +251,7 @@ This document is a rough work in progress and isn't necessarily correct (yet). - name: runtime-logs-claim namespace: malcolm spec: - storageClassName: gp2-retain + storageClassName: io1 accessModes: - ReadWriteMany volumeMode: Filesystem @@ -260,7 +275,7 @@ This document is a rough work in progress and isn't necessarily correct (yet). - accessModes: - ReadWriteMany persistentVolumeReclaimPolicy: Retain - storageClassName: gp2-retain + storageClassName: io1 awsElasticBlockStore: fsType: xfs volumeID: aws://us-east-1a/vol-01234567895ff99a1 @@ -272,7 +287,7 @@ This document is a rough work in progress and isn't necessarily correct (yet). - name: opensearch-claim namespace: malcolm spec: - storageClassName: gp2-retain + storageClassName: io1 accessModes: - ReadWriteMany volumeMode: Filesystem @@ -296,7 +311,7 @@ This document is a rough work in progress and isn't necessarily correct (yet). - accessModes: - ReadWriteMany persistentVolumeReclaimPolicy: Retain - storageClassName: gp2-retain + storageClassName: io1 awsElasticBlockStore: fsType: xfs volumeID: aws://us-east-1a/vol-01234567891150804 @@ -308,7 +323,7 @@ This document is a rough work in progress and isn't necessarily correct (yet). - name: opensearch-backup-claim namespace: malcolm spec: - storageClassName: gp2-retain + storageClassName: io1 accessModes: - ReadWriteMany volumeMode: Filesystem diff --git a/scripts/control.py b/scripts/control.py index d82df1738..e346a2cf5 100755 --- a/scripts/control.py +++ b/scripts/control.py @@ -395,17 +395,19 @@ def status(): else: eprint("Failed to display Malcolm status\n") eprint("\n".join(out)) - exit(err) elif orchMode is OrchestrationFramework.KUBERNETES: try: PrintNodeStatus() print() + except Exception as e: + if args.debug: + eprint(f'Error getting node status: {e}') + try: PrintPodStatus(namespace=args.namespace) print() except Exception as e: eprint(f'Error getting {args.namespace} status: {e}') - exit(-1) else: raise Exception(f'{sys._getframe().f_code.co_name} does not yet support {orchMode}') From 9d21b2c6cac6e656f37e71756a3056c701ec0c75 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 10 May 2023 10:25:58 -0600 Subject: [PATCH 24/47] opensearch claims can be readwriteonce --- kubernetes/01-volumes.yml.example | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kubernetes/01-volumes.yml.example b/kubernetes/01-volumes.yml.example index 60555b6da..089654eeb 100644 --- a/kubernetes/01-volumes.yml.example +++ b/kubernetes/01-volumes.yml.example @@ -226,7 +226,7 @@ spec: storage: 500Gi volumeMode: Filesystem accessModes: - - ReadWriteMany + - ReadWriteOnce persistentVolumeReclaimPolicy: Retain storageClassName: nfs mountOptions: @@ -249,7 +249,7 @@ metadata: spec: storageClassName: nfs accessModes: - - ReadWriteMany + - ReadWriteOnce volumeMode: Filesystem resources: requests: @@ -269,7 +269,7 @@ spec: storage: 500Gi volumeMode: Filesystem accessModes: - - ReadWriteMany + - ReadWriteOnce persistentVolumeReclaimPolicy: Retain storageClassName: nfs mountOptions: @@ -292,7 +292,7 @@ metadata: spec: storageClassName: nfs accessModes: - - ReadWriteMany + - ReadWriteOnce volumeMode: Filesystem resources: requests: From 22db236284711c2e967b82ed6e0e6dcc080dd193 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 10 May 2023 10:26:21 -0600 Subject: [PATCH 25/47] documentation --- docs/kubernetes-eks.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/kubernetes-eks.md b/docs/kubernetes-eks.md index c5d7fe36c..b3108fecf 100644 --- a/docs/kubernetes-eks.md +++ b/docs/kubernetes-eks.md @@ -273,7 +273,7 @@ Prerequisites: storage: 500Gi volumeMode: Filesystem accessModes: - - ReadWriteMany + - ReadWriteOnce persistentVolumeReclaimPolicy: Retain storageClassName: io1 awsElasticBlockStore: @@ -289,7 +289,7 @@ Prerequisites: spec: storageClassName: io1 accessModes: - - ReadWriteMany + - ReadWriteOnce volumeMode: Filesystem resources: requests: @@ -309,7 +309,7 @@ Prerequisites: storage: 500Gi volumeMode: Filesystem accessModes: - - ReadWriteMany + - ReadWriteOnce persistentVolumeReclaimPolicy: Retain storageClassName: io1 awsElasticBlockStore: @@ -325,7 +325,7 @@ Prerequisites: spec: storageClassName: io1 accessModes: - - ReadWriteMany + - ReadWriteOnce volumeMode: Filesystem resources: requests: From 93932d1a5311a04d0353ca0fe646b7af3496f9db Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 10 May 2023 12:24:38 -0600 Subject: [PATCH 26/47] documentation --- docs/kubernetes-eks.md | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/docs/kubernetes-eks.md b/docs/kubernetes-eks.md index b3108fecf..285a0fdbe 100644 --- a/docs/kubernetes-eks.md +++ b/docs/kubernetes-eks.md @@ -19,17 +19,22 @@ Prerequisites: ```bash kubectl --kubeconfig=malcolmeks.yaml apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml ``` +1. [Deploy]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/kubernetes/vagrant/deploy_ingress_nginx.sh) [ingress-nginx](kubernetes.md#Ingress) +1. Associate IAM OIDC provider with cluster + ```bash + eksctl utils associate-iam-oidc-provider --region=us-east-1 --cluster=cluster-name + ``` 1. [Create Amazon EBS CSI driver IAM role](https://docs.aws.amazon.com/eks/latest/userguide/csi-iam-role.html) 1. [Add the Amazon EBS CSI add-on](https://docs.aws.amazon.com/eks/latest/userguide/managing-ebs-csi.html) 1. Create volumes (**p**cap, **z**eek, **s**uricata, **c**onfig, **r**untime-**l**ogs, **o**pensearch, **b**ackup), got volume IDs ```bash - aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 500 --iops 16000 --volume-type io1 - aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 250 --iops 16000 --volume-type io1 - aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 100 --iops 16000 --volume-type io1 - aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 25 --iops 16000 --volume-type io1 - aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 25 --iops 16000 --volume-type io1 - aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 500 --iops 16000 --volume-type io1 - aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --size 500 --iops 16000 --volume-type io1 + aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --tag-specifications 'ResourceType=volume,Tags=[{Key=malcolm,Value=""}]' --size 500 --iops 4000 --volume-type io1 + aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --tag-specifications 'ResourceType=volume,Tags=[{Key=malcolm,Value=""}]' --size 250 --iops 4000 --volume-type io1 + aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --tag-specifications 'ResourceType=volume,Tags=[{Key=malcolm,Value=""}]' --size 100 --iops 4000 --volume-type io1 + aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --tag-specifications 'ResourceType=volume,Tags=[{Key=malcolm,Value=""}]' --size 25 --iops 1000 --volume-type io1 + aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --tag-specifications 'ResourceType=volume,Tags=[{Key=malcolm,Value=""}]' --size 25 --iops 1000 --volume-type io1 + aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --tag-specifications 'ResourceType=volume,Tags=[{Key=malcolm,Value=""}]' --size 500 --volume-type gp2 + aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --tag-specifications 'ResourceType=volume,Tags=[{Key=malcolm,Value=""}]' --size 500 --volume-type gp2 ``` ``` p vol-0123456789c82a042 @@ -275,7 +280,7 @@ Prerequisites: accessModes: - ReadWriteOnce persistentVolumeReclaimPolicy: Retain - storageClassName: io1 + storageClassName: gp2-retain awsElasticBlockStore: fsType: xfs volumeID: aws://us-east-1a/vol-01234567895ff99a1 @@ -287,7 +292,7 @@ Prerequisites: name: opensearch-claim namespace: malcolm spec: - storageClassName: io1 + storageClassName: gp2-retain accessModes: - ReadWriteOnce volumeMode: Filesystem @@ -311,7 +316,7 @@ Prerequisites: accessModes: - ReadWriteOnce persistentVolumeReclaimPolicy: Retain - storageClassName: io1 + storageClassName: gp2-retain awsElasticBlockStore: fsType: xfs volumeID: aws://us-east-1a/vol-01234567891150804 @@ -323,7 +328,7 @@ Prerequisites: name: opensearch-backup-claim namespace: malcolm spec: - storageClassName: io1 + storageClassName: gp2-retain accessModes: - ReadWriteOnce volumeMode: Filesystem From 3dadcd63c6ccead0560c5f9be53bdd9f56fe265b Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 10 May 2023 17:21:04 -0600 Subject: [PATCH 27/47] documentation --- docs/kubernetes-eks.md | 133 +++++++++++++++++++++-------------------- 1 file changed, 68 insertions(+), 65 deletions(-) diff --git a/docs/kubernetes-eks.md b/docs/kubernetes-eks.md index 285a0fdbe..7fb03c379 100644 --- a/docs/kubernetes-eks.md +++ b/docs/kubernetes-eks.md @@ -9,80 +9,83 @@ Prerequisites: 1. Create [VPC](https://us-east-1.console.aws.amazon.com/vpc/home?region=us-east-1#vpcs:) with subnets in 2 availability zones 1. Create [security group](https://us-east-1.console.aws.amazon.com/vpc/home?region=us-east-1#SecurityGroups:) for VPC +1. [Create and launch an EC2 instance](https://docs.aws.amazon.com/efs/latest/ug/gs-step-one-create-ec2-resources.html) +1. SSH to instance and initialize NFS subdirectories + - set up malcolm subdirectory + ```bash + sudo touch /mnt/efs/fs1/test-file.txt + sudo mkdir -p /mnt/efs/fs1/malcolm + sudo chown 1000:1000 /mnt/efs/fs1/malcolm + ``` + - `/mnt/efs/fs1/malcolm/init_storage.sh` + ```bash + #!/bin/bash + + if [ -z "$BASH_VERSION" ]; then + echo "Wrong interpreter, please run \"$0\" with bash" + exit 1 + fi + + ENCODING="utf-8" + + RUN_PATH="$(pwd)" + [[ "$(uname -s)" = 'Darwin' ]] && REALPATH=grealpath || REALPATH=realpath + [[ "$(uname -s)" = 'Darwin' ]] && DIRNAME=gdirname || DIRNAME=dirname + if ! (type "$REALPATH" && type "$DIRNAME") > /dev/null; then + echo "$(basename "${BASH_SOURCE[0]}") requires $REALPATH and $DIRNAME" + exit 1 + fi + SCRIPT_PATH="$($DIRNAME $($REALPATH -e "${BASH_SOURCE[0]}"))" + pushd "$SCRIPT_PATH" >/dev/null 2>&1 + + rm -rf ./opensearch/* ./opensearch-backup/* ./pcap/* ./suricata-logs/* ./zeek-logs/* ./config/netbox/* ./config/zeek/* + mkdir -vp ./config/auth ./config/htadmin ./config/opensearch ./config/logstash ./config/netbox/media ./config/netbox/postgres ./config/netbox/redis ./config/zeek/intel/MISP ./config/zeek/intel/STIX ./opensearch ./opensearch-backup ./pcap/upload ./pcap/processed ./suricata-logs ./zeek-logs/current ./zeek-logs/upload ./zeek-logs/extract_files + + popd >/dev/null 2>&1 + ``` + ```bash + /mnt/efs/fs1/malcolm/init_storage.sh + mkdir: created directory './config/netbox/media' + mkdir: created directory './config/netbox/postgres' + mkdir: created directory './config/netbox/redis' + mkdir: created directory './config/zeek/intel' + mkdir: created directory './config/zeek/intel/MISP' + mkdir: created directory './config/zeek/intel/STIX' + mkdir: created directory './pcap/upload' + mkdir: created directory './pcap/processed' + mkdir: created directory './zeek-logs/current' + mkdir: created directory './zeek-logs/upload' + mkdir: created directory './zeek-logs/extract_files' + ``` +1. I set up [access points](https://docs.aws.amazon.com/efs/latest/ug/efs-access-points.html), but I don't know (yet) if that will be useful + ``` + opensearch-backup /malcolm/opensearch-backup + opensearch /malcolm/opensearch + pcap /malcolm/pcap + config /malcolm/config + suricata-logs /malcolm/suricata-logs + zeek-logs /malcolm/zeek-logs + ``` 1. Create [EKS cluster](https://us-east-1.console.aws.amazon.com/eks/home?region=us-east-1#/clusters) 1. Create [node group](https://us-east-1.console.aws.amazon.com/eks/home?region=us-east-1#/clusters/cluster-name/add-node-group) 1. Generate kubeconfig file if you need to - ```bash - aws eks update-kubeconfig --region us-east-1 --name cluster-name --kubeconfig malcolmeks.yaml - ``` -1. [Deploy](https://docs.aws.amazon.com/eks/latest/userguide/metrics-server.html) `metrics-server` - ```bash - kubectl --kubeconfig=malcolmeks.yaml apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml - ``` -1. [Deploy]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/kubernetes/vagrant/deploy_ingress_nginx.sh) [ingress-nginx](kubernetes.md#Ingress) -1. Associate IAM OIDC provider with cluster - ```bash - eksctl utils associate-iam-oidc-provider --region=us-east-1 --cluster=cluster-name - ``` -1. [Create Amazon EBS CSI driver IAM role](https://docs.aws.amazon.com/eks/latest/userguide/csi-iam-role.html) -1. [Add the Amazon EBS CSI add-on](https://docs.aws.amazon.com/eks/latest/userguide/managing-ebs-csi.html) -1. Create volumes (**p**cap, **z**eek, **s**uricata, **c**onfig, **r**untime-**l**ogs, **o**pensearch, **b**ackup), got volume IDs ```bash - aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --tag-specifications 'ResourceType=volume,Tags=[{Key=malcolm,Value=""}]' --size 500 --iops 4000 --volume-type io1 - aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --tag-specifications 'ResourceType=volume,Tags=[{Key=malcolm,Value=""}]' --size 250 --iops 4000 --volume-type io1 - aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --tag-specifications 'ResourceType=volume,Tags=[{Key=malcolm,Value=""}]' --size 100 --iops 4000 --volume-type io1 - aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --tag-specifications 'ResourceType=volume,Tags=[{Key=malcolm,Value=""}]' --size 25 --iops 1000 --volume-type io1 - aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --tag-specifications 'ResourceType=volume,Tags=[{Key=malcolm,Value=""}]' --size 25 --iops 1000 --volume-type io1 - aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --tag-specifications 'ResourceType=volume,Tags=[{Key=malcolm,Value=""}]' --size 500 --volume-type gp2 - aws ec2 create-volume --region us-east-1 --availability-zone us-east-1a --tag-specifications 'ResourceType=volume,Tags=[{Key=malcolm,Value=""}]' --size 500 --volume-type gp2 - ``` - ``` - p vol-0123456789c82a042 - z vol-0123456789c67edd9 - s vol-0123456789dccd75e - c vol-0123456789429a231 - r vol-0123456789dc2ea7a - o vol-01234567895ff99a1 - b vol-01234567891150804 + aws eks update-kubeconfig --region us-east-1 --name cluster-name --kubeconfig malcolmeks.yaml ``` -1. Create EC2 instance, attach volumes - ```bash - aws ec2 attach-volume --volume-id vol-0123456789c82a042 --instance-id i-0123456789abcdef0 --device /dev/xvdp - aws ec2 attach-volume --volume-id vol-0123456789c67edd9 --instance-id i-0123456789abcdef0 --device /dev/xvdz - aws ec2 attach-volume --volume-id vol-0123456789dccd75e --instance-id i-0123456789abcdef0 --device /dev/xvds - aws ec2 attach-volume --volume-id vol-0123456789429a231 --instance-id i-0123456789abcdef0 --device /dev/xvdc - aws ec2 attach-volume --volume-id vol-0123456789dc2ea7a --instance-id i-0123456789abcdef0 --device /dev/xvdr - aws ec2 attach-volume --volume-id vol-01234567895ff99a1 --instance-id i-0123456789abcdef0 --device /dev/xvdo - aws ec2 attach-volume --volume-id vol-01234567891150804 --instance-id i-0123456789abcdef0 --device /dev/xvdb - ``` -1. Format attached volumes as XFS - ```bash - for DRV in p z s c r o b; do sudo mkfs.xfs -f /dev/xvd${DRV}; done - ``` -1. Mount drives and set permissions - ```bash - for DRV in p z s c r o b; do sudo umount -f /dev/xvd${DRV} 2>/dev/null; sudo mkdir -vp /media/xvd${DRV}; sudo mount /dev/xvd${DRV} /media/xvd${DRV}; sudo chown -R $(id -u):$(id -g) /media/xvd${DRV}; df -h /media/xvd${DRV}; done - ``` -1. Create necessary subdirectories inside of some directories (config, pcap, zeek) - ```bash - mkdir -vp /media/xvdc/{auth,htadmin,opensearch,logstash,netbox/media,netbox/postgres,netbox/redis,zeek/intel/MISP,zeek/intel/STIX} - mkdir -vp /media/xvdp/{upload,proceessed} - mkdir -vp /media/xvdz/{current,upload,extract_files} - ``` -1. Unmount drives +1. [Deploy](https://docs.aws.amazon.com/eks/latest/userguide/metrics-server.html) `metrics-server` ```bash - for DRV in p z s c r o b; do sudo umount -f /dev/xvd${DRV}; done + kubectl --kubeconfig=malcolmeks.yaml apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml ``` -1. Detach volumes +1. [Deploy]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/kubernetes/vagrant/deploy_ingress_nginx.sh) [ingress-nginx](kubernetes.md#Ingress) +1. Associate IAM OIDC provider with cluster ```bash - aws ec2 detach-volume --volume-id vol-0123456789c82a042 --instance-id i-0123456789abcdef0 - aws ec2 detach-volume --volume-id vol-0123456789c67edd9 --instance-id i-0123456789abcdef0 - aws ec2 detach-volume --volume-id vol-0123456789dccd75e --instance-id i-0123456789abcdef0 - aws ec2 detach-volume --volume-id vol-0123456789429a231 --instance-id i-0123456789abcdef0 - aws ec2 detach-volume --volume-id vol-0123456789dc2ea7a --instance-id i-0123456789abcdef0 - aws ec2 detach-volume --volume-id vol-01234567895ff99a1 --instance-id i-0123456789abcdef0 - aws ec2 detach-volume --volume-id vol-01234567891150804 --instance-id i-0123456789abcdef0 + eksctl utils associate-iam-oidc-provider --region=us-east-1 --cluster=cluster-name ``` +1. [deploy Amazon EFS CSI driver](https://docs.aws.amazon.com/eks/latest/userguide/efs-csi.html) + * look at **Prerequisites** + * do **Create an IAM policy and role** + * do **Install the Amazon EFS driver** + * do **Create an Amazon [EFS file system](https://docs.aws.amazon.com/efs/latest/ug/gs-step-two-create-efs-resources.html)** 1. Create YAML for persistent volumes and volume claims from the EBS Volume ID ```yaml --- From 2ef991d4b3a13e2b6f0ab6a58bb2870c2823a283 Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 11 May 2023 09:18:39 -0600 Subject: [PATCH 28/47] documentation wip --- docs/kubernetes-eks.md | 139 +++++++++++++++++++++-------------------- 1 file changed, 71 insertions(+), 68 deletions(-) diff --git a/docs/kubernetes-eks.md b/docs/kubernetes-eks.md index 7fb03c379..1ffa7fcbb 100644 --- a/docs/kubernetes-eks.md +++ b/docs/kubernetes-eks.md @@ -9,6 +9,26 @@ Prerequisites: 1. Create [VPC](https://us-east-1.console.aws.amazon.com/vpc/home?region=us-east-1#vpcs:) with subnets in 2 availability zones 1. Create [security group](https://us-east-1.console.aws.amazon.com/vpc/home?region=us-east-1#SecurityGroups:) for VPC +1. Create [EKS cluster](https://us-east-1.console.aws.amazon.com/eks/home?region=us-east-1#/clusters) +1. Generate kubeconfig file if you need to + ```bash + aws eks update-kubeconfig --region us-east-1 --name cluster-name --kubeconfig malcolmeks.yaml + ``` +1. Create [node group](https://us-east-1.console.aws.amazon.com/eks/home?region=us-east-1#/clusters/cluster-name/add-node-group) +1. [Deploy](https://docs.aws.amazon.com/eks/latest/userguide/metrics-server.html) `metrics-server` + ```bash + kubectl --kubeconfig=malcolmeks.yaml apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml + ``` +1. [Deploy]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/kubernetes/vagrant/deploy_ingress_nginx.sh) [ingress-nginx](kubernetes.md#Ingress) +1. Associate IAM OIDC provider with cluster + ```bash + eksctl utils associate-iam-oidc-provider --region=us-east-1 --cluster=cluster-name --approve + ``` +1. [deploy Amazon EFS CSI driver](https://docs.aws.amazon.com/eks/latest/userguide/efs-csi.html) + * look at **Prerequisites** + * do **Create an IAM policy and role** + * do **Install the Amazon EFS driver** + * do **Create an Amazon [EFS file system](https://docs.aws.amazon.com/efs/latest/ug/gs-step-two-create-efs-resources.html)** 1. [Create and launch an EC2 instance](https://docs.aws.amazon.com/efs/latest/ug/gs-step-one-create-ec2-resources.html) 1. SSH to instance and initialize NFS subdirectories - set up malcolm subdirectory @@ -38,8 +58,8 @@ Prerequisites: SCRIPT_PATH="$($DIRNAME $($REALPATH -e "${BASH_SOURCE[0]}"))" pushd "$SCRIPT_PATH" >/dev/null 2>&1 - rm -rf ./opensearch/* ./opensearch-backup/* ./pcap/* ./suricata-logs/* ./zeek-logs/* ./config/netbox/* ./config/zeek/* - mkdir -vp ./config/auth ./config/htadmin ./config/opensearch ./config/logstash ./config/netbox/media ./config/netbox/postgres ./config/netbox/redis ./config/zeek/intel/MISP ./config/zeek/intel/STIX ./opensearch ./opensearch-backup ./pcap/upload ./pcap/processed ./suricata-logs ./zeek-logs/current ./zeek-logs/upload ./zeek-logs/extract_files + rm -rf ./opensearch/* ./opensearch-backup/* ./pcap/* ./suricata-logs/* ./zeek-logs/* ./config/netbox/* ./config/zeek/* ./runtime-logs/* + mkdir -vp ./config/auth ./config/htadmin ./config/opensearch ./config/logstash ./config/netbox/media ./config/netbox/postgres ./config/netbox/redis ./config/zeek/intel/MISP ./config/zeek/intel/STIX ./opensearch ./opensearch-backup ./pcap/upload ./pcap/processed ./suricata-logs ./zeek-logs/current ./zeek-logs/upload ./zeek-logs/extract_files ./runtime-logs popd >/dev/null 2>&1 ``` @@ -56,39 +76,22 @@ Prerequisites: mkdir: created directory './zeek-logs/current' mkdir: created directory './zeek-logs/upload' mkdir: created directory './zeek-logs/extract_files' + mkdir: created directory './runtime-logs' ``` -1. I set up [access points](https://docs.aws.amazon.com/efs/latest/ug/efs-access-points.html), but I don't know (yet) if that will be useful - ``` - opensearch-backup /malcolm/opensearch-backup - opensearch /malcolm/opensearch - pcap /malcolm/pcap - config /malcolm/config - suricata-logs /malcolm/suricata-logs - zeek-logs /malcolm/zeek-logs - ``` -1. Create [EKS cluster](https://us-east-1.console.aws.amazon.com/eks/home?region=us-east-1#/clusters) -1. Create [node group](https://us-east-1.console.aws.amazon.com/eks/home?region=us-east-1#/clusters/cluster-name/add-node-group) -1. Generate kubeconfig file if you need to - ```bash - aws eks update-kubeconfig --region us-east-1 --name cluster-name --kubeconfig malcolmeks.yaml - ``` -1. [Deploy](https://docs.aws.amazon.com/eks/latest/userguide/metrics-server.html) `metrics-server` - ```bash - kubectl --kubeconfig=malcolmeks.yaml apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml - ``` -1. [Deploy]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/kubernetes/vagrant/deploy_ingress_nginx.sh) [ingress-nginx](kubernetes.md#Ingress) -1. Associate IAM OIDC provider with cluster - ```bash - eksctl utils associate-iam-oidc-provider --region=us-east-1 --cluster=cluster-name - ``` -1. [deploy Amazon EFS CSI driver](https://docs.aws.amazon.com/eks/latest/userguide/efs-csi.html) - * look at **Prerequisites** - * do **Create an IAM policy and role** - * do **Install the Amazon EFS driver** - * do **Create an Amazon [EFS file system](https://docs.aws.amazon.com/efs/latest/ug/gs-step-two-create-efs-resources.html)** -1. Create YAML for persistent volumes and volume claims from the EBS Volume ID +1. Set up [access points](https://docs.aws.amazon.com/efs/latest/ug/efs-access-points.html), and note the **Access point ID**s to put in your YAML in the next step + + | name | mountpoint | access point ID | + | ----------------- | -------------------------- | ---------------------- | + | config | /malcolm/config | fsap-config | + | opensearch | /malcolm/opensearch | fsap-opensearch | + | opensearch-backup | /malcolm/opensearch-backup | fsap-opensearch-backup | + | pcap | /malcolm/pcap | fsap-pcap | + | runtime-logs | /malcolm/runtime-logs | fsap-runtime-logs | + | suricata-logs | /malcolm/suricata-logs | fsap-suricata-logs | + | zeek-logs | /malcolm/zeek-logs | fsap-zeek-logs | + +1. Create YAML for persistent volumes and volume claims from the EBS Volume ID. In this example, replace `fs-FILESYSTEMID` with your EFS filesystem ID and `fsap-XXXXXXXX` with the appropriate access point ID ```yaml - --- apiVersion: v1 kind: PersistentVolume metadata: @@ -103,10 +106,10 @@ Prerequisites: accessModes: - ReadWriteMany persistentVolumeReclaimPolicy: Retain - storageClassName: io1 - awsElasticBlockStore: - fsType: xfs - volumeID: aws://us-east-1a/vol-0123456789c82a042 + storageClassName: efs-sc + csi: + driver: efs.csi.aws.com + volumeHandle: fs-FILESYSTEMID::fsap-pcap --- apiVersion: v1 @@ -115,7 +118,7 @@ Prerequisites: name: pcap-claim namespace: malcolm spec: - storageClassName: io1 + storageClassName: efs-sc accessModes: - ReadWriteMany volumeMode: Filesystem @@ -139,10 +142,10 @@ Prerequisites: accessModes: - ReadWriteMany persistentVolumeReclaimPolicy: Retain - storageClassName: io1 - awsElasticBlockStore: - fsType: xfs - volumeID: aws://us-east-1a/vol-0123456789c67edd9 + storageClassName: efs-sc + csi: + driver: efs.csi.aws.com + volumeHandle: fs-FILESYSTEMID::fsap-zeek-logs --- apiVersion: v1 @@ -151,7 +154,7 @@ Prerequisites: name: zeek-claim namespace: malcolm spec: - storageClassName: io1 + storageClassName: efs-sc accessModes: - ReadWriteMany volumeMode: Filesystem @@ -175,10 +178,10 @@ Prerequisites: accessModes: - ReadWriteMany persistentVolumeReclaimPolicy: Retain - storageClassName: io1 - awsElasticBlockStore: - fsType: xfs - volumeID: aws://us-east-1a/vol-0123456789dccd75e + storageClassName: efs-sc + csi: + driver: efs.csi.aws.com + volumeHandle: fs-FILESYSTEMID::fsap-suricata-logs --- apiVersion: v1 @@ -187,7 +190,7 @@ Prerequisites: name: suricata-claim namespace: malcolm spec: - storageClassName: io1 + storageClassName: efs-sc accessModes: - ReadWriteMany volumeMode: Filesystem @@ -211,10 +214,10 @@ Prerequisites: accessModes: - ReadWriteMany persistentVolumeReclaimPolicy: Retain - storageClassName: io1 - awsElasticBlockStore: - fsType: xfs - volumeID: aws://us-east-1a/vol-0123456789429a231 + storageClassName: efs-sc + csi: + driver: efs.csi.aws.com + volumeHandle: fs-FILESYSTEMID::fsap-config --- apiVersion: v1 @@ -223,7 +226,7 @@ Prerequisites: name: config-claim namespace: malcolm spec: - storageClassName: io1 + storageClassName: efs-sc accessModes: - ReadWriteMany volumeMode: Filesystem @@ -247,10 +250,10 @@ Prerequisites: accessModes: - ReadWriteMany persistentVolumeReclaimPolicy: Retain - storageClassName: io1 - awsElasticBlockStore: - fsType: xfs - volumeID: aws://us-east-1a/vol-0123456789dc2ea7a + storageClassName: efs-sc + csi: + driver: efs.csi.aws.com + volumeHandle: fs-02997421cdc55b8e4::fsap-runtime-logs --- apiVersion: v1 @@ -259,7 +262,7 @@ Prerequisites: name: runtime-logs-claim namespace: malcolm spec: - storageClassName: io1 + storageClassName: efs-sc accessModes: - ReadWriteMany volumeMode: Filesystem @@ -283,10 +286,10 @@ Prerequisites: accessModes: - ReadWriteOnce persistentVolumeReclaimPolicy: Retain - storageClassName: gp2-retain - awsElasticBlockStore: - fsType: xfs - volumeID: aws://us-east-1a/vol-01234567895ff99a1 + storageClassName: efs-sc + csi: + driver: efs.csi.aws.com + volumeHandle: fs-FILESYSTEMID::fsap-opensearch --- apiVersion: v1 @@ -295,7 +298,7 @@ Prerequisites: name: opensearch-claim namespace: malcolm spec: - storageClassName: gp2-retain + storageClassName: efs-sc accessModes: - ReadWriteOnce volumeMode: Filesystem @@ -319,10 +322,10 @@ Prerequisites: accessModes: - ReadWriteOnce persistentVolumeReclaimPolicy: Retain - storageClassName: gp2-retain - awsElasticBlockStore: - fsType: xfs - volumeID: aws://us-east-1a/vol-01234567891150804 + storageClassName: efs-sc + csi: + driver: efs.csi.aws.com + volumeHandle: fs-FILESYSTEMID::fsap-opensearch-backup --- apiVersion: v1 @@ -331,7 +334,7 @@ Prerequisites: name: opensearch-backup-claim namespace: malcolm spec: - storageClassName: gp2-retain + storageClassName: efs-sc accessModes: - ReadWriteOnce volumeMode: Filesystem From 4d97107fab656cc8e134f91896933feea7b99a8a Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 11 May 2023 09:37:49 -0600 Subject: [PATCH 29/47] documentation wip --- docs/kubernetes-eks.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/kubernetes-eks.md b/docs/kubernetes-eks.md index 1ffa7fcbb..581f41ce7 100644 --- a/docs/kubernetes-eks.md +++ b/docs/kubernetes-eks.md @@ -59,7 +59,7 @@ Prerequisites: pushd "$SCRIPT_PATH" >/dev/null 2>&1 rm -rf ./opensearch/* ./opensearch-backup/* ./pcap/* ./suricata-logs/* ./zeek-logs/* ./config/netbox/* ./config/zeek/* ./runtime-logs/* - mkdir -vp ./config/auth ./config/htadmin ./config/opensearch ./config/logstash ./config/netbox/media ./config/netbox/postgres ./config/netbox/redis ./config/zeek/intel/MISP ./config/zeek/intel/STIX ./opensearch ./opensearch-backup ./pcap/upload ./pcap/processed ./suricata-logs ./zeek-logs/current ./zeek-logs/upload ./zeek-logs/extract_files ./runtime-logs + mkdir -vp ./config/auth ./config/htadmin ./config/opensearch ./config/logstash ./config/netbox/media ./config/netbox/postgres ./config/netbox/redis ./config/zeek/intel/MISP ./config/zeek/intel/STIX ./opensearch ./opensearch-backup ./pcap/upload ./pcap/processed ./suricata-logs ./zeek-logs/current ./zeek-logs/upload ./zeek-logs/extract_files ./runtime-logs/arkime ./runtime-logs/nginx popd >/dev/null 2>&1 ``` From 8ecc88bc74f9d95a503f6fffb9216c23c47840aa Mon Sep 17 00:00:00 2001 From: SG Date: Thu, 11 May 2023 11:19:04 -0600 Subject: [PATCH 30/47] documentation --- docs/README.md | 5 +++++ docs/kubernetes-eks.md | 35 ++++++++++++++++++++--------------- docs/kubernetes.md | 1 + 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/docs/README.md b/docs/README.md index cdcc3f161..1e9e9d0d2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -98,6 +98,11 @@ For smaller networks, use at home by network security enthusiasts, or in the fie - [Setup](malcolm-iso.md#ISOSetup) - [Time synchronization](time-sync.md#ConfigTime) * [Deploying Malcolm with Kubernetes](kubernetes.md#Kubernetes) + - [Configuration](kubernetes.md#Config) + - [Running Malcolm](kubernetes.md#Running) + - [Deployment Example](kubernetes.md#Example) + - [Future Enhancements](kubernetes.md#Future) + - [Deploying Malcolm on Amazon Elastic Kubernetes Service (EKS)](kubernetes-eks.md#KubernetesEKS) * [Hardening](hardening.md#Hardening) - [Compliance Exceptions](hardening.md#ComplianceExceptions) * [Installation example using Ubuntu 22.04 LTS](ubuntu-install-example.md#InstallationExample) diff --git a/docs/kubernetes-eks.md b/docs/kubernetes-eks.md index 581f41ce7..85e5ed745 100644 --- a/docs/kubernetes-eks.md +++ b/docs/kubernetes-eks.md @@ -1,35 +1,39 @@ -## Notes for deploying Malcolm on AWS EKS +# Deploying Malcolm on Amazon Elastic Kubernetes Service (EKS) -This document is a rough work in progress and isn't necessarily correct (yet). -SG +This document outlines the process of setting up a cluster on [Amazon Elastic Kubernetes Service (EKS)](https://aws.amazon.com/eks/) using [Amazon Web Services](https://aws.amazon.com/) in preparation for [**Deploying Malcolm with Kubernetes**](kubernetes.md). -Prerequisites: +This is a work-in-progress document that is still a bit rough around the edges. You'll need to replace things like `cluster-name` and `us-east-1` with the values that are appliable to your cluster. Any feedback is welcome in the [relevant issue](https://github.com/idaholab/Malcolm/issues/194) on GitHub. -* [aws cli](https://aws.amazon.com/cli/) +## Prerequisites + +* [aws cli](https://aws.amazon.com/cli/) with functioning access to your AWS infrastructure * [eksctl](https://eksctl.io/) -1. Create [VPC](https://us-east-1.console.aws.amazon.com/vpc/home?region=us-east-1#vpcs:) with subnets in 2 availability zones -1. Create [security group](https://us-east-1.console.aws.amazon.com/vpc/home?region=us-east-1#SecurityGroups:) for VPC -1. Create [EKS cluster](https://us-east-1.console.aws.amazon.com/eks/home?region=us-east-1#/clusters) -1. Generate kubeconfig file if you need to +## Procedure + +1. Create a [VPC](https://us-east-1.console.aws.amazon.com/vpc/home?region=us-east-1#vpcs:) with subnets in 2 or more availability zones +1. Create a [security group](https://us-east-1.console.aws.amazon.com/vpc/home?region=us-east-1#SecurityGroups:) for VPC +1. Create an [EKS cluster](https://us-east-1.console.aws.amazon.com/eks/home?region=us-east-1#/clusters) +1. Generate a kubeconfig file to use with Malcolm's control scripts (`malcolmeks.yaml` is used in this example) ```bash aws eks update-kubeconfig --region us-east-1 --name cluster-name --kubeconfig malcolmeks.yaml ``` -1. Create [node group](https://us-east-1.console.aws.amazon.com/eks/home?region=us-east-1#/clusters/cluster-name/add-node-group) +1. Create a [node group](https://us-east-1.console.aws.amazon.com/eks/home?region=us-east-1#/clusters/cluster-name/add-node-group) 1. [Deploy](https://docs.aws.amazon.com/eks/latest/userguide/metrics-server.html) `metrics-server` ```bash kubectl --kubeconfig=malcolmeks.yaml apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml ``` -1. [Deploy]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/kubernetes/vagrant/deploy_ingress_nginx.sh) [ingress-nginx](kubernetes.md#Ingress) +1. [Deploy]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/kubernetes/vagrant/deploy_ingress_nginx.sh) ingress-nginx as described [here](kubernetes.md#Ingress) 1. Associate IAM OIDC provider with cluster ```bash eksctl utils associate-iam-oidc-provider --region=us-east-1 --cluster=cluster-name --approve ``` 1. [deploy Amazon EFS CSI driver](https://docs.aws.amazon.com/eks/latest/userguide/efs-csi.html) - * look at **Prerequisites** - * do **Create an IAM policy and role** - * do **Install the Amazon EFS driver** - * do **Create an Amazon [EFS file system](https://docs.aws.amazon.com/efs/latest/ug/gs-step-two-create-efs-resources.html)** -1. [Create and launch an EC2 instance](https://docs.aws.amazon.com/efs/latest/ug/gs-step-one-create-ec2-resources.html) + * review **Prerequisites** + * follow steps for **Create an IAM policy and role** + * follow steps for **Install the Amazon EFS driver** + * follow steps for **Create an Amazon [EFS file system](https://docs.aws.amazon.com/efs/latest/ug/gs-step-two-create-efs-resources.html)** +1. [Create and launch an EC2 instance](https://docs.aws.amazon.com/efs/latest/ug/gs-step-one-create-ec2-resources.html) for initializing the directory structure on the EFS filesystem (this can be a very small instance, e.g., t2.micro). Make sure when configuring this instance you give configure to the EFS file system in the storage configuration. 1. SSH to instance and initialize NFS subdirectories - set up malcolm subdirectory ```bash @@ -343,3 +347,4 @@ Prerequisites: storage: 500Gi volumeName: opensearch-backup-volume ``` +1. Finish [configuring](kubernetes.md#Config) and [configuring](kubernetes.md#Running) Malcolm as described in [**Deploying Malcolm with Kubernetes**](kubernetes.md) \ No newline at end of file diff --git a/docs/kubernetes.md b/docs/kubernetes.md index 0f8411462..9c7fea917 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -13,6 +13,7 @@ - [Live Traffic Analysis](#FutureLiveCap) - [Horizontal Scaling](#FutureScaleOut) - [Helm Chart](#FutureHelmChart) +* [Deploying Malcolm on Amazon Elastic Kubernetes Service (EKS)](kubernetes-eks.md#KubernetesEKS) ## System From 4a9e060eccc2f7238edafd402dbf18c3aa47ba1c Mon Sep 17 00:00:00 2001 From: SG Date: Fri, 12 May 2023 07:36:04 -0600 Subject: [PATCH 31/47] tweak deploy_ingress_nginx to allow low providing external access to multiple Kubernetes services in my Amazon EKS cluster (see https://repost.aws/knowledge-center/eks-access-kubernetes-services) --- kubernetes/vagrant/deploy_ingress_nginx.sh | 34 ++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/kubernetes/vagrant/deploy_ingress_nginx.sh b/kubernetes/vagrant/deploy_ingress_nginx.sh index 2dfbbbaf7..96e84e318 100755 --- a/kubernetes/vagrant/deploy_ingress_nginx.sh +++ b/kubernetes/vagrant/deploy_ingress_nginx.sh @@ -16,10 +16,20 @@ ENCODING="utf-8" # script variables LOAD_BALANCER_ADDITIONAL_PORTS="{\"appProtocol\": \"tcp\", \"name\": \"lumberjack\", \"port\": 5044, \"targetPort\": 5044, \"protocol\": \"TCP\"}, {\"appProtocol\": \"tcp\", \"name\": \"tcpjson\", \"port\": 5045, \"targetPort\": 5045, \"protocol\": \"TCP\"}, {\"appProtocol\": \"tcp\", \"name\": \"sftp\", \"port\": 8022, \"targetPort\": 8022, \"protocol\": \"TCP\"}, {\"appProtocol\": \"tcp\", \"name\": \"opensearch\", \"port\": 9200, \"targetPort\": 9200, \"protocol\": \"TCP\"}" DEPLOYMENT_ADDITIONAL_PORTS="{\"name\": \"lumberjack\", \"containerPort\": 5044, \"protocol\": \"TCP\"}, {\"name\": \"tcpjson\", \"containerPort\": 5045, \"protocol\": \"TCP\"}, {\"name\": \"sftp\", \"containerPort\": 8022, \"protocol\": \"TCP\"}, {\"name\": \"opensearch\", \"containerPort\": 9200, \"protocol\": \"TCP\"}" +AWS_EXPOSE_ANNOTATIONS=( + # see https://repost.aws/knowledge-center/eks-access-kubernetes-services (Option 1), step 2. + "{\"service.beta.kubernetes.io/aws-load-balancer-backend-protocol\":\"tcp\"}" + "{\"service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled\":\"true\"}" + "{\"service.beta.kubernetes.io/aws-load-balancer-type\":\"external\"}" + "{\"service.beta.kubernetes.io/aws-load-balancer-nlb-target-type\":\"instance\"}" + "{\"service.beta.kubernetes.io/aws-load-balancer-scheme\":\"internet-facing\"}" +) INGRESS_NGINX_CONTROLLER_VERSION=1.7.0 KUBECONFIG= DEPLOY_YAML_FILE= DRY_RUN=none +INGRESS_NGINX_PROVIDER=cloud +EXPOSE_VIA_AWS_LB= ############################################################################### # show script usage @@ -30,12 +40,14 @@ function help() { echo "-k kubeconfig kubeconfig file" >&2 echo "-d dryrunval --dry-run=dryrunval for kubectl (dryrunval=none|server|client)" >&2 echo "-i version ingress-nginx controller version" >&2 + echo "-a use AWS provider for ingress-nginx" >&2 + echo "-e expose ingress-nginx via AWS load balancer" >&2 exit 1 } ############################################################################### # parse command-line parameters -while getopts 'vhd:k:i:' OPTION; do +while getopts 'vhaed:k:i:' OPTION; do case "$OPTION" in v) @@ -47,6 +59,15 @@ while getopts 'vhd:k:i:' OPTION; do DRY_RUN="${OPTARG}" ;; + a) + INGRESS_NGINX_PROVIDER="aws" + ;; + + e) + INGRESS_NGINX_PROVIDER="aws" + EXPOSE_VIA_AWS_LB="true" + ;; + k) KUBECONFIG="${OPTARG}" ;; @@ -84,10 +105,19 @@ trap "cleanup" EXIT DEPLOY_YAML_FILE="$(mktemp --suffix=.yaml)" -curl -fsSL -o "${DEPLOY_YAML_FILE}" "https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v${INGRESS_NGINX_CONTROLLER_VERSION}/deploy/static/provider/cloud/deploy.yaml" +curl -fsSL -o "${DEPLOY_YAML_FILE}" "https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v${INGRESS_NGINX_CONTROLLER_VERSION}/deploy/static/provider/${INGRESS_NGINX_PROVIDER}/deploy.yaml" yq -i '( select(.kind == "Deployment").spec.template.spec.containers[].args[] | select(contains("/nginx-ingress-controller")) | parent ) += ["--enable-ssl-passthrough", "--tcp-services-configmap=ingress-nginx/tcp-services"]' "${DEPLOY_YAML_FILE}" yq -i "( select(.kind == \"Deployment\").spec.template.spec.containers[].args[] | select(contains(\"/nginx-ingress-controller\")) | parent | parent | .ports ) += [${DEPLOYMENT_ADDITIONAL_PORTS}]" "${DEPLOY_YAML_FILE}" yq -i "( select(.kind == \"Service\" and .spec.type == \"LoadBalancer\").spec.ports ) += [${LOAD_BALANCER_ADDITIONAL_PORTS}]" "${DEPLOY_YAML_FILE}" +if [[ "${EXPOSE_VIA_AWS_LB}" == "true" ]]; then + # see https://repost.aws/knowledge-center/eks-access-kubernetes-services (Option 1), step 2. + for OLDKEY in $(yq "select(.kind == \"Service\" and .spec.type == \"LoadBalancer\").metadata.annotations | keys | .[] | select(. == \"service.beta.kubernetes.io*\")" "${DEPLOY_YAML_FILE}"); do + yq -i "select(.kind == \"Service\" and .spec.type == \"LoadBalancer\") | del(.metadata.annotations.\"$OLDKEY\")" "${DEPLOY_YAML_FILE}" + done + for NEWKEY in ${AWS_EXPOSE_ANNOTATIONS[@]}; do + yq -i "( select(.kind == \"Service\" and .spec.type == \"LoadBalancer\").metadata.annotations ) += ${NEWKEY}" "${DEPLOY_YAML_FILE}" + done +fi [[ -n "${VERBOSE_FLAG}" ]] && cat "${DEPLOY_YAML_FILE}" From 2f8baeaa716017aab51d70681d245dd5fd7188e7 Mon Sep 17 00:00:00 2001 From: SG Date: Fri, 12 May 2023 07:40:28 -0600 Subject: [PATCH 32/47] documentation --- docs/kubernetes-eks.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/kubernetes-eks.md b/docs/kubernetes-eks.md index 85e5ed745..62f3b1293 100644 --- a/docs/kubernetes-eks.md +++ b/docs/kubernetes-eks.md @@ -23,7 +23,7 @@ This is a work-in-progress document that is still a bit rough around the edges. ```bash kubectl --kubeconfig=malcolmeks.yaml apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml ``` -1. [Deploy]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/kubernetes/vagrant/deploy_ingress_nginx.sh) ingress-nginx as described [here](kubernetes.md#Ingress) +1. Deploy ingress-nginx as described [here](kubernetes.md#Ingress). [This script (`deploy_ingress_nginx.sh`)]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/kubernetes/vagrant/deploy_ingress_nginx.sh) may be helpful in doing so. To [provide external access](https://repost.aws/knowledge-center/eks-access-kubernetes-services) to services in the EKS cluster, pass `-a -e` to `deploy_ingress_nginx.sh` 1. Associate IAM OIDC provider with cluster ```bash eksctl utils associate-iam-oidc-provider --region=us-east-1 --cluster=cluster-name --approve From 029ce4ee2823731ab4082015efd9c15c4ef97347 Mon Sep 17 00:00:00 2001 From: SG Date: Fri, 12 May 2023 08:43:03 -0600 Subject: [PATCH 33/47] Script poc for creating eks cluster --- kubernetes/eks/aws_eks_malcolm_prep.sh | 85 ++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100755 kubernetes/eks/aws_eks_malcolm_prep.sh diff --git a/kubernetes/eks/aws_eks_malcolm_prep.sh b/kubernetes/eks/aws_eks_malcolm_prep.sh new file mode 100755 index 000000000..0e863df41 --- /dev/null +++ b/kubernetes/eks/aws_eks_malcolm_prep.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash + +if [ -z "$BASH_VERSION" ]; then + echo "Wrong interpreter, please run \"$0\" with bash" >&2 + exit 1 +fi + +############################################################################### +# script options +set -o pipefail +set -e +shopt -s nocasematch +ENCODING="utf-8" + +############################################################################### +# script variables +VERBOSE_FLAG= +DRY_RUN=none +REGION=us-east-1 +CLUSTER_NAME= + +############################################################################### +# show script usage +function help() { + echo -e "$(basename $0)\n" + echo "-v enable bash verbosity" + echo "-c cluster-name cluster name" + echo "-d dryrunval --dry-run=dryrunval for kubectl (dryrunval=none|server|client)" + echo "-r region region code (default: us-east-1)" + exit 1 +} + +############################################################################### +# parse command-line parameters +while getopts 'vc:d:' OPTION; do + case "$OPTION" in + + v) + VERBOSE_FLAG="-v" + set -x + ;; + + c) + CLUSTER_NAME="${OPTARG}" + ;; + + d) + DRY_RUN="${OPTARG}" + ;; + + ?) + help >&2 + exit 1; + ;; + + esac +done +shift "$(($OPTIND -1))" + +############################################################################### +function cleanup { + true +} + +if ! command -v aws >/dev/null 2>&1 || ! command -v eksctl >/dev/null 2>&1 || ! command -v kubectl >/dev/null 2>&1 || ! command -v jq >/dev/null 2>&1; then + echo "$(basename $0) requires aws, eksctl kubectl and jq" >&2 + exit 1 +fi + +############################################################################### + +if [[ -z "${CLUSTER_NAME}" ]]; then + echo "Please specify -c cluster-name" >&2 + help >&2 + exit 1 +fi + +ACCOUNT="$(aws --output=json sts get-caller-identity | jq -r '.Account')" +if [[ -z "${ACCOUNT}" ]]; then + echo "Could not get account ID from aws sts get-caller-identity" >&2 + exit 1 +fi +[[ -n "${VERBOSE_FLAG}" ]] && echo "Account ID: ${ACCOUNT}" + +trap "cleanup" EXIT From 66e5c2866ac4438f4e7797cb2a3881a1d53120a8 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Fri, 12 May 2023 15:53:02 -0600 Subject: [PATCH 34/47] convenience script for deploying ingress-nginx --- kubernetes/vagrant/Vagrantfile | 2 +- kubernetes/vagrant/deploy_ingress_nginx.sh | 178 ++++++++++++++------- 2 files changed, 119 insertions(+), 61 deletions(-) diff --git a/kubernetes/vagrant/Vagrantfile b/kubernetes/vagrant/Vagrantfile index d76a5780b..fefc7d6d8 100644 --- a/kubernetes/vagrant/Vagrantfile +++ b/kubernetes/vagrant/Vagrantfile @@ -111,7 +111,7 @@ server_script_1 = <<-SHELL curl -sfL https://get.k3s.io | sh - echo "Waiting for k3s to start..." sleep 30 - bash /tmp/deploy_ingress_nginx.sh -k /etc/rancher/k3s/k3s.yaml + bash /tmp/deploy_ingress_nginx.sh -s -t -k /etc/rancher/k3s/k3s.yaml until [ -f /var/lib/rancher/k3s/server/token ] && [ -f /etc/rancher/k3s/k3s.yaml ]; do sleep 5; done cp -v /var/lib/rancher/k3s/server/token /vagrant_shared cp -v /etc/rancher/k3s/k3s.yaml /vagrant_shared diff --git a/kubernetes/vagrant/deploy_ingress_nginx.sh b/kubernetes/vagrant/deploy_ingress_nginx.sh index 96e84e318..b9d17be93 100755 --- a/kubernetes/vagrant/deploy_ingress_nginx.sh +++ b/kubernetes/vagrant/deploy_ingress_nginx.sh @@ -17,77 +17,103 @@ ENCODING="utf-8" LOAD_BALANCER_ADDITIONAL_PORTS="{\"appProtocol\": \"tcp\", \"name\": \"lumberjack\", \"port\": 5044, \"targetPort\": 5044, \"protocol\": \"TCP\"}, {\"appProtocol\": \"tcp\", \"name\": \"tcpjson\", \"port\": 5045, \"targetPort\": 5045, \"protocol\": \"TCP\"}, {\"appProtocol\": \"tcp\", \"name\": \"sftp\", \"port\": 8022, \"targetPort\": 8022, \"protocol\": \"TCP\"}, {\"appProtocol\": \"tcp\", \"name\": \"opensearch\", \"port\": 9200, \"targetPort\": 9200, \"protocol\": \"TCP\"}" DEPLOYMENT_ADDITIONAL_PORTS="{\"name\": \"lumberjack\", \"containerPort\": 5044, \"protocol\": \"TCP\"}, {\"name\": \"tcpjson\", \"containerPort\": 5045, \"protocol\": \"TCP\"}, {\"name\": \"sftp\", \"containerPort\": 8022, \"protocol\": \"TCP\"}, {\"name\": \"opensearch\", \"containerPort\": 9200, \"protocol\": \"TCP\"}" AWS_EXPOSE_ANNOTATIONS=( - # see https://repost.aws/knowledge-center/eks-access-kubernetes-services (Option 1), step 2. - "{\"service.beta.kubernetes.io/aws-load-balancer-backend-protocol\":\"tcp\"}" - "{\"service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled\":\"true\"}" - "{\"service.beta.kubernetes.io/aws-load-balancer-type\":\"external\"}" - "{\"service.beta.kubernetes.io/aws-load-balancer-nlb-target-type\":\"instance\"}" - "{\"service.beta.kubernetes.io/aws-load-balancer-scheme\":\"internet-facing\"}" + # see https://repost.aws/knowledge-center/eks-access-kubernetes-services (Option 1), step 2. + "{\"service.beta.kubernetes.io/aws-load-balancer-backend-protocol\":\"tcp\"}" + "{\"service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled\":\"true\"}" + "{\"service.beta.kubernetes.io/aws-load-balancer-type\":\"external\"}" + "{\"service.beta.kubernetes.io/aws-load-balancer-nlb-target-type\":\"instance\"}" + "{\"service.beta.kubernetes.io/aws-load-balancer-scheme\":\"internet-facing\"}" ) INGRESS_NGINX_CONTROLLER_VERSION=1.7.0 KUBECONFIG= -DEPLOY_YAML_FILE= +WORKDIR= DRY_RUN=none INGRESS_NGINX_PROVIDER=cloud EXPOSE_VIA_AWS_LB= +SSL_PASSTHROUGH= +OTHER_TCP_SERVICES= ############################################################################### # show script usage function help() { - echo -e "$(basename $0)\n" >&2 - echo "-v enable bash verbosity" >&2 - echo "-h display help" >&2 - echo "-k kubeconfig kubeconfig file" >&2 - echo "-d dryrunval --dry-run=dryrunval for kubectl (dryrunval=none|server|client)" >&2 - echo "-i version ingress-nginx controller version" >&2 - echo "-a use AWS provider for ingress-nginx" >&2 - echo "-e expose ingress-nginx via AWS load balancer" >&2 + echo -e "\n$(basename $0)\n" + echo -e "-h display help\n" + echo -e "-v enable bash verbosity\n" + echo -e "-k kubeconfig kubeconfig file\n" + echo -e "-d dryrunval --dry-run=dryrunval for kubectl apply (none|server|client)\n" + echo -e "-i version ingress-nginx controller version" + echo -e " https://github.com/kubernetes/ingress-nginx/releases\n" + echo -e "-a use AWS provider for ingress-nginx" + echo -e " OR" + echo -e "-p provider specify provider for ingress-nginx" + echo -e " https://github.com/kubernetes/ingress-nginx/tree/main/deploy/static/provider\n" + echo -e "-e expose ingress-nginx via AWS load balancer (only applies to -a/-p aws)" + echo -e " https://repost.aws/knowledge-center/eks-access-kubernetes-services\n" + echo -e "-s start ingress-nginx with --enable-ssl-passthrough" + echo -e " https://kubernetes.github.io/ingress-nginx/user-guide/tls/#ssl-passthrough\n" + echo -e "-t start ingress-nginx with --tcp-services-configmap=ingress-nginx/tcp-services" + echo -e " https://kubernetes.github.io/ingress-nginx/user-guide/exposing-tcp-udp-services\n" exit 1 } ############################################################################### # parse command-line parameters -while getopts 'vhaed:k:i:' OPTION; do - case "$OPTION" in +while getopts 'vhaestp:d:k:i:' OPTION; do + case "$OPTION" in - v) - VERBOSE_FLAG="-v" - set -x - ;; + v) + VERBOSE_FLAG="-v" + # set -x + ;; - d) - DRY_RUN="${OPTARG}" - ;; + d) + DRY_RUN="${OPTARG}" + ;; - a) - INGRESS_NGINX_PROVIDER="aws" - ;; + p) + INGRESS_NGINX_PROVIDER="${OPTARG}" + ;; - e) - INGRESS_NGINX_PROVIDER="aws" - EXPOSE_VIA_AWS_LB="true" - ;; + a) + INGRESS_NGINX_PROVIDER="aws" + ;; - k) - KUBECONFIG="${OPTARG}" - ;; + e) + EXPOSE_VIA_AWS_LB="true" + ;; - i) - INGRESS_NGINX_CONTROLLER_VERSION="${OPTARG}" - ;; + s) + SSL_PASSTHROUGH="true" + ;; - ?) - help >&2 - exit 1; - ;; + t) + OTHER_TCP_SERVICES="true" + ;; - esac + k) + KUBECONFIG="${OPTARG}" + ;; + + i) + INGRESS_NGINX_CONTROLLER_VERSION="${OPTARG}" + ;; + + ?) + help >&2 + exit 1; + ;; + + esac done shift "$(($OPTIND -1))" ############################################################################### function cleanup { - [[ -n "${DEPLOY_YAML_FILE}" ]] && [[ -f "${DEPLOY_YAML_FILE}" ]] && rm ${VERBOSE_FLAG} -f "${DEPLOY_YAML_FILE}" + set +e + if [[ -n "${WORKDIR}" ]] && [[ -d "${WORKDIR}" ]]; then + popd >/dev/null >/dev/null 2>&1 + rm ${VERBOSE_FLAG} -r -f "${WORKDIR}" >/dev/null 2>&1 + fi } if ! command -v curl >/dev/null 2>&1 || ! command -v yq >/dev/null 2>&1 || ! command -v kubectl >/dev/null 2>&1; then @@ -103,22 +129,54 @@ fi trap "cleanup" EXIT -DEPLOY_YAML_FILE="$(mktemp --suffix=.yaml)" - -curl -fsSL -o "${DEPLOY_YAML_FILE}" "https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v${INGRESS_NGINX_CONTROLLER_VERSION}/deploy/static/provider/${INGRESS_NGINX_PROVIDER}/deploy.yaml" -yq -i '( select(.kind == "Deployment").spec.template.spec.containers[].args[] | select(contains("/nginx-ingress-controller")) | parent ) += ["--enable-ssl-passthrough", "--tcp-services-configmap=ingress-nginx/tcp-services"]' "${DEPLOY_YAML_FILE}" -yq -i "( select(.kind == \"Deployment\").spec.template.spec.containers[].args[] | select(contains(\"/nginx-ingress-controller\")) | parent | parent | .ports ) += [${DEPLOYMENT_ADDITIONAL_PORTS}]" "${DEPLOY_YAML_FILE}" -yq -i "( select(.kind == \"Service\" and .spec.type == \"LoadBalancer\").spec.ports ) += [${LOAD_BALANCER_ADDITIONAL_PORTS}]" "${DEPLOY_YAML_FILE}" -if [[ "${EXPOSE_VIA_AWS_LB}" == "true" ]]; then - # see https://repost.aws/knowledge-center/eks-access-kubernetes-services (Option 1), step 2. - for OLDKEY in $(yq "select(.kind == \"Service\" and .spec.type == \"LoadBalancer\").metadata.annotations | keys | .[] | select(. == \"service.beta.kubernetes.io*\")" "${DEPLOY_YAML_FILE}"); do - yq -i "select(.kind == \"Service\" and .spec.type == \"LoadBalancer\") | del(.metadata.annotations.\"$OLDKEY\")" "${DEPLOY_YAML_FILE}" - done - for NEWKEY in ${AWS_EXPOSE_ANNOTATIONS[@]}; do - yq -i "( select(.kind == \"Service\" and .spec.type == \"LoadBalancer\").metadata.annotations ) += ${NEWKEY}" "${DEPLOY_YAML_FILE}" - done -fi +WORKDIR="$(mktemp -d -t malcolm-XXXXXX)" +pushd "${WORKDIR}" >/dev/null 2>&1 + +INGRESS_NGINX_DEPLOY_FILE_ORIG=ingress-nginx-orig.yaml +INGRESS_NGINX_DEPLOY_FILE_NEW=ingress-nginx-new.yaml + +curl -fsSL -o "${INGRESS_NGINX_DEPLOY_FILE_ORIG}" "https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v${INGRESS_NGINX_CONTROLLER_VERSION}/deploy/static/provider/${INGRESS_NGINX_PROVIDER}/deploy.yaml" +yq --split-exp '"deploy_" + $index' --no-doc "${INGRESS_NGINX_DEPLOY_FILE_ORIG}" + +readarray -d '' DEPLOY_FILES_SPLIT < <(printf '%s\0' deploy_*.yml | sort -zV) +for DEPLOY_FILE in "${DEPLOY_FILES_SPLIT[@]}"; do + + if (( $(yq 'select(.kind == "Deployment")' "${DEPLOY_FILE}" | wc -l) > 0 )); then + + if [[ "${SSL_PASSTHROUGH}" == "true" ]]; then + yq -i '( select(.kind == "Deployment").spec.template.spec.containers[].args[] | select(contains("/nginx-ingress-controller")) | parent ) += ["--enable-ssl-passthrough"]' "${DEPLOY_FILE}" + fi + + if [[ "${OTHER_TCP_SERVICES}" == "true" ]]; then + yq -i '( select(.kind == "Deployment").spec.template.spec.containers[].args[] | select(contains("/nginx-ingress-controller")) | parent ) += ["--tcp-services-configmap=ingress-nginx/tcp-services"]' "${DEPLOY_FILE}" + yq -i "( select(.kind == \"Deployment\").spec.template.spec.containers[].args[] | select(contains(\"/nginx-ingress-controller\")) | parent | parent | .ports ) += [${DEPLOYMENT_ADDITIONAL_PORTS}]" "${DEPLOY_FILE}" + fi + fi + + if (( $(yq 'select(.kind == "Service" and .spec.type == "LoadBalancer")' "${DEPLOY_FILE}" | wc -l) > 0 )); then + + if [[ "${OTHER_TCP_SERVICES}" == "true" ]]; then + yq -i "( select(.kind == \"Service\" and .spec.type == \"LoadBalancer\").spec.ports ) += [${LOAD_BALANCER_ADDITIONAL_PORTS}]" "${DEPLOY_FILE}" + fi + + if [[ "${EXPOSE_VIA_AWS_LB}" == "true" ]]; then + # see https://repost.aws/knowledge-center/eks-access-kubernetes-services (Option 1), step 2. + for OLDKEY in $(yq "select(.kind == \"Service\" and .spec.type == \"LoadBalancer\").metadata.annotations | keys | .[] | select(. == \"service.beta.kubernetes.io*\")" "${DEPLOY_FILE}"); do + yq -i "( select(.kind == \"Service\" and .spec.type == \"LoadBalancer\") ) | del(.metadata.annotations.\"$OLDKEY\")" "${DEPLOY_FILE}" + done + for NEWKEY in ${AWS_EXPOSE_ANNOTATIONS[@]}; do + yq -i "( select(.kind == \"Service\" and .spec.type == \"LoadBalancer\").metadata.annotations ) += ${NEWKEY}" "${DEPLOY_FILE}" + done + fi + fi + + [[ -f "${INGRESS_NGINX_DEPLOY_FILE_NEW}" ]] && echo "---" >> "${INGRESS_NGINX_DEPLOY_FILE_NEW}" + cat "${DEPLOY_FILE}" >> "${INGRESS_NGINX_DEPLOY_FILE_NEW}" + +done + +[[ -n "${VERBOSE_FLAG}" ]] && cat "${INGRESS_NGINX_DEPLOY_FILE_NEW}" -[[ -n "${VERBOSE_FLAG}" ]] && cat "${DEPLOY_YAML_FILE}" +kubectl --kubeconfig "${KUBECONFIG}" apply --dry-run="${DRY_RUN}" -f "${INGRESS_NGINX_DEPLOY_FILE_NEW}" -kubectl --kubeconfig "${KUBECONFIG}" apply --dry-run="${DRY_RUN}" -f "${DEPLOY_YAML_FILE}" +exit 0 \ No newline at end of file From fcffa8299c3a9f2f5a8582db9fe992d6b511f707 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Fri, 12 May 2023 16:14:02 -0600 Subject: [PATCH 35/47] remove script i'm not going to do --- kubernetes/eks/aws_eks_malcolm_prep.sh | 85 -------------------------- 1 file changed, 85 deletions(-) delete mode 100755 kubernetes/eks/aws_eks_malcolm_prep.sh diff --git a/kubernetes/eks/aws_eks_malcolm_prep.sh b/kubernetes/eks/aws_eks_malcolm_prep.sh deleted file mode 100755 index 0e863df41..000000000 --- a/kubernetes/eks/aws_eks_malcolm_prep.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env bash - -if [ -z "$BASH_VERSION" ]; then - echo "Wrong interpreter, please run \"$0\" with bash" >&2 - exit 1 -fi - -############################################################################### -# script options -set -o pipefail -set -e -shopt -s nocasematch -ENCODING="utf-8" - -############################################################################### -# script variables -VERBOSE_FLAG= -DRY_RUN=none -REGION=us-east-1 -CLUSTER_NAME= - -############################################################################### -# show script usage -function help() { - echo -e "$(basename $0)\n" - echo "-v enable bash verbosity" - echo "-c cluster-name cluster name" - echo "-d dryrunval --dry-run=dryrunval for kubectl (dryrunval=none|server|client)" - echo "-r region region code (default: us-east-1)" - exit 1 -} - -############################################################################### -# parse command-line parameters -while getopts 'vc:d:' OPTION; do - case "$OPTION" in - - v) - VERBOSE_FLAG="-v" - set -x - ;; - - c) - CLUSTER_NAME="${OPTARG}" - ;; - - d) - DRY_RUN="${OPTARG}" - ;; - - ?) - help >&2 - exit 1; - ;; - - esac -done -shift "$(($OPTIND -1))" - -############################################################################### -function cleanup { - true -} - -if ! command -v aws >/dev/null 2>&1 || ! command -v eksctl >/dev/null 2>&1 || ! command -v kubectl >/dev/null 2>&1 || ! command -v jq >/dev/null 2>&1; then - echo "$(basename $0) requires aws, eksctl kubectl and jq" >&2 - exit 1 -fi - -############################################################################### - -if [[ -z "${CLUSTER_NAME}" ]]; then - echo "Please specify -c cluster-name" >&2 - help >&2 - exit 1 -fi - -ACCOUNT="$(aws --output=json sts get-caller-identity | jq -r '.Account')" -if [[ -z "${ACCOUNT}" ]]; then - echo "Could not get account ID from aws sts get-caller-identity" >&2 - exit 1 -fi -[[ -n "${VERBOSE_FLAG}" ]] && echo "Account ID: ${ACCOUNT}" - -trap "cleanup" EXIT From 30ce2aecde0348f3987b0ad08a862e8fb3eb1129 Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 15 May 2023 08:10:51 -0600 Subject: [PATCH 36/47] allow install.py to work with older versions of dotenv (like ubuntu's python3-dotenv package) by falling back to calling set_key without encoding should the first attempt fail --- scripts/install.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/scripts/install.py b/scripts/install.py index 8506a26bd..2076e03e5 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -1375,15 +1375,28 @@ def tweak_malcolm_runtime( pass try: - dotenv_imported.set_key( - val.envFile, - val.key, - val.value, - quote_mode='never', - encoding='utf-8', - ) + oldDotEnvVersion = False + try: + dotenv_imported.set_key( + val.envFile, + val.key, + str(val.value), + quote_mode='never', + encoding='utf-8', + ) + except TypeError: + oldDotEnvVersion = True + + if oldDotEnvVersion: + dotenv_imported.set_key( + val.envFile, + val.key, + str(val.value), + quote_mode='never', + ) + except Exception as e: - eprint(f"Setting value for {val.key} in {val.envFile} module failed: {e}") + eprint(f"Setting value for {val.key} in {val.envFile} module failed ({type(e).__name__}): {e}") if self.orchMode is OrchestrationFramework.DOCKER_COMPOSE: # modify docker-compose specific values (port mappings, volume bind mounts, etc.) in-place in docker-compose files From b935078f1c5458db2b7c5cb261d4c83060051224 Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 15 May 2023 08:24:09 -0600 Subject: [PATCH 37/47] don't write bytcode --- scripts/control.py | 5 ++++- scripts/install.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/control.py b/scripts/control.py index e346a2cf5..2eea83cb1 100755 --- a/scripts/control.py +++ b/scripts/control.py @@ -3,6 +3,10 @@ # Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +import sys + +sys.dont_write_bytecode = True + import argparse import errno import fileinput @@ -18,7 +22,6 @@ import signal import stat import string -import sys import tarfile import time diff --git a/scripts/install.py b/scripts/install.py index 2076e03e5..d5c3575f3 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -3,6 +3,10 @@ # Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +import sys + +sys.dont_write_bytecode = True + import argparse import datetime import errno @@ -17,7 +21,6 @@ import math import re import shutil -import sys import tarfile import tempfile import time From 9adff1bfde0706aa925fa768f966715c3139087e Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 15 May 2023 08:34:50 -0600 Subject: [PATCH 38/47] do a better job at getting default UID and GID --- scripts/install.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/scripts/install.py b/scripts/install.py index d5c3575f3..57383b789 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -408,8 +408,14 @@ def tweak_malcolm_runtime( raise Exception("Could not determine configuration directory containing Malcolm's .env files") # figure out what UID/GID to run non-rood processes under docker as - puid = '1000' - pgid = '1000' + defaultUid = '1000' + defaultGid = '1000' + if ((self.platform == PLATFORM_LINUX) or (self.platform == PLATFORM_MAC)) and (self.scriptUser == "root"): + defaultUid = os.stat(malcolm_install_path).st_uid + defaultGid = os.stat(malcolm_install_path).st_gid + + puid = defaultUid + pgid = defaultGid try: if self.platform == PLATFORM_LINUX: puid = str(os.getuid()) @@ -417,8 +423,8 @@ def tweak_malcolm_runtime( if (puid == '0') or (pgid == '0'): raise Exception('it is preferrable not to run Malcolm as root, prompting for UID/GID instead') except Exception: - puid = '1000' - pgid = '1000' + puid = defaultUid + pgid = defaultGid while ( (not puid.isdigit()) From 3401addf4c02163027d89e5d88b0a905064d40f5 Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 15 May 2023 08:35:50 -0600 Subject: [PATCH 39/47] do a better job at getting default UID and GID --- scripts/install.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/install.py b/scripts/install.py index 57383b789..33c611499 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -411,8 +411,8 @@ def tweak_malcolm_runtime( defaultUid = '1000' defaultGid = '1000' if ((self.platform == PLATFORM_LINUX) or (self.platform == PLATFORM_MAC)) and (self.scriptUser == "root"): - defaultUid = os.stat(malcolm_install_path).st_uid - defaultGid = os.stat(malcolm_install_path).st_gid + defaultUid = str(os.stat(malcolm_install_path).st_uid) + defaultGid = str(os.stat(malcolm_install_path).st_gid) puid = defaultUid pgid = defaultGid From c3bb897f812c2b43d3352dcf7f09b05fda62ee4e Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 15 May 2023 08:51:18 -0600 Subject: [PATCH 40/47] Fix file permissions issue on .env files (see cisagov/Malcolm#253) --- scripts/install.py | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/scripts/install.py b/scripts/install.py index 33c611499..768a50753 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -984,28 +984,6 @@ def tweak_malcolm_runtime( if not os.path.isfile(envFile): shutil.copyfile(envExampleFile, envFile) - # change ownership of .envs file to match puid/pgid - if ( - ((self.platform == PLATFORM_LINUX) or (self.platform == PLATFORM_MAC)) - and (self.scriptUser == "root") - and (getpwuid(os.stat(args.configDir).st_uid).pw_name == self.scriptUser) - ): - if args.debug: - eprint(f"Setting permissions of {args.configDir} to {puid}:{pgid}") - os.chown(args.configDir, int(puid), int(pgid)) - envFiles = [] - for exts in ('*.env', '*.env.example'): - envFiles.extend(glob.glob(os.path.join(args.configDir, exts))) - for envFile in envFiles: - if ( - ((self.platform == PLATFORM_LINUX) or (self.platform == PLATFORM_MAC)) - and (self.scriptUser == "root") - and (getpwuid(os.stat(envFile).st_uid).pw_name == self.scriptUser) - ): - if args.debug: - eprint(f"Setting permissions of {envFile} to {puid}:{pgid}") - os.chown(envFile, int(puid), int(pgid)) - # define environment variables to be set in .env files EnvValue = namedtuple("EnvValue", ["envFile", "key", "value"], rename=False) @@ -1407,6 +1385,28 @@ def tweak_malcolm_runtime( except Exception as e: eprint(f"Setting value for {val.key} in {val.envFile} module failed ({type(e).__name__}): {e}") + # change ownership of .envs file to match puid/pgid + if ( + ((self.platform == PLATFORM_LINUX) or (self.platform == PLATFORM_MAC)) + and (self.scriptUser == "root") + and (getpwuid(os.stat(args.configDir).st_uid).pw_name == self.scriptUser) + ): + if args.debug: + eprint(f"Setting permissions of {args.configDir} to {puid}:{pgid}") + os.chown(args.configDir, int(puid), int(pgid)) + envFiles = [] + for exts in ('*.env', '*.env.example'): + envFiles.extend(glob.glob(os.path.join(args.configDir, exts))) + for envFile in envFiles: + if ( + ((self.platform == PLATFORM_LINUX) or (self.platform == PLATFORM_MAC)) + and (self.scriptUser == "root") + and (getpwuid(os.stat(envFile).st_uid).pw_name == self.scriptUser) + ): + if args.debug: + eprint(f"Setting permissions of {envFile} to {puid}:{pgid}") + os.chown(envFile, int(puid), int(pgid)) + if self.orchMode is OrchestrationFramework.DOCKER_COMPOSE: # modify docker-compose specific values (port mappings, volume bind mounts, etc.) in-place in docker-compose files for composeFile in configFiles: From 4cb99b94a4f464008f9c13d3f3417d0d2ca512f6 Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 15 May 2023 08:54:24 -0600 Subject: [PATCH 41/47] chown ownership of .configured file --- scripts/install.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/install.py b/scripts/install.py index 768a50753..aa2fbea38 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -1766,6 +1766,8 @@ def tweak_malcolm_runtime( try: touch(MalcolmCfgRunOnceFile) + if ((self.platform == PLATFORM_LINUX) or (self.platform == PLATFORM_MAC)) and (self.scriptUser == "root"): + os.chown(MalcolmCfgRunOnceFile, int(puid), int(pgid)) except Exception: pass From 462d3cc1eda62fab9f048df1c7e7b716092eea25 Mon Sep 17 00:00:00 2001 From: SG Date: Mon, 15 May 2023 09:01:27 -0600 Subject: [PATCH 42/47] tweak kubernetes manifests for v23.05.1 release prep --- kubernetes/02-opensearch.yml | 2 +- kubernetes/03-dashboards.yml | 2 +- kubernetes/04-upload.yml | 2 +- kubernetes/05-pcap-monitor.yml | 2 +- kubernetes/06-arkime.yml | 2 +- kubernetes/07-api.yml | 2 +- kubernetes/08-dashboards-helper.yml | 2 +- kubernetes/09-zeek.yml | 2 +- kubernetes/10-suricata.yml | 2 +- kubernetes/11-file-monitor.yml | 2 +- kubernetes/12-filebeat.yml | 2 +- kubernetes/13-logstash.yml | 2 +- kubernetes/15-netbox-redis.yml | 2 +- kubernetes/16-netbox-redis-cache.yml | 2 +- kubernetes/17-netbox-postgres.yml | 2 +- kubernetes/18-netbox.yml | 2 +- kubernetes/19-htadmin.yml | 2 +- kubernetes/20-pcap-capture.yml | 2 +- kubernetes/21-zeek-live.yml | 2 +- kubernetes/22-suricata-live.yml | 2 +- kubernetes/23-freq.yml | 2 +- kubernetes/99-nginx-proxy.yml | 2 +- 22 files changed, 22 insertions(+), 22 deletions(-) diff --git a/kubernetes/02-opensearch.yml b/kubernetes/02-opensearch.yml index e2f70e54c..bac0e641f 100644 --- a/kubernetes/02-opensearch.yml +++ b/kubernetes/02-opensearch.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: opensearch-container - image: ghcr.io/mmguero-dev/malcolm/opensearch:development + image: ghcr.io/idaholab/malcolm/opensearch:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/03-dashboards.yml b/kubernetes/03-dashboards.yml index cfbb8b422..2ab9d7ad6 100644 --- a/kubernetes/03-dashboards.yml +++ b/kubernetes/03-dashboards.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: dashboards-container - image: ghcr.io/mmguero-dev/malcolm/dashboards:development + image: ghcr.io/idaholab/malcolm/dashboards:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/04-upload.yml b/kubernetes/04-upload.yml index 19915597c..7a0233e38 100644 --- a/kubernetes/04-upload.yml +++ b/kubernetes/04-upload.yml @@ -34,7 +34,7 @@ spec: spec: containers: - name: upload-container - image: ghcr.io/mmguero-dev/malcolm/file-upload:development + image: ghcr.io/idaholab/malcolm/file-upload:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/05-pcap-monitor.yml b/kubernetes/05-pcap-monitor.yml index b8b76084b..8cb55fb9a 100644 --- a/kubernetes/05-pcap-monitor.yml +++ b/kubernetes/05-pcap-monitor.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: pcap-monitor-container - image: ghcr.io/mmguero-dev/malcolm/pcap-monitor:development + image: ghcr.io/idaholab/malcolm/pcap-monitor:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/06-arkime.yml b/kubernetes/06-arkime.yml index ef24eb44b..f74812290 100644 --- a/kubernetes/06-arkime.yml +++ b/kubernetes/06-arkime.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: arkime-container - image: ghcr.io/mmguero-dev/malcolm/arkime:development + image: ghcr.io/idaholab/malcolm/arkime:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/07-api.yml b/kubernetes/07-api.yml index 3f611da69..33fa6d1e0 100644 --- a/kubernetes/07-api.yml +++ b/kubernetes/07-api.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: api-container - image: ghcr.io/mmguero-dev/malcolm/api:development + image: ghcr.io/idaholab/malcolm/api:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/08-dashboards-helper.yml b/kubernetes/08-dashboards-helper.yml index 529647210..4ae09aaed 100644 --- a/kubernetes/08-dashboards-helper.yml +++ b/kubernetes/08-dashboards-helper.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: dashboards-helper-container - image: ghcr.io/mmguero-dev/malcolm/dashboards-helper:development + image: ghcr.io/idaholab/malcolm/dashboards-helper:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/09-zeek.yml b/kubernetes/09-zeek.yml index f7fd93b7f..ab83065ca 100644 --- a/kubernetes/09-zeek.yml +++ b/kubernetes/09-zeek.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: zeek-offline-container - image: ghcr.io/mmguero-dev/malcolm/zeek:development + image: ghcr.io/idaholab/malcolm/zeek:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/10-suricata.yml b/kubernetes/10-suricata.yml index da525b557..d89e987a7 100644 --- a/kubernetes/10-suricata.yml +++ b/kubernetes/10-suricata.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: suricata-offline-container - image: ghcr.io/mmguero-dev/malcolm/suricata:development + image: ghcr.io/idaholab/malcolm/suricata:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/11-file-monitor.yml b/kubernetes/11-file-monitor.yml index 39a29b65d..4d65038a5 100644 --- a/kubernetes/11-file-monitor.yml +++ b/kubernetes/11-file-monitor.yml @@ -33,7 +33,7 @@ spec: spec: containers: - name: file-monitor-container - image: ghcr.io/mmguero-dev/malcolm/file-monitor:development + image: ghcr.io/idaholab/malcolm/file-monitor:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/12-filebeat.yml b/kubernetes/12-filebeat.yml index 7676193c0..86e0e1c83 100644 --- a/kubernetes/12-filebeat.yml +++ b/kubernetes/12-filebeat.yml @@ -31,7 +31,7 @@ spec: spec: containers: - name: filebeat-container - image: ghcr.io/mmguero-dev/malcolm/filebeat-oss:development + image: ghcr.io/idaholab/malcolm/filebeat-oss:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/13-logstash.yml b/kubernetes/13-logstash.yml index 081c47ed3..2a3920b40 100644 --- a/kubernetes/13-logstash.yml +++ b/kubernetes/13-logstash.yml @@ -47,7 +47,7 @@ spec: # topologyKey: "kubernetes.io/hostname" containers: - name: logstash-container - image: ghcr.io/mmguero-dev/malcolm/logstash-oss:development + image: ghcr.io/idaholab/malcolm/logstash-oss:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/15-netbox-redis.yml b/kubernetes/15-netbox-redis.yml index 984110449..dcada13b0 100644 --- a/kubernetes/15-netbox-redis.yml +++ b/kubernetes/15-netbox-redis.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-redis-container - image: ghcr.io/mmguero-dev/malcolm/redis:development + image: ghcr.io/idaholab/malcolm/redis:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/16-netbox-redis-cache.yml b/kubernetes/16-netbox-redis-cache.yml index 22f1be5a4..5d8ff37e9 100644 --- a/kubernetes/16-netbox-redis-cache.yml +++ b/kubernetes/16-netbox-redis-cache.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-redis-cache-container - image: ghcr.io/mmguero-dev/malcolm/redis:development + image: ghcr.io/idaholab/malcolm/redis:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/17-netbox-postgres.yml b/kubernetes/17-netbox-postgres.yml index 1497c1bea..70f70002f 100644 --- a/kubernetes/17-netbox-postgres.yml +++ b/kubernetes/17-netbox-postgres.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-postgres-container - image: ghcr.io/mmguero-dev/malcolm/postgresql:development + image: ghcr.io/idaholab/malcolm/postgresql:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/18-netbox.yml b/kubernetes/18-netbox.yml index 5a613fc8e..ac53304b5 100644 --- a/kubernetes/18-netbox.yml +++ b/kubernetes/18-netbox.yml @@ -36,7 +36,7 @@ spec: spec: containers: - name: netbox-container - image: ghcr.io/mmguero-dev/malcolm/netbox:development + image: ghcr.io/idaholab/malcolm/netbox:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/19-htadmin.yml b/kubernetes/19-htadmin.yml index 716a46363..918d3a8fb 100644 --- a/kubernetes/19-htadmin.yml +++ b/kubernetes/19-htadmin.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: htadmin-container - image: ghcr.io/mmguero-dev/malcolm/htadmin:development + image: ghcr.io/idaholab/malcolm/htadmin:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/20-pcap-capture.yml b/kubernetes/20-pcap-capture.yml index 63360e5a4..816c491dc 100644 --- a/kubernetes/20-pcap-capture.yml +++ b/kubernetes/20-pcap-capture.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: pcap-capture-container - image: ghcr.io/mmguero-dev/malcolm/pcap-capture:development + image: ghcr.io/idaholab/malcolm/pcap-capture:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/21-zeek-live.yml b/kubernetes/21-zeek-live.yml index 3eb2cf92f..835654692 100644 --- a/kubernetes/21-zeek-live.yml +++ b/kubernetes/21-zeek-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: zeek-live-container - image: ghcr.io/mmguero-dev/malcolm/zeek:development + image: ghcr.io/idaholab/malcolm/zeek:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/22-suricata-live.yml b/kubernetes/22-suricata-live.yml index fe0ee1d21..2b48ed2d5 100644 --- a/kubernetes/22-suricata-live.yml +++ b/kubernetes/22-suricata-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: suricata-live-container - image: ghcr.io/mmguero-dev/malcolm/suricata:development + image: ghcr.io/idaholab/malcolm/suricata:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/23-freq.yml b/kubernetes/23-freq.yml index b9dc580df..86f316139 100644 --- a/kubernetes/23-freq.yml +++ b/kubernetes/23-freq.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: freq-container - image: ghcr.io/mmguero-dev/malcolm/freq:development + image: ghcr.io/idaholab/malcolm/freq:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/99-nginx-proxy.yml b/kubernetes/99-nginx-proxy.yml index 5c2f49b4c..ef3f0e74a 100644 --- a/kubernetes/99-nginx-proxy.yml +++ b/kubernetes/99-nginx-proxy.yml @@ -37,7 +37,7 @@ spec: spec: containers: - name: nginx-proxy-container - image: ghcr.io/mmguero-dev/malcolm/nginx-proxy:development + image: ghcr.io/idaholab/malcolm/nginx-proxy:23.05.1 imagePullPolicy: Always stdin: false tty: true From b54b04fe8a0a3bea7c4c18c2d4bc8f38d03f7448 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 15 May 2023 13:22:11 -0600 Subject: [PATCH 43/47] cd to Malcolm directory in desktop files --- .../skel/.config/xfce4/panel/launcher-25/16343171677.desktop | 2 +- .../skel/.config/xfce4/panel/launcher-26/16343171699.desktop | 2 +- .../skel/.config/xfce4/panel/launcher-27/16343171722.desktop | 2 +- .../skel/.config/xfce4/panel/launcher-28/16343171811.desktop | 2 +- .../includes.chroot/usr/share/applications/malcolm-logs.desktop | 2 +- .../usr/share/applications/malcolm-restart.desktop | 2 +- .../usr/share/applications/malcolm-start.desktop | 2 +- .../includes.chroot/usr/share/applications/malcolm-stop.desktop | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-25/16343171677.desktop b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-25/16343171677.desktop index aead73d5a..a20408949 100644 --- a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-25/16343171677.desktop +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-25/16343171677.desktop @@ -1,6 +1,6 @@ [Desktop Entry] Name=Start Malcolm -Exec=tilix -e /bin/bash -l -c "/usr/bin/python3 ~/Malcolm/scripts/control.py --start" +Exec=tilix -e /bin/bash -l -c "cd ~/Malcolm && /usr/bin/python3 ~/Malcolm/scripts/control.py --start" Comment=Start Malcolm Terminal=false Type=Application diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-26/16343171699.desktop b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-26/16343171699.desktop index 854b12df3..9074a72fb 100644 --- a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-26/16343171699.desktop +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-26/16343171699.desktop @@ -1,6 +1,6 @@ [Desktop Entry] Name=Restart Malcolm -Exec=tilix -e /bin/bash -l -c "/usr/bin/python3 ~/Malcolm/scripts/control.py --restart" +Exec=tilix -e /bin/bash -l -c "cd ~/Malcolm && /usr/bin/python3 ~/Malcolm/scripts/control.py --restart" Comment=Restart Malcolm Terminal=false Type=Application diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-27/16343171722.desktop b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-27/16343171722.desktop index 87de10b44..e3a97a508 100644 --- a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-27/16343171722.desktop +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-27/16343171722.desktop @@ -1,6 +1,6 @@ [Desktop Entry] Name=Stop Malcolm -Exec=tilix -e /bin/bash -l -c "/usr/bin/python3 ~/Malcolm/scripts/control.py --stop" +Exec=tilix -e /bin/bash -l -c "cd ~/Malcolm && /usr/bin/python3 ~/Malcolm/scripts/control.py --stop" Comment=Stop Malcolm Terminal=false Type=Application diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-28/16343171811.desktop b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-28/16343171811.desktop index b67b49d66..bffcb003d 100644 --- a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-28/16343171811.desktop +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-28/16343171811.desktop @@ -1,6 +1,6 @@ [Desktop Entry] Name=Malcolm Debug Logs -Exec=tilix -e /bin/bash -l -c "/usr/bin/python3 ~/Malcolm/scripts/control.py --logs" +Exec=tilix -e /bin/bash -l -c "cd ~/Malcolm && /usr/bin/python3 ~/Malcolm/scripts/control.py --logs" Comment=Monitor the debug output of Malcolm containers Terminal=false Type=Application diff --git a/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-logs.desktop b/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-logs.desktop index cb3660de9..5ea913299 100644 --- a/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-logs.desktop +++ b/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-logs.desktop @@ -1,6 +1,6 @@ [Desktop Entry] Name=Malcolm Debug Logs -Exec=tilix -e /bin/bash -l -c "/usr/bin/python3 ~/Malcolm/scripts/control.py --logs" +Exec=tilix -e /bin/bash -l -c "cd ~/Malcolm && /usr/bin/python3 ~/Malcolm/scripts/control.py --logs" Comment=Monitor the debug output of Malcolm containers Terminal=false Type=Application diff --git a/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-restart.desktop b/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-restart.desktop index 6c0f0a06d..1194f84c1 100644 --- a/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-restart.desktop +++ b/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-restart.desktop @@ -1,6 +1,6 @@ [Desktop Entry] Name=Restart Malcolm -Exec=tilix -e /bin/bash -l -c "/usr/bin/python3 ~/Malcolm/scripts/control.py --restart" +Exec=tilix -e /bin/bash -l -c "cd ~/Malcolm && /usr/bin/python3 ~/Malcolm/scripts/control.py --restart" Comment=Restart Malcolm Terminal=false Type=Application diff --git a/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-start.desktop b/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-start.desktop index 007e8e8c5..39301d22b 100644 --- a/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-start.desktop +++ b/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-start.desktop @@ -1,6 +1,6 @@ [Desktop Entry] Name=Start Malcolm -Exec=tilix -e /bin/bash -l -c "/usr/bin/python3 ~/Malcolm/scripts/control.py --start" +Exec=tilix -e /bin/bash -l -c "cd ~/Malcolm && /usr/bin/python3 ~/Malcolm/scripts/control.py --start" Comment=Start Malcolm Terminal=false Type=Application diff --git a/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-stop.desktop b/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-stop.desktop index ac18f0e3c..53bb34ef1 100644 --- a/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-stop.desktop +++ b/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-stop.desktop @@ -1,6 +1,6 @@ [Desktop Entry] Name=Stop Malcolm -Exec=tilix -e /bin/bash -l -c "/usr/bin/python3 ~/Malcolm/scripts/control.py --stop" +Exec=tilix -e /bin/bash -l -c "cd ~/Malcolm && /usr/bin/python3 ~/Malcolm/scripts/control.py --stop" Comment=Stop Malcolm Terminal=false Type=Application From 7693df4237e56b578ae54d360513e7e0ccbf64af Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 15 May 2023 13:52:20 -0600 Subject: [PATCH 44/47] Fixes for netbox v3.5.1 docker image --- Dockerfiles/netbox.Dockerfile | 3 ++- malcolm-iso/config/includes.chroot/etc/bash.bash_functions | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfiles/netbox.Dockerfile b/Dockerfiles/netbox.Dockerfile index a546542aa..b1beda6e7 100644 --- a/Dockerfiles/netbox.Dockerfile +++ b/Dockerfiles/netbox.Dockerfile @@ -73,7 +73,8 @@ RUN apt-get -q update && \ mv /etc/unit/nginx-unit-new.json /etc/unit/nginx-unit.json && \ chmod 644 /etc/unit/nginx-unit.json && \ tr -cd '\11\12\15\40-\176' < /opt/netbox/netbox/netbox/configuration.py > /opt/netbox/netbox/netbox/configuration_ascii.py && \ - mv /opt/netbox/netbox/netbox/configuration_ascii.py /opt/netbox/netbox/netbox/configuration.py + mv /opt/netbox/netbox/netbox/configuration_ascii.py /opt/netbox/netbox/netbox/configuration.py && \ + sed -i -E 's@^([[:space:]]*\-\-(state|tmp))([[:space:]])@\1dir\3@g' /opt/netbox/launch-netbox.sh COPY --chmod=755 shared/bin/docker-uid-gid-setup.sh /usr/local/bin/ COPY --chmod=755 shared/bin/service_check_passthrough.sh /usr/local/bin/ diff --git a/malcolm-iso/config/includes.chroot/etc/bash.bash_functions b/malcolm-iso/config/includes.chroot/etc/bash.bash_functions index 7705973e3..bfc6b2487 100644 --- a/malcolm-iso/config/includes.chroot/etc/bash.bash_functions +++ b/malcolm-iso/config/includes.chroot/etc/bash.bash_functions @@ -551,7 +551,7 @@ function malcolmmonitor () { split-window -v \; \ split-window -v \; \ select-pane -t 1 \; \ - send-keys '~/Malcolm/scripts/logs' C-m \; \ + send-keys 'pushd ~/Malcolm >/dev/null 2>&1; ~/Malcolm/scripts/logs; popd >/dev/null 2>&1' C-m \; \ select-pane -t 2 \; \ send-keys "docker stats --format 'table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}'" C-m \; \ select-pane -t 3 \; \ From 32d7bbb29e57bdd87951c4977a248802524d69cc Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 15 May 2023 23:35:13 -0600 Subject: [PATCH 45/47] update sha256 sums for ISOs --- docs/download.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/download.md b/docs/download.md index daf9fb7d0..2dbe7101a 100644 --- a/docs/download.md +++ b/docs/download.md @@ -16,7 +16,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | ISO | SHA256 | |---|---| -| [malcolm-23.05.1.iso](/iso/malcolm-23.05.1.iso) (5.3GiB) | [`e9e00694f25b9d0dcc286496490e184930611ddbed6c52dfab77a935d2afa850`](/iso/malcolm-23.05.1.iso.sha256.txt) | +| [malcolm-23.05.1.iso](/iso/malcolm-23.05.1.iso) (5.4GiB) | [`ab035fe1c3e316e792a707e214278fa6067c611bd92356808bab1600396dc0e4`](/iso/malcolm-23.05.1.iso.sha256.txt) | ## Hedgehog Linux @@ -26,7 +26,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | ISO | SHA256 | |---|---| -| [hedgehog-23.05.1.iso](/iso/hedgehog-23.05.1.iso) (2.3GiB) | [`f850ecd3b62731b46ac0366bdcdd62437da30220c23f94013873c6c92cbddff7`](/iso/hedgehog-23.05.1.iso.sha256.txt) | +| [hedgehog-23.05.1.iso](/iso/hedgehog-23.05.1.iso) (2.3GiB) | [`d674be7f5c61783c47fb4b49f48358e2eb3d48ff3b34a9a831cc65f05b11a313`](/iso/hedgehog-23.05.1.iso.sha256.txt) | ## Warning From 0c62f7722d10ac0692d79198b76b15edf7484fdd Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 16 May 2023 07:11:50 -0600 Subject: [PATCH 46/47] fix idaholab/Malcolm#196, both zeek and zeek-live containers are trying to pull intel feeds on startup --- .github/workflows/api-build-and-push-ghcr.yml | 1 + .github/workflows/arkime-build-and-push-ghcr.yml | 1 + .github/workflows/dashboards-build-and-push-ghcr.yml | 1 + .github/workflows/dashboards-helper-build-and-push-ghcr.yml | 1 + .github/workflows/file-monitor-build-and-push-ghcr.yml | 1 + .github/workflows/file-upload-build-and-push-ghcr.yml | 1 + .github/workflows/filebeat-build-and-push-ghcr.yml | 1 + .github/workflows/freq-build-and-push-ghcr.yml | 1 + .github/workflows/htadmin-build-and-push-ghcr.yml | 1 + .github/workflows/logstash-build-and-push-ghcr.yml | 1 + .github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml | 1 + .github/workflows/netbox-build-and-push-ghcr.yml | 1 + .github/workflows/nginx-build-and-push-ghcr.yml | 1 + .github/workflows/opensearch-build-and-push-ghcr.yml | 1 + .github/workflows/pcap-capture-build-and-push-ghcr.yml | 1 + .github/workflows/pcap-monitor-build-and-push-ghcr.yml | 1 + .github/workflows/postgresql-build-and-push-ghcr.yml | 1 + .github/workflows/redis-build-and-push-ghcr.yml | 1 + .github/workflows/suricata-build-and-push-ghcr.yml | 1 + shared/bin/zeek_intel_setup.sh | 3 ++- zeek/scripts/docker_entrypoint.sh | 2 +- 21 files changed, 22 insertions(+), 2 deletions(-) diff --git a/.github/workflows/api-build-and-push-ghcr.yml b/.github/workflows/api-build-and-push-ghcr.yml index 939007b3b..5cfd3ead3 100644 --- a/.github/workflows/api-build-and-push-ghcr.yml +++ b/.github/workflows/api-build-and-push-ghcr.yml @@ -14,6 +14,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_workflow_build' workflow_dispatch: repository_dispatch: diff --git a/.github/workflows/arkime-build-and-push-ghcr.yml b/.github/workflows/arkime-build-and-push-ghcr.yml index 32016cab4..2effb3957 100644 --- a/.github/workflows/arkime-build-and-push-ghcr.yml +++ b/.github/workflows/arkime-build-and-push-ghcr.yml @@ -14,6 +14,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_workflow_build' workflow_dispatch: repository_dispatch: diff --git a/.github/workflows/dashboards-build-and-push-ghcr.yml b/.github/workflows/dashboards-build-and-push-ghcr.yml index 1452ed146..1c0ab778c 100644 --- a/.github/workflows/dashboards-build-and-push-ghcr.yml +++ b/.github/workflows/dashboards-build-and-push-ghcr.yml @@ -14,6 +14,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_workflow_build' workflow_dispatch: repository_dispatch: diff --git a/.github/workflows/dashboards-helper-build-and-push-ghcr.yml b/.github/workflows/dashboards-helper-build-and-push-ghcr.yml index 7f8bf5804..fd862543f 100644 --- a/.github/workflows/dashboards-helper-build-and-push-ghcr.yml +++ b/.github/workflows/dashboards-helper-build-and-push-ghcr.yml @@ -14,6 +14,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_workflow_build' workflow_dispatch: repository_dispatch: diff --git a/.github/workflows/file-monitor-build-and-push-ghcr.yml b/.github/workflows/file-monitor-build-and-push-ghcr.yml index bafd62550..fbb6bbbfd 100644 --- a/.github/workflows/file-monitor-build-and-push-ghcr.yml +++ b/.github/workflows/file-monitor-build-and-push-ghcr.yml @@ -14,6 +14,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*.sh' - '.trigger_workflow_build' workflow_dispatch: repository_dispatch: diff --git a/.github/workflows/file-upload-build-and-push-ghcr.yml b/.github/workflows/file-upload-build-and-push-ghcr.yml index f1a5b3113..b49ae4bea 100644 --- a/.github/workflows/file-upload-build-and-push-ghcr.yml +++ b/.github/workflows/file-upload-build-and-push-ghcr.yml @@ -14,6 +14,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_workflow_build' workflow_dispatch: repository_dispatch: diff --git a/.github/workflows/filebeat-build-and-push-ghcr.yml b/.github/workflows/filebeat-build-and-push-ghcr.yml index e52a73691..10be9650e 100644 --- a/.github/workflows/filebeat-build-and-push-ghcr.yml +++ b/.github/workflows/filebeat-build-and-push-ghcr.yml @@ -14,6 +14,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_workflow_build' workflow_dispatch: repository_dispatch: diff --git a/.github/workflows/freq-build-and-push-ghcr.yml b/.github/workflows/freq-build-and-push-ghcr.yml index 84a3254e0..d27e67429 100644 --- a/.github/workflows/freq-build-and-push-ghcr.yml +++ b/.github/workflows/freq-build-and-push-ghcr.yml @@ -14,6 +14,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_workflow_build' workflow_dispatch: repository_dispatch: diff --git a/.github/workflows/htadmin-build-and-push-ghcr.yml b/.github/workflows/htadmin-build-and-push-ghcr.yml index eff564dee..06fbcabac 100644 --- a/.github/workflows/htadmin-build-and-push-ghcr.yml +++ b/.github/workflows/htadmin-build-and-push-ghcr.yml @@ -14,6 +14,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_workflow_build' workflow_dispatch: repository_dispatch: diff --git a/.github/workflows/logstash-build-and-push-ghcr.yml b/.github/workflows/logstash-build-and-push-ghcr.yml index 6e8ed88cd..997d18d2f 100644 --- a/.github/workflows/logstash-build-and-push-ghcr.yml +++ b/.github/workflows/logstash-build-and-push-ghcr.yml @@ -14,6 +14,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_workflow_build' workflow_dispatch: repository_dispatch: diff --git a/.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml b/.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml index fc9a44507..80aef51ce 100644 --- a/.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml +++ b/.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml @@ -9,6 +9,7 @@ on: - 'malcolm-iso/**' - 'shared/bin/*' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_iso_workflow_build' - '.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml' workflow_dispatch: diff --git a/.github/workflows/netbox-build-and-push-ghcr.yml b/.github/workflows/netbox-build-and-push-ghcr.yml index a8ecbc443..2693f9323 100644 --- a/.github/workflows/netbox-build-and-push-ghcr.yml +++ b/.github/workflows/netbox-build-and-push-ghcr.yml @@ -14,6 +14,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_workflow_build' workflow_dispatch: repository_dispatch: diff --git a/.github/workflows/nginx-build-and-push-ghcr.yml b/.github/workflows/nginx-build-and-push-ghcr.yml index 4489ce37a..b89a4d859 100644 --- a/.github/workflows/nginx-build-and-push-ghcr.yml +++ b/.github/workflows/nginx-build-and-push-ghcr.yml @@ -14,6 +14,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_workflow_build' - '_config.yml' - '_includes/**' diff --git a/.github/workflows/opensearch-build-and-push-ghcr.yml b/.github/workflows/opensearch-build-and-push-ghcr.yml index c6faee5d6..08f5967fd 100644 --- a/.github/workflows/opensearch-build-and-push-ghcr.yml +++ b/.github/workflows/opensearch-build-and-push-ghcr.yml @@ -13,6 +13,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_workflow_build' workflow_dispatch: repository_dispatch: diff --git a/.github/workflows/pcap-capture-build-and-push-ghcr.yml b/.github/workflows/pcap-capture-build-and-push-ghcr.yml index 057d4cfc9..f3a224290 100644 --- a/.github/workflows/pcap-capture-build-and-push-ghcr.yml +++ b/.github/workflows/pcap-capture-build-and-push-ghcr.yml @@ -14,6 +14,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_workflow_build' workflow_dispatch: repository_dispatch: diff --git a/.github/workflows/pcap-monitor-build-and-push-ghcr.yml b/.github/workflows/pcap-monitor-build-and-push-ghcr.yml index 6a69b2bad..eab99d9d1 100644 --- a/.github/workflows/pcap-monitor-build-and-push-ghcr.yml +++ b/.github/workflows/pcap-monitor-build-and-push-ghcr.yml @@ -14,6 +14,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_workflow_build' workflow_dispatch: repository_dispatch: diff --git a/.github/workflows/postgresql-build-and-push-ghcr.yml b/.github/workflows/postgresql-build-and-push-ghcr.yml index 703730e6d..e916b4360 100644 --- a/.github/workflows/postgresql-build-and-push-ghcr.yml +++ b/.github/workflows/postgresql-build-and-push-ghcr.yml @@ -13,6 +13,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_workflow_build' workflow_dispatch: repository_dispatch: diff --git a/.github/workflows/redis-build-and-push-ghcr.yml b/.github/workflows/redis-build-and-push-ghcr.yml index ed496a575..c36708dcf 100644 --- a/.github/workflows/redis-build-and-push-ghcr.yml +++ b/.github/workflows/redis-build-and-push-ghcr.yml @@ -13,6 +13,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_workflow_build' workflow_dispatch: repository_dispatch: diff --git a/.github/workflows/suricata-build-and-push-ghcr.yml b/.github/workflows/suricata-build-and-push-ghcr.yml index 764a2737f..73af7da1b 100644 --- a/.github/workflows/suricata-build-and-push-ghcr.yml +++ b/.github/workflows/suricata-build-and-push-ghcr.yml @@ -14,6 +14,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_workflow_build' workflow_dispatch: repository_dispatch: diff --git a/shared/bin/zeek_intel_setup.sh b/shared/bin/zeek_intel_setup.sh index c02955e02..186b95538 100755 --- a/shared/bin/zeek_intel_setup.sh +++ b/shared/bin/zeek_intel_setup.sh @@ -25,7 +25,8 @@ function finish { rmdir -- "$LOCK_DIR" || echo "Failed to remove lock directory '$LOCK_DIR'" >&2 } -if mkdir -p -- "$LOCK_DIR" 2>/dev/null; then +mkdir -p -- "$(dirname "$LOCK_DIR")" +if mkdir -- "$LOCK_DIR" 2>/dev/null; then trap finish EXIT # create directive to @load every subdirectory in /opt/zeek/share/zeek/site/intel diff --git a/zeek/scripts/docker_entrypoint.sh b/zeek/scripts/docker_entrypoint.sh index 6fb0b975c..2d9d4f972 100755 --- a/zeek/scripts/docker_entrypoint.sh +++ b/zeek/scripts/docker_entrypoint.sh @@ -8,7 +8,7 @@ setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' "${ZEEK_DIR}"/bin/ze setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' "${ZEEK_DIR}"/bin/capstats || true if [[ "${ZEEK_LIVE_CAPTURE:-false}" != "true" ]] && [[ -x "${ZEEK_DIR}"/bin/zeek_intel_setup.sh ]]; then - sleep 5 # give the "live" instance, if there is one, a chance to go first + sleep 15 # give the "live" instance, if there is one, a chance to go first if [[ "$(id -u)" == "0" ]] && [[ -n "$PUSER" ]]; then su -s /bin/bash -p ${PUSER} << EOF "${ZEEK_DIR}"/bin/zeek_intel_setup.sh /bin/true From 246651cfbeb60ee9cd37d9b6f73fee4c3e314677 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 16 May 2023 11:06:41 -0600 Subject: [PATCH 47/47] update sha256 sums for ISOs --- docs/download.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/download.md b/docs/download.md index 2dbe7101a..da4b31a3f 100644 --- a/docs/download.md +++ b/docs/download.md @@ -16,7 +16,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | ISO | SHA256 | |---|---| -| [malcolm-23.05.1.iso](/iso/malcolm-23.05.1.iso) (5.4GiB) | [`ab035fe1c3e316e792a707e214278fa6067c611bd92356808bab1600396dc0e4`](/iso/malcolm-23.05.1.iso.sha256.txt) | +| [malcolm-23.05.1.iso](/iso/malcolm-23.05.1.iso) (5.4GiB) | [`03e3d3cc9fbd334c04c6eef7e83debea203503fe3f5dba665ebb654c26056792`](/iso/malcolm-23.05.1.iso.sha256.txt) | ## Hedgehog Linux @@ -26,7 +26,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | ISO | SHA256 | |---|---| -| [hedgehog-23.05.1.iso](/iso/hedgehog-23.05.1.iso) (2.3GiB) | [`d674be7f5c61783c47fb4b49f48358e2eb3d48ff3b34a9a831cc65f05b11a313`](/iso/hedgehog-23.05.1.iso.sha256.txt) | +| [hedgehog-23.05.1.iso](/iso/hedgehog-23.05.1.iso) (2.3GiB) | [`ad14e0e51cf51966a3c54b117e668ff588fc6a94fb5a5147c373d6c5b3b3990d`](/iso/hedgehog-23.05.1.iso.sha256.txt) | ## Warning