From b422d579a93389f158814eaebcb4a4e82d734e48 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Thu, 1 Aug 2024 07:52:36 -0600 Subject: [PATCH 01/47] bump for v24.08.0 development --- docker-compose-dev.yml | 46 +++++----- docker-compose.yml | 46 +++++----- docs/api-version.md | 2 +- docs/contributing-github-runners.md | 48 +++++----- docs/contributing-pcap.md | 2 +- docs/hedgehog-iso-build.md | 2 +- docs/kubernetes.md | 88 +++++++++---------- docs/malcolm-iso.md | 2 +- docs/quickstart.md | 38 ++++---- docs/third-party-envs.md | 10 +-- docs/ubuntu-install-example.md | 38 ++++---- kubernetes/03-opensearch.yml | 4 +- kubernetes/04-dashboards.yml | 2 +- kubernetes/05-upload.yml | 4 +- kubernetes/06-pcap-monitor.yml | 4 +- kubernetes/07-arkime.yml | 4 +- kubernetes/08-api.yml | 2 +- kubernetes/09-dashboards-helper.yml | 2 +- kubernetes/10-zeek.yml | 4 +- kubernetes/11-suricata.yml | 4 +- kubernetes/12-file-monitor.yml | 4 +- kubernetes/13-filebeat.yml | 4 +- kubernetes/14-logstash.yml | 4 +- kubernetes/15-netbox-redis.yml | 4 +- kubernetes/16-netbox-redis-cache.yml | 2 +- kubernetes/17-netbox-postgres.yml | 4 +- kubernetes/18-netbox.yml | 4 +- kubernetes/19-htadmin.yml | 4 +- kubernetes/20-pcap-capture.yml | 4 +- kubernetes/21-zeek-live.yml | 4 +- kubernetes/22-suricata-live.yml | 4 +- kubernetes/23-arkime-live.yml | 4 +- kubernetes/24-freq.yml | 2 +- kubernetes/98-nginx-proxy.yml | 4 +- .../aws/ami/packer_vars.json.example | 2 +- .../aws/ami/scripts/Malcolm_AMI_Setup.sh | 2 +- 36 files changed, 204 insertions(+), 204 deletions(-) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 26f963962..cb0706bd9 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -5,7 +5,7 @@ services: build: context: . dockerfile: Dockerfiles/opensearch.Dockerfile - image: ghcr.io/idaholab/malcolm/opensearch:24.07.0 + image: ghcr.io/idaholab/malcolm/opensearch:24.08.0 # Technically the "hedgehog" profile doesn't have OpenSearch, but in that case # OPENSEARCH_PRIMARY will be set to remote, which means the container will # start but not actually run OpenSearch. It's included in both profiles to @@ -82,7 +82,7 @@ services: build: context: . dockerfile: Dockerfiles/dashboards-helper.Dockerfile - image: ghcr.io/idaholab/malcolm/dashboards-helper:24.07.0 + image: ghcr.io/idaholab/malcolm/dashboards-helper:24.08.0 profiles: ["malcolm"] logging: driver: local @@ -135,7 +135,7 @@ services: build: context: . dockerfile: Dockerfiles/dashboards.Dockerfile - image: ghcr.io/idaholab/malcolm/dashboards:24.07.0 + image: ghcr.io/idaholab/malcolm/dashboards:24.08.0 profiles: ["malcolm"] logging: driver: local @@ -181,7 +181,7 @@ services: build: context: . dockerfile: Dockerfiles/logstash.Dockerfile - image: ghcr.io/idaholab/malcolm/logstash-oss:24.07.0 + image: ghcr.io/idaholab/malcolm/logstash-oss:24.08.0 profiles: ["malcolm"] logging: driver: local @@ -295,7 +295,7 @@ services: build: context: . dockerfile: Dockerfiles/filebeat.Dockerfile - image: ghcr.io/idaholab/malcolm/filebeat-oss:24.07.0 + image: ghcr.io/idaholab/malcolm/filebeat-oss:24.08.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -371,7 +371,7 @@ services: build: context: . dockerfile: Dockerfiles/arkime.Dockerfile - image: ghcr.io/idaholab/malcolm/arkime:24.07.0 + image: ghcr.io/idaholab/malcolm/arkime:24.08.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -450,7 +450,7 @@ services: build: context: . dockerfile: Dockerfiles/arkime.Dockerfile - image: ghcr.io/idaholab/malcolm/arkime:24.07.0 + image: ghcr.io/idaholab/malcolm/arkime:24.08.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -528,7 +528,7 @@ services: build: context: . dockerfile: Dockerfiles/zeek.Dockerfile - image: ghcr.io/idaholab/malcolm/zeek:24.07.0 + image: ghcr.io/idaholab/malcolm/zeek:24.08.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -600,7 +600,7 @@ services: build: context: . dockerfile: Dockerfiles/zeek.Dockerfile - image: ghcr.io/idaholab/malcolm/zeek:24.07.0 + image: ghcr.io/idaholab/malcolm/zeek:24.08.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -664,7 +664,7 @@ services: build: context: . dockerfile: Dockerfiles/suricata.Dockerfile - image: ghcr.io/idaholab/malcolm/suricata:24.07.0 + image: ghcr.io/idaholab/malcolm/suricata:24.08.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -723,7 +723,7 @@ services: build: context: . dockerfile: Dockerfiles/suricata.Dockerfile - image: ghcr.io/idaholab/malcolm/suricata:24.07.0 + image: ghcr.io/idaholab/malcolm/suricata:24.08.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -780,7 +780,7 @@ services: build: context: . dockerfile: Dockerfiles/file-monitor.Dockerfile - image: ghcr.io/idaholab/malcolm/file-monitor:24.07.0 + image: ghcr.io/idaholab/malcolm/file-monitor:24.08.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -835,7 +835,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-capture.Dockerfile - image: ghcr.io/idaholab/malcolm/pcap-capture:24.07.0 + image: ghcr.io/idaholab/malcolm/pcap-capture:24.08.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -881,7 +881,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-monitor.Dockerfile - image: ghcr.io/idaholab/malcolm/pcap-monitor:24.07.0 + image: ghcr.io/idaholab/malcolm/pcap-monitor:24.08.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -935,7 +935,7 @@ services: build: context: . dockerfile: Dockerfiles/file-upload.Dockerfile - image: ghcr.io/idaholab/malcolm/file-upload:24.07.0 + image: ghcr.io/idaholab/malcolm/file-upload:24.08.0 profiles: ["malcolm"] logging: driver: local @@ -976,7 +976,7 @@ services: retries: 3 start_period: 60s htadmin: - image: ghcr.io/idaholab/malcolm/htadmin:24.07.0 + image: ghcr.io/idaholab/malcolm/htadmin:24.08.0 profiles: ["malcolm"] logging: driver: local @@ -1028,7 +1028,7 @@ services: retries: 3 start_period: 60s freq: - image: ghcr.io/idaholab/malcolm/freq:24.07.0 + image: ghcr.io/idaholab/malcolm/freq:24.08.0 profiles: ["malcolm"] logging: driver: local @@ -1065,7 +1065,7 @@ services: retries: 3 start_period: 60s netbox: - image: ghcr.io/idaholab/malcolm/netbox:24.07.0 + image: ghcr.io/idaholab/malcolm/netbox:24.08.0 profiles: ["malcolm"] logging: driver: local @@ -1125,7 +1125,7 @@ services: retries: 3 start_period: 120s netbox-postgres: - image: ghcr.io/idaholab/malcolm/postgresql:24.07.0 + image: ghcr.io/idaholab/malcolm/postgresql:24.08.0 profiles: ["malcolm"] logging: driver: local @@ -1168,7 +1168,7 @@ services: retries: 3 start_period: 45s netbox-redis: - image: ghcr.io/idaholab/malcolm/redis:24.07.0 + image: ghcr.io/idaholab/malcolm/redis:24.08.0 profiles: ["malcolm"] logging: driver: local @@ -1215,7 +1215,7 @@ services: retries: 3 start_period: 45s netbox-redis-cache: - image: ghcr.io/idaholab/malcolm/redis:24.07.0 + image: ghcr.io/idaholab/malcolm/redis:24.08.0 profiles: ["malcolm"] logging: driver: local @@ -1257,7 +1257,7 @@ services: retries: 3 start_period: 45s api: - image: ghcr.io/idaholab/malcolm/api:24.07.0 + image: ghcr.io/idaholab/malcolm/api:24.08.0 profiles: ["malcolm"] logging: driver: local @@ -1305,7 +1305,7 @@ services: build: context: . dockerfile: Dockerfiles/nginx.Dockerfile - image: ghcr.io/idaholab/malcolm/nginx-proxy:24.07.0 + image: ghcr.io/idaholab/malcolm/nginx-proxy:24.08.0 profiles: ["malcolm"] logging: driver: local diff --git a/docker-compose.yml b/docker-compose.yml index 6226c9144..68d4d5ad5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ services: opensearch: - image: ghcr.io/idaholab/malcolm/opensearch:24.07.0 + image: ghcr.io/idaholab/malcolm/opensearch:24.08.0 # Technically the "hedgehog" profile doesn't have OpenSearch, but in that case # OPENSEARCH_PRIMARY will be set to remote, which means the container will # start but not actually run OpenSearch. It's included in both profiles to @@ -76,7 +76,7 @@ services: retries: 3 start_period: 180s dashboards-helper: - image: ghcr.io/idaholab/malcolm/dashboards-helper:24.07.0 + image: ghcr.io/idaholab/malcolm/dashboards-helper:24.08.0 profiles: ["malcolm"] logging: driver: local @@ -126,7 +126,7 @@ services: retries: 3 start_period: 30s dashboards: - image: ghcr.io/idaholab/malcolm/dashboards:24.07.0 + image: ghcr.io/idaholab/malcolm/dashboards:24.08.0 profiles: ["malcolm"] logging: driver: local @@ -169,7 +169,7 @@ services: retries: 3 start_period: 210s logstash: - image: ghcr.io/idaholab/malcolm/logstash-oss:24.07.0 + image: ghcr.io/idaholab/malcolm/logstash-oss:24.08.0 profiles: ["malcolm"] logging: driver: local @@ -256,7 +256,7 @@ services: retries: 3 start_period: 600s filebeat: - image: ghcr.io/idaholab/malcolm/filebeat-oss:24.07.0 + image: ghcr.io/idaholab/malcolm/filebeat-oss:24.08.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -329,7 +329,7 @@ services: retries: 3 start_period: 60s arkime: - image: ghcr.io/idaholab/malcolm/arkime:24.07.0 + image: ghcr.io/idaholab/malcolm/arkime:24.08.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -387,7 +387,7 @@ services: retries: 3 start_period: 210s arkime-live: - image: ghcr.io/idaholab/malcolm/arkime:24.07.0 + image: ghcr.io/idaholab/malcolm/arkime:24.08.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -444,7 +444,7 @@ services: source: ./pcap target: /data/pcap zeek: - image: ghcr.io/idaholab/malcolm/zeek:24.07.0 + image: ghcr.io/idaholab/malcolm/zeek:24.08.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -507,7 +507,7 @@ services: retries: 3 start_period: 60s zeek-live: - image: ghcr.io/idaholab/malcolm/zeek:24.07.0 + image: ghcr.io/idaholab/malcolm/zeek:24.08.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -562,7 +562,7 @@ services: target: /opt/zeek/share/zeek/site/custom read_only: true suricata: - image: ghcr.io/idaholab/malcolm/suricata:24.07.0 + image: ghcr.io/idaholab/malcolm/suricata:24.08.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -618,7 +618,7 @@ services: retries: 3 start_period: 120s suricata-live: - image: ghcr.io/idaholab/malcolm/suricata:24.07.0 + image: ghcr.io/idaholab/malcolm/suricata:24.08.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -672,7 +672,7 @@ services: target: /opt/suricata/include-configs read_only: true file-monitor: - image: ghcr.io/idaholab/malcolm/file-monitor:24.07.0 + image: ghcr.io/idaholab/malcolm/file-monitor:24.08.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -724,7 +724,7 @@ services: retries: 3 start_period: 60s pcap-capture: - image: ghcr.io/idaholab/malcolm/pcap-capture:24.07.0 + image: ghcr.io/idaholab/malcolm/pcap-capture:24.08.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -767,7 +767,7 @@ services: source: ./pcap/upload target: /pcap pcap-monitor: - image: ghcr.io/idaholab/malcolm/pcap-monitor:24.07.0 + image: ghcr.io/idaholab/malcolm/pcap-monitor:24.08.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -818,7 +818,7 @@ services: retries: 3 start_period: 90s upload: - image: ghcr.io/idaholab/malcolm/file-upload:24.07.0 + image: ghcr.io/idaholab/malcolm/file-upload:24.08.0 profiles: ["malcolm"] logging: driver: local @@ -859,7 +859,7 @@ services: retries: 3 start_period: 60s htadmin: - image: ghcr.io/idaholab/malcolm/htadmin:24.07.0 + image: ghcr.io/idaholab/malcolm/htadmin:24.08.0 profiles: ["malcolm"] logging: driver: local @@ -908,7 +908,7 @@ services: retries: 3 start_period: 60s freq: - image: ghcr.io/idaholab/malcolm/freq:24.07.0 + image: ghcr.io/idaholab/malcolm/freq:24.08.0 profiles: ["malcolm"] logging: driver: local @@ -942,7 +942,7 @@ services: retries: 3 start_period: 60s netbox: - image: ghcr.io/idaholab/malcolm/netbox:24.07.0 + image: ghcr.io/idaholab/malcolm/netbox:24.08.0 profiles: ["malcolm"] logging: driver: local @@ -999,7 +999,7 @@ services: retries: 3 start_period: 120s netbox-postgres: - image: ghcr.io/idaholab/malcolm/postgresql:24.07.0 + image: ghcr.io/idaholab/malcolm/postgresql:24.08.0 profiles: ["malcolm"] logging: driver: local @@ -1039,7 +1039,7 @@ services: retries: 3 start_period: 45s netbox-redis: - image: ghcr.io/idaholab/malcolm/redis:24.07.0 + image: ghcr.io/idaholab/malcolm/redis:24.08.0 profiles: ["malcolm"] logging: driver: local @@ -1083,7 +1083,7 @@ services: retries: 3 start_period: 45s netbox-redis-cache: - image: ghcr.io/idaholab/malcolm/redis:24.07.0 + image: ghcr.io/idaholab/malcolm/redis:24.08.0 profiles: ["malcolm"] logging: driver: local @@ -1122,7 +1122,7 @@ services: retries: 3 start_period: 45s api: - image: ghcr.io/idaholab/malcolm/api:24.07.0 + image: ghcr.io/idaholab/malcolm/api:24.08.0 profiles: ["malcolm"] logging: driver: local @@ -1164,7 +1164,7 @@ services: retries: 3 start_period: 60s nginx-proxy: - image: ghcr.io/idaholab/malcolm/nginx-proxy:24.07.0 + image: ghcr.io/idaholab/malcolm/nginx-proxy:24.08.0 profiles: ["malcolm"] logging: driver: local diff --git a/docs/api-version.md b/docs/api-version.md index 57859f9fc..da67ccdcf 100644 --- a/docs/api-version.md +++ b/docs/api-version.md @@ -49,6 +49,6 @@ Returns version information about Malcolm and version/[health](https://opensearc } }, "sha": "dad18b1", - "version": "24.07.0" + "version": "24.08.0" } ``` diff --git a/docs/contributing-github-runners.md b/docs/contributing-github-runners.md index 87965ec95..d009ef552 100644 --- a/docs/contributing-github-runners.md +++ b/docs/contributing-github-runners.md @@ -89,7 +89,7 @@ Each container build workflow actually runs two paths in parallel: one for build ## Convenience scripts for development -As mentioned earlier, Malcolm images built using the instructions in this document are are named according to the pattern `ghcr.io/username/malcolm/image:branch`. However, note that the `image:` values found in [`docker-compose.yml`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/docker-compose.yml) (and in the [Kubernetes](kubernetes.md#Kubernetes) [manifests]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/kubernetes/)) look like `ghcr.io/idaholab/malcolm/opensearch:24.07.0`, using the OpenSearch container as an example. To run a local instance of Malcolm using these images instead of the official `ghcr.io/idaholab` ones, users will need to edit their `docker-compose.yml` file(s) and replace the `image:` tags according to this new pattern, or use the bash helper script [`./scripts/github_image_helper.sh`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/scripts/github_image_helper.sh) to pull the repository images and re-tag them with `ghcr.io/idaholab` and the current Malcolm version (e.g., `24.07.0`). +As mentioned earlier, Malcolm images built using the instructions in this document are are named according to the pattern `ghcr.io/username/malcolm/image:branch`. However, note that the `image:` values found in [`docker-compose.yml`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/docker-compose.yml) (and in the [Kubernetes](kubernetes.md#Kubernetes) [manifests]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/kubernetes/)) look like `ghcr.io/idaholab/malcolm/opensearch:24.08.0`, using the OpenSearch container as an example. To run a local instance of Malcolm using these images instead of the official `ghcr.io/idaholab` ones, users will need to edit their `docker-compose.yml` file(s) and replace the `image:` tags according to this new pattern, or use the bash helper script [`./scripts/github_image_helper.sh`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/scripts/github_image_helper.sh) to pull the repository images and re-tag them with `ghcr.io/idaholab` and the current Malcolm version (e.g., `24.08.0`). Before explaining that script, a discussion of the workflow files for the [Hedgehog Linux](live-analysis.md#Hedgehog) ([hedgehog-iso-build-docker-wrap-push-ghcr.yml ]({{ site.github.repository_url }}/tree/{{ site.github.build_revision }}/.github/workflows/hedgehog-iso-build-docker-wrap-push-ghcr.yml)) and [Malcolm](malcolm-iso.md#ISO) ([malcolm-iso-build-docker-wrap-push-ghcr.yml @@ -126,9 +126,9 @@ These menu options are described below: 3. GithubTriggerPackagesBuild * This option will trigger a [repository dispatch](https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#repository_dispatch) via the [GitHub API](https://docs.github.com/en/rest/actions/workflows?apiVersion=2022-11-28#create-a-workflow-dispatch-event) using `curl`. In order for this operation to work, an environment variable named `GITHUB_OAUTH_TOKEN` must be defined containing a [personal access token](https://docs.github.com/en/rest/authentication/authenticating-to-the-rest-api?apiVersion=2022-11-28#basic-authentication) created for your GitHub user account with the "Actions (write)" repository permission. 4. PullAndTagGithubWorkflowImages - * This option will pull latest Malcolm container images (excluding the installer ISO wrapper container images) from ghcr.io for the user's Malcolm fork, and re-tag them with `ghcr.io/idaholab` and the current Malcolm version (e.g., `24.07.0`) so that they may be run without modifying the local `docker-compose.yml` file. This is probably the option users will select most often. Note that this is different from the action performed in steps 1 and 0 above: this pulls the images directly from the container registry, it does **not** extract them from the Malcolm installer ISO wrapper container image. + * This option will pull latest Malcolm container images (excluding the installer ISO wrapper container images) from ghcr.io for the user's Malcolm fork, and re-tag them with `ghcr.io/idaholab` and the current Malcolm version (e.g., `24.08.0`) so that they may be run without modifying the local `docker-compose.yml` file. This is probably the option users will select most often. Note that this is different from the action performed in steps 1 and 0 above: this pulls the images directly from the container registry, it does **not** extract them from the Malcolm installer ISO wrapper container image. 5. PullAndTagGithubWorkflowISOImages - * This option will pull latest Malcolm installer ISO wrapper container images from ghcr.io for the user's Malcolm fork, and re-tag them with `ghcr.io/idaholab` and the current Malcolm version (e.g., `24.07.0`). + * This option will pull latest Malcolm installer ISO wrapper container images from ghcr.io for the user's Malcolm fork, and re-tag them with `ghcr.io/idaholab` and the current Malcolm version (e.g., `24.08.0`). The script can also be run non-interactively by specifying the option number on the command line (e.g., `./scripts/github_image_helper.sh 4`). @@ -167,7 +167,7 @@ $ ./scripts/github_image_helper.sh 5 PullAndTagGithubWorkflowISOImages Operation:4 PullAndTagGithubWorkflowImages -Pulling images from ghcr.io/romeogdetlevjr (main) and tagging as 24.07.0... +Pulling images from ghcr.io/romeogdetlevjr (main) and tagging as 24.08.0... main: Pulling from romeogdetlevjr/malcolm/api Digest: sha256:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx Status: Image is up to date for ghcr.io/romeogdetlevjr/malcolm/api:main @@ -181,46 +181,46 @@ ghcr.io/romeogdetlevjr/malcolm/zeek:main xxxxxxxxxxxx: Pull complete ``` -Verify that the images were pulled. Note that users will see two tags for each image: one tagged with the username and branch (e.g., `ghcr.io/romeogdetlevjr/malcolm/api:main`) and another tagged with `ghcr.io/idaholab` and the Malcolm version (e.g., `ghcr.io/idaholab/malcolm/api:24.07.0`). +Verify that the images were pulled. Note that users will see two tags for each image: one tagged with the username and branch (e.g., `ghcr.io/romeogdetlevjr/malcolm/api:main`) and another tagged with `ghcr.io/idaholab` and the Malcolm version (e.g., `ghcr.io/idaholab/malcolm/api:24.08.0`). ```bash $ docker images | grep romeogdetlevjr/malcolm -ghcr.io/idaholab/malcolm/zeek 24.07.0 xxxxxxxxxxxx 10 minutes ago 1.39GB +ghcr.io/idaholab/malcolm/zeek 24.08.0 xxxxxxxxxxxx 10 minutes ago 1.39GB ghcr.io/romeogdetlevjr/malcolm/zeek main xxxxxxxxxxxx 10 minutes ago 1.39GB -ghcr.io/idaholab/malcolm/dashboards 24.07.0 xxxxxxxxxxxx 13 minutes ago 1.55GB +ghcr.io/idaholab/malcolm/dashboards 24.08.0 xxxxxxxxxxxx 13 minutes ago 1.55GB ghcr.io/romeogdetlevjr/malcolm/dashboards main xxxxxxxxxxxx 13 minutes ago 1.55GB -ghcr.io/idaholab/malcolm/suricata 24.07.0 xxxxxxxxxxxx 14 minutes ago 339MB +ghcr.io/idaholab/malcolm/suricata 24.08.0 xxxxxxxxxxxx 14 minutes ago 339MB ghcr.io/romeogdetlevjr/malcolm/suricata main xxxxxxxxxxxx 14 minutes ago 339MB -ghcr.io/idaholab/malcolm/file-monitor 24.07.0 xxxxxxxxxxxx 15 minutes ago 712MB +ghcr.io/idaholab/malcolm/file-monitor 24.08.0 xxxxxxxxxxxx 15 minutes ago 712MB ghcr.io/romeogdetlevjr/malcolm/file-monitor main xxxxxxxxxxxx 15 minutes ago 712MB -ghcr.io/idaholab/malcolm/redis 24.07.0 xxxxxxxxxxxx 15 minutes ago 55.4MB +ghcr.io/idaholab/malcolm/redis 24.08.0 xxxxxxxxxxxx 15 minutes ago 55.4MB ghcr.io/romeogdetlevjr/malcolm/redis main xxxxxxxxxxxx 15 minutes ago 55.4MB -ghcr.io/idaholab/malcolm/nginx-proxy 24.07.0 xxxxxxxxxxxx 16 minutes ago 160MB +ghcr.io/idaholab/malcolm/nginx-proxy 24.08.0 xxxxxxxxxxxx 16 minutes ago 160MB ghcr.io/romeogdetlevjr/malcolm/nginx-proxy main xxxxxxxxxxxx 16 minutes ago 160MB -ghcr.io/idaholab/malcolm/pcap-capture 24.07.0 xxxxxxxxxxxx 16 minutes ago 137MB +ghcr.io/idaholab/malcolm/pcap-capture 24.08.0 xxxxxxxxxxxx 16 minutes ago 137MB ghcr.io/romeogdetlevjr/malcolm/pcap-capture main xxxxxxxxxxxx 16 minutes ago 137MB -ghcr.io/idaholab/malcolm/htadmin 24.07.0 xxxxxxxxxxxx 16 minutes ago 246MB +ghcr.io/idaholab/malcolm/htadmin 24.08.0 xxxxxxxxxxxx 16 minutes ago 246MB ghcr.io/romeogdetlevjr/malcolm/htadmin main xxxxxxxxxxxx 16 minutes ago 246MB ghcr.io/romeogdetlevjr/malcolm/file-upload main xxxxxxxxxxxx 16 minutes ago 250MB -ghcr.io/idaholab/malcolm/file-upload 24.07.0 xxxxxxxxxxxx 16 minutes ago 250MB -ghcr.io/idaholab/malcolm/logstash-oss 24.07.0 xxxxxxxxxxxx 16 minutes ago 1.49GB +ghcr.io/idaholab/malcolm/file-upload 24.08.0 xxxxxxxxxxxx 16 minutes ago 250MB +ghcr.io/idaholab/malcolm/logstash-oss 24.08.0 xxxxxxxxxxxx 16 minutes ago 1.49GB ghcr.io/romeogdetlevjr/malcolm/logstash-oss main xxxxxxxxxxxx 16 minutes ago 1.49GB -ghcr.io/idaholab/malcolm/netbox 24.07.0 xxxxxxxxxxxx 17 minutes ago 1.66GB +ghcr.io/idaholab/malcolm/netbox 24.08.0 xxxxxxxxxxxx 17 minutes ago 1.66GB ghcr.io/romeogdetlevjr/malcolm/netbox main xxxxxxxxxxxx 17 minutes ago 1.66GB ghcr.io/romeogdetlevjr/malcolm/filebeat-oss main xxxxxxxxxxxx 18 minutes ago 405MB -ghcr.io/idaholab/malcolm/filebeat-oss 24.07.0 xxxxxxxxxxxx 18 minutes ago 405MB +ghcr.io/idaholab/malcolm/filebeat-oss 24.08.0 xxxxxxxxxxxx 18 minutes ago 405MB ghcr.io/romeogdetlevjr/malcolm/postgresql main xxxxxxxxxxxx 18 minutes ago 303MB -ghcr.io/idaholab/malcolm/postgresql 24.07.0 xxxxxxxxxxxx 18 minutes ago 303MB -ghcr.io/idaholab/malcolm/arkime 24.07.0 xxxxxxxxxxxx 18 minutes ago 802MB +ghcr.io/idaholab/malcolm/postgresql 24.08.0 xxxxxxxxxxxx 18 minutes ago 303MB +ghcr.io/idaholab/malcolm/arkime 24.08.0 xxxxxxxxxxxx 18 minutes ago 802MB ghcr.io/romeogdetlevjr/malcolm/arkime main xxxxxxxxxxxx 18 minutes ago 802MB -ghcr.io/idaholab/malcolm/opensearch 24.07.0 xxxxxxxxxxxx 18 minutes ago 1.42GB +ghcr.io/idaholab/malcolm/opensearch 24.08.0 xxxxxxxxxxxx 18 minutes ago 1.42GB ghcr.io/romeogdetlevjr/malcolm/opensearch main xxxxxxxxxxxx 18 minutes ago 1.42GB -ghcr.io/idaholab/malcolm/pcap-monitor 24.07.0 xxxxxxxxxxxx 18 minutes ago 176MB +ghcr.io/idaholab/malcolm/pcap-monitor 24.08.0 xxxxxxxxxxxx 18 minutes ago 176MB ghcr.io/romeogdetlevjr/malcolm/pcap-monitor main xxxxxxxxxxxx 18 minutes ago 176MB -ghcr.io/idaholab/malcolm/dashboards-helper 24.07.0 xxxxxxxxxxxx 18 minutes ago 233MB +ghcr.io/idaholab/malcolm/dashboards-helper 24.08.0 xxxxxxxxxxxx 18 minutes ago 233MB ghcr.io/romeogdetlevjr/malcolm/dashboards-helpermain xxxxxxxxxxxx 18 minutes ago 233MB -ghcr.io/idaholab/malcolm/freq 24.07.0 xxxxxxxxxxxx 18 minutes ago 153MB +ghcr.io/idaholab/malcolm/freq 24.08.0 xxxxxxxxxxxx 18 minutes ago 153MB ghcr.io/romeogdetlevjr/malcolm/freq main xxxxxxxxxxxx 18 minutes ago 153MB -ghcr.io/idaholab/malcolm/api 24.07.0 xxxxxxxxxxxx 18 minutes ago 169MB +ghcr.io/idaholab/malcolm/api 24.08.0 xxxxxxxxxxxx 18 minutes ago 169MB ghcr.io/romeogdetlevjr/malcolm/api main xxxxxxxxxxxx 18 minutes ago 169MB ``` diff --git a/docs/contributing-pcap.md b/docs/contributing-pcap.md index 97d3babae..21343e93d 100644 --- a/docs/contributing-pcap.md +++ b/docs/contributing-pcap.md @@ -1,6 +1,6 @@ # PCAP processors -When a PCAP is uploaded (either through Malcolm's [upload web interface](upload.md#Upload) or just copied manually into the `./pcap/upload` directory), the `pcap-monitor` container has a script that picks up those PCAP files and publishes to a [ZeroMQ](https://zeromq.org/) topic that can be subscribed to by any other process that wants to analyze that PCAP. In Malcolm (at the time of the [v24.07.0 release]({{ site.github.repository_url }}/releases/tag/v24.07.0)), there are three such ZeroMQ topics: the `zeek`, `suricata` and `arkime` containers. These actually share the [same script]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/pcap_processor.py) to run the PCAP through Zeek, Suricata, and Arkime, respectively. For an example to follow, the `zeek` container is the less complicated of the two. To integrate a new PCAP processing tool into Malcolm (named `cooltool` for this example) the process would entail: +When a PCAP is uploaded (either through Malcolm's [upload web interface](upload.md#Upload) or just copied manually into the `./pcap/upload` directory), the `pcap-monitor` container has a script that picks up those PCAP files and publishes to a [ZeroMQ](https://zeromq.org/) topic that can be subscribed to by any other process that wants to analyze that PCAP. In Malcolm (at the time of the [v24.08.0 release]({{ site.github.repository_url }}/releases/tag/v24.08.0)), there are three such ZeroMQ topics: the `zeek`, `suricata` and `arkime` containers. These actually share the [same script]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/pcap_processor.py) to run the PCAP through Zeek, Suricata, and Arkime, respectively. For an example to follow, the `zeek` container is the less complicated of the two. To integrate a new PCAP processing tool into Malcolm (named `cooltool` for this example) the process would entail: 1. Define the service as instructed in the [Adding a new service](contributing-new-image.md#NewImage) section * Note how the existing `zeek` and `arkime` services use [bind mounts](contributing-local-modifications.md#Bind) to access the local `./pcap` directory diff --git a/docs/hedgehog-iso-build.md b/docs/hedgehog-iso-build.md index 2763df805..6218210ec 100644 --- a/docs/hedgehog-iso-build.md +++ b/docs/hedgehog-iso-build.md @@ -29,7 +29,7 @@ Building the ISO may take 90 minutes or more depending on your system. As the bu ``` … -Finished, created "/sensor-build/hedgehog-24.07.0.iso" +Finished, created "/sensor-build/hedgehog-24.08.0.iso" … ``` diff --git a/docs/kubernetes.md b/docs/kubernetes.md index 71049e9e5..2d71d2b49 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -272,28 +272,28 @@ agent2 | agent2 | 192.168.56.12 | agent2 | k3s | 6000m | agent1 | agent1 | 192.168.56.11 | agent1 | k3s | 6000m | 861.34m | 14.36% | 19.55Gi | 9.29Gi | 61.28Gi | 11 | Pod Name | State | Pod IP | Pod Kind | Worker Node | CPU Usage | Memory Usage | Container Name:Restarts | Container Image | -api-deployment-6f4686cf59-bn286 | Running | 10.42.2.14 | ReplicaSet | agent1 | 0.11m | 59.62Mi | api-container:0 | api:24.07.0 | -file-monitor-deployment-855646bd75-vk7st | Running | 10.42.2.16 | ReplicaSet | agent1 | 8.47m | 1.46Gi | file-monitor-container:0 | file-monitor:24.07.0 | -zeek-live-deployment-64b69d4b6f-947vr | Running | 10.42.2.17 | ReplicaSet | agent1 | 0.02m | 12.44Mi | zeek-live-container:0 | zeek:24.07.0 | -dashboards-helper-deployment-69dc54f6b6-ln4sq | Running | 10.42.2.15 | ReplicaSet | agent1 | 10.77m | 38.43Mi | dashboards-helper-container:0 | dashboards-helper:24.07.0 | -upload-deployment-586568844b-4jnk9 | Running | 10.42.2.18 | ReplicaSet | agent1 | 0.15m | 29.78Mi | upload-container:0 | file-upload:24.07.0 | -filebeat-deployment-6ff8bc444f-t7h49 | Running | 10.42.2.20 | ReplicaSet | agent1 | 2.84m | 70.71Mi | filebeat-container:0 | filebeat-oss:24.07.0 | -zeek-offline-deployment-844f4865bd-g2sdm | Running | 10.42.2.21 | ReplicaSet | agent1 | 0.17m | 41.92Mi | zeek-offline-container:0 | zeek:24.07.0 | -logstash-deployment-6fbc9fdcd5-hwx8s | Running | 10.42.2.22 | ReplicaSet | agent1 | 85.55m | 2.91Gi | logstash-container:0 | logstash-oss:24.07.0 | -netbox-deployment-cdcff4977-hbbw5 | Running | 10.42.2.23 | ReplicaSet | agent1 | 807.64m | 702.86Mi | netbox-container:0 | netbox:24.07.0 | -suricata-offline-deployment-6ccdb89478-z5696 | Running | 10.42.2.19 | ReplicaSet | agent1 | 0.22m | 34.88Mi | suricata-offline-container:0 | suricata:24.07.0 | -dashboards-deployment-69b5465db-vz88g | Running | 10.42.1.14 | ReplicaSet | agent2 | 0.94m | 100.12Mi | dashboards-container:0 | dashboards:24.07.0 | -netbox-redis-cache-deployment-5f77d47b8b-z7t2z | Running | 10.42.1.15 | ReplicaSet | agent2 | 3.57m | 7.36Mi | netbox-redis-cache-container:0 | redis:24.07.0 | -suricata-live-deployment-6494c77759-9rlnt | Running | 10.42.1.16 | ReplicaSet | agent2 | 0.02m | 9.69Mi | suricata-live-container:0 | suricata:24.07.0 | -freq-deployment-cfd84fd97-dnngf | Running | 10.42.1.17 | ReplicaSet | agent2 | 0.2m | 26.36Mi | freq-container:0 | freq:24.07.0 | -arkime-deployment-56999cdd66-s98pp | Running | 10.42.1.18 | ReplicaSet | agent2 | 4.15m | 113.07Mi | arkime-container:0 | arkime:24.07.0 | -pcap-monitor-deployment-594ff674c4-fsm7m | Running | 10.42.1.19 | ReplicaSet | agent2 | 1.24m | 48.44Mi | pcap-monitor-container:0 | pcap-monitor:24.07.0 | -pcap-capture-deployment-7c8bf6957-jzpzn | Running | 10.42.1.20 | ReplicaSet | agent2 | 0.02m | 9.64Mi | pcap-capture-container:0 | pcap-capture:24.07.0 | -netbox-postgres-deployment-5879b8dffc-kkt56 | Running | 10.42.1.21 | ReplicaSet | agent2 | 70.91m | 33.02Mi | netbox-postgres-container:0 | postgresql:24.07.0 | -htadmin-deployment-6fc46888b9-sq6ln | Running | 10.42.1.23 | ReplicaSet | agent2 | 0.14m | 30.53Mi | htadmin-container:0 | htadmin:24.07.0 | -netbox-redis-deployment-5bcd8f6c96-j5xpf | Running | 10.42.1.24 | ReplicaSet | agent2 | 1.46m | 7.34Mi | netbox-redis-container:0 | redis:24.07.0 | -nginx-proxy-deployment-69fcc4968d-f68tq | Running | 10.42.1.22 | ReplicaSet | agent2 | 0.31m | 22.63Mi | nginx-proxy-container:0 | nginx-proxy:24.07.0 | -opensearch-deployment-75498799f6-4zmwd | Running | 10.42.1.25 | ReplicaSet | agent2 | 89.8m | 11.03Gi | opensearch-container:0 | opensearch:24.07.0 | +api-deployment-6f4686cf59-bn286 | Running | 10.42.2.14 | ReplicaSet | agent1 | 0.11m | 59.62Mi | api-container:0 | api:24.08.0 | +file-monitor-deployment-855646bd75-vk7st | Running | 10.42.2.16 | ReplicaSet | agent1 | 8.47m | 1.46Gi | file-monitor-container:0 | file-monitor:24.08.0 | +zeek-live-deployment-64b69d4b6f-947vr | Running | 10.42.2.17 | ReplicaSet | agent1 | 0.02m | 12.44Mi | zeek-live-container:0 | zeek:24.08.0 | +dashboards-helper-deployment-69dc54f6b6-ln4sq | Running | 10.42.2.15 | ReplicaSet | agent1 | 10.77m | 38.43Mi | dashboards-helper-container:0 | dashboards-helper:24.08.0 | +upload-deployment-586568844b-4jnk9 | Running | 10.42.2.18 | ReplicaSet | agent1 | 0.15m | 29.78Mi | upload-container:0 | file-upload:24.08.0 | +filebeat-deployment-6ff8bc444f-t7h49 | Running | 10.42.2.20 | ReplicaSet | agent1 | 2.84m | 70.71Mi | filebeat-container:0 | filebeat-oss:24.08.0 | +zeek-offline-deployment-844f4865bd-g2sdm | Running | 10.42.2.21 | ReplicaSet | agent1 | 0.17m | 41.92Mi | zeek-offline-container:0 | zeek:24.08.0 | +logstash-deployment-6fbc9fdcd5-hwx8s | Running | 10.42.2.22 | ReplicaSet | agent1 | 85.55m | 2.91Gi | logstash-container:0 | logstash-oss:24.08.0 | +netbox-deployment-cdcff4977-hbbw5 | Running | 10.42.2.23 | ReplicaSet | agent1 | 807.64m | 702.86Mi | netbox-container:0 | netbox:24.08.0 | +suricata-offline-deployment-6ccdb89478-z5696 | Running | 10.42.2.19 | ReplicaSet | agent1 | 0.22m | 34.88Mi | suricata-offline-container:0 | suricata:24.08.0 | +dashboards-deployment-69b5465db-vz88g | Running | 10.42.1.14 | ReplicaSet | agent2 | 0.94m | 100.12Mi | dashboards-container:0 | dashboards:24.08.0 | +netbox-redis-cache-deployment-5f77d47b8b-z7t2z | Running | 10.42.1.15 | ReplicaSet | agent2 | 3.57m | 7.36Mi | netbox-redis-cache-container:0 | redis:24.08.0 | +suricata-live-deployment-6494c77759-9rlnt | Running | 10.42.1.16 | ReplicaSet | agent2 | 0.02m | 9.69Mi | suricata-live-container:0 | suricata:24.08.0 | +freq-deployment-cfd84fd97-dnngf | Running | 10.42.1.17 | ReplicaSet | agent2 | 0.2m | 26.36Mi | freq-container:0 | freq:24.08.0 | +arkime-deployment-56999cdd66-s98pp | Running | 10.42.1.18 | ReplicaSet | agent2 | 4.15m | 113.07Mi | arkime-container:0 | arkime:24.08.0 | +pcap-monitor-deployment-594ff674c4-fsm7m | Running | 10.42.1.19 | ReplicaSet | agent2 | 1.24m | 48.44Mi | pcap-monitor-container:0 | pcap-monitor:24.08.0 | +pcap-capture-deployment-7c8bf6957-jzpzn | Running | 10.42.1.20 | ReplicaSet | agent2 | 0.02m | 9.64Mi | pcap-capture-container:0 | pcap-capture:24.08.0 | +netbox-postgres-deployment-5879b8dffc-kkt56 | Running | 10.42.1.21 | ReplicaSet | agent2 | 70.91m | 33.02Mi | netbox-postgres-container:0 | postgresql:24.08.0 | +htadmin-deployment-6fc46888b9-sq6ln | Running | 10.42.1.23 | ReplicaSet | agent2 | 0.14m | 30.53Mi | htadmin-container:0 | htadmin:24.08.0 | +netbox-redis-deployment-5bcd8f6c96-j5xpf | Running | 10.42.1.24 | ReplicaSet | agent2 | 1.46m | 7.34Mi | netbox-redis-container:0 | redis:24.08.0 | +nginx-proxy-deployment-69fcc4968d-f68tq | Running | 10.42.1.22 | ReplicaSet | agent2 | 0.31m | 22.63Mi | nginx-proxy-container:0 | nginx-proxy:24.08.0 | +opensearch-deployment-75498799f6-4zmwd | Running | 10.42.1.25 | ReplicaSet | agent2 | 89.8m | 11.03Gi | opensearch-container:0 | opensearch:24.08.0 | ``` The other control scripts (`stop`, `restart`, `logs`, etc.) work in a similar manner as in a Docker-based deployment. One notable difference is the `wipe` script: data on PersistentVolume storage cannot be deleted by `wipe`. It must be deleted manually on the storage media underlying the PersistentVolumes. @@ -554,28 +554,28 @@ agent1 | agent1 | 192.168.56.11 | agent1 | k3s | 6000m | agent2 | agent2 | 192.168.56.12 | agent2 | k3s | 6000m | 552.71m | 9.21% | 19.55Gi | 13.27Gi | 61.28Gi | 12 | Pod Name | State | Pod IP | Pod Kind | Worker Node | CPU Usage | Memory Usage | Container Name:Restarts | Container Image | -netbox-redis-cache-deployment-5f77d47b8b-jr9nt | Running | 10.42.2.6 | ReplicaSet | agent2 | 1.89m | 7.24Mi | netbox-redis-cache-container:0 | redis:24.07.0 | -netbox-redis-deployment-5bcd8f6c96-bkzmh | Running | 10.42.2.5 | ReplicaSet | agent2 | 1.62m | 7.52Mi | netbox-redis-container:0 | redis:24.07.0 | -dashboards-helper-deployment-69dc54f6b6-ks7ps | Running | 10.42.2.4 | ReplicaSet | agent2 | 12.95m | 40.75Mi | dashboards-helper-container:0 | dashboards-helper:24.07.0 | -freq-deployment-cfd84fd97-5bwp6 | Running | 10.42.2.8 | ReplicaSet | agent2 | 0.11m | 26.33Mi | freq-container:0 | freq:24.07.0 | -pcap-capture-deployment-7c8bf6957-hkvkn | Running | 10.42.2.12 | ReplicaSet | agent2 | 0.02m | 9.21Mi | pcap-capture-container:0 | pcap-capture:24.07.0 | -nginx-proxy-deployment-69fcc4968d-m57rz | Running | 10.42.2.10 | ReplicaSet | agent2 | 0.91m | 22.72Mi | nginx-proxy-container:0 | nginx-proxy:24.07.0 | -htadmin-deployment-6fc46888b9-vpt7l | Running | 10.42.2.7 | ReplicaSet | agent2 | 0.16m | 30.21Mi | htadmin-container:0 | htadmin:24.07.0 | -opensearch-deployment-75498799f6-5v92w | Running | 10.42.2.13 | ReplicaSet | agent2 | 139.2m | 10.86Gi | opensearch-container:0 | opensearch:24.07.0 | -zeek-live-deployment-64b69d4b6f-fcb6n | Running | 10.42.2.9 | ReplicaSet | agent2 | 0.02m | 109.55Mi | zeek-live-container:0 | zeek:24.07.0 | -dashboards-deployment-69b5465db-kgsqk | Running | 10.42.2.3 | ReplicaSet | agent2 | 14.98m | 108.85Mi | dashboards-container:0 | dashboards:24.07.0 | -arkime-deployment-56999cdd66-xxpw9 | Running | 10.42.2.11 | ReplicaSet | agent2 | 208.95m | 78.42Mi | arkime-container:0 | arkime:24.07.0 | -api-deployment-6f4686cf59-xt9md | Running | 10.42.1.3 | ReplicaSet | agent1 | 0.14m | 56.88Mi | api-container:0 | api:24.07.0 | -netbox-postgres-deployment-5879b8dffc-lb4qm | Running | 10.42.1.6 | ReplicaSet | agent1 | 141.2m | 48.02Mi | netbox-postgres-container:0 | postgresql:24.07.0 | -pcap-monitor-deployment-594ff674c4-fwq7g | Running | 10.42.1.12 | ReplicaSet | agent1 | 3.93m | 46.44Mi | pcap-monitor-container:0 | pcap-monitor:24.07.0 | -suricata-offline-deployment-6ccdb89478-j5fgj | Running | 10.42.1.10 | ReplicaSet | agent1 | 10.42m | 35.12Mi | suricata-offline-container:0 | suricata:24.07.0 | -suricata-live-deployment-6494c77759-rpt48 | Running | 10.42.1.8 | ReplicaSet | agent1 | 0.01m | 9.62Mi | suricata-live-container:0 | suricata:24.07.0 | -netbox-deployment-cdcff4977-7ns2q | Running | 10.42.1.7 | ReplicaSet | agent1 | 830.47m | 530.7Mi | netbox-container:0 | netbox:24.07.0 | -zeek-offline-deployment-844f4865bd-7x68b | Running | 10.42.1.9 | ReplicaSet | agent1 | 1.44m | 43.66Mi | zeek-offline-container:0 | zeek:24.07.0 | -filebeat-deployment-6ff8bc444f-pdgzj | Running | 10.42.1.11 | ReplicaSet | agent1 | 0.78m | 75.25Mi | filebeat-container:0 | filebeat-oss:24.07.0 | -file-monitor-deployment-855646bd75-nbngq | Running | 10.42.1.4 | ReplicaSet | agent1 | 1.69m | 1.46Gi | file-monitor-container:0 | file-monitor:24.07.0 | -upload-deployment-586568844b-9s7f5 | Running | 10.42.1.13 | ReplicaSet | agent1 | 0.14m | 29.62Mi | upload-container:0 | file-upload:24.07.0 | -logstash-deployment-6fbc9fdcd5-2hhx8 | Running | 10.42.1.5 | ReplicaSet | agent1 | 3236.29m | 357.36Mi | logstash-container:0 | logstash-oss:24.07.0 | +netbox-redis-cache-deployment-5f77d47b8b-jr9nt | Running | 10.42.2.6 | ReplicaSet | agent2 | 1.89m | 7.24Mi | netbox-redis-cache-container:0 | redis:24.08.0 | +netbox-redis-deployment-5bcd8f6c96-bkzmh | Running | 10.42.2.5 | ReplicaSet | agent2 | 1.62m | 7.52Mi | netbox-redis-container:0 | redis:24.08.0 | +dashboards-helper-deployment-69dc54f6b6-ks7ps | Running | 10.42.2.4 | ReplicaSet | agent2 | 12.95m | 40.75Mi | dashboards-helper-container:0 | dashboards-helper:24.08.0 | +freq-deployment-cfd84fd97-5bwp6 | Running | 10.42.2.8 | ReplicaSet | agent2 | 0.11m | 26.33Mi | freq-container:0 | freq:24.08.0 | +pcap-capture-deployment-7c8bf6957-hkvkn | Running | 10.42.2.12 | ReplicaSet | agent2 | 0.02m | 9.21Mi | pcap-capture-container:0 | pcap-capture:24.08.0 | +nginx-proxy-deployment-69fcc4968d-m57rz | Running | 10.42.2.10 | ReplicaSet | agent2 | 0.91m | 22.72Mi | nginx-proxy-container:0 | nginx-proxy:24.08.0 | +htadmin-deployment-6fc46888b9-vpt7l | Running | 10.42.2.7 | ReplicaSet | agent2 | 0.16m | 30.21Mi | htadmin-container:0 | htadmin:24.08.0 | +opensearch-deployment-75498799f6-5v92w | Running | 10.42.2.13 | ReplicaSet | agent2 | 139.2m | 10.86Gi | opensearch-container:0 | opensearch:24.08.0 | +zeek-live-deployment-64b69d4b6f-fcb6n | Running | 10.42.2.9 | ReplicaSet | agent2 | 0.02m | 109.55Mi | zeek-live-container:0 | zeek:24.08.0 | +dashboards-deployment-69b5465db-kgsqk | Running | 10.42.2.3 | ReplicaSet | agent2 | 14.98m | 108.85Mi | dashboards-container:0 | dashboards:24.08.0 | +arkime-deployment-56999cdd66-xxpw9 | Running | 10.42.2.11 | ReplicaSet | agent2 | 208.95m | 78.42Mi | arkime-container:0 | arkime:24.08.0 | +api-deployment-6f4686cf59-xt9md | Running | 10.42.1.3 | ReplicaSet | agent1 | 0.14m | 56.88Mi | api-container:0 | api:24.08.0 | +netbox-postgres-deployment-5879b8dffc-lb4qm | Running | 10.42.1.6 | ReplicaSet | agent1 | 141.2m | 48.02Mi | netbox-postgres-container:0 | postgresql:24.08.0 | +pcap-monitor-deployment-594ff674c4-fwq7g | Running | 10.42.1.12 | ReplicaSet | agent1 | 3.93m | 46.44Mi | pcap-monitor-container:0 | pcap-monitor:24.08.0 | +suricata-offline-deployment-6ccdb89478-j5fgj | Running | 10.42.1.10 | ReplicaSet | agent1 | 10.42m | 35.12Mi | suricata-offline-container:0 | suricata:24.08.0 | +suricata-live-deployment-6494c77759-rpt48 | Running | 10.42.1.8 | ReplicaSet | agent1 | 0.01m | 9.62Mi | suricata-live-container:0 | suricata:24.08.0 | +netbox-deployment-cdcff4977-7ns2q | Running | 10.42.1.7 | ReplicaSet | agent1 | 830.47m | 530.7Mi | netbox-container:0 | netbox:24.08.0 | +zeek-offline-deployment-844f4865bd-7x68b | Running | 10.42.1.9 | ReplicaSet | agent1 | 1.44m | 43.66Mi | zeek-offline-container:0 | zeek:24.08.0 | +filebeat-deployment-6ff8bc444f-pdgzj | Running | 10.42.1.11 | ReplicaSet | agent1 | 0.78m | 75.25Mi | filebeat-container:0 | filebeat-oss:24.08.0 | +file-monitor-deployment-855646bd75-nbngq | Running | 10.42.1.4 | ReplicaSet | agent1 | 1.69m | 1.46Gi | file-monitor-container:0 | file-monitor:24.08.0 | +upload-deployment-586568844b-9s7f5 | Running | 10.42.1.13 | ReplicaSet | agent1 | 0.14m | 29.62Mi | upload-container:0 | file-upload:24.08.0 | +logstash-deployment-6fbc9fdcd5-2hhx8 | Running | 10.42.1.5 | ReplicaSet | agent1 | 3236.29m | 357.36Mi | logstash-container:0 | logstash-oss:24.08.0 | ``` View container logs for the Malcolm deployment with `./scripts/logs` (if **[stern](https://github.com/stern/stern)** present in `$PATH`): diff --git a/docs/malcolm-iso.md b/docs/malcolm-iso.md index e01ef48c7..3edfa5033 100644 --- a/docs/malcolm-iso.md +++ b/docs/malcolm-iso.md @@ -41,7 +41,7 @@ Building the ISO may take 30 minutes or more depending on the system. As the bui ``` … -Finished, created "/malcolm-build/malcolm-iso/malcolm-24.07.0.iso" +Finished, created "/malcolm-build/malcolm-iso/malcolm-24.08.0.iso" … ``` diff --git a/docs/quickstart.md b/docs/quickstart.md index d8024dd72..906272f21 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -54,25 +54,25 @@ You can then observe the images have been retrieved by running `docker images`: ``` $ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -ghcr.io/idaholab/malcolm/api 24.07.0 xxxxxxxxxxxx 3 days ago 158MB -ghcr.io/idaholab/malcolm/arkime 24.07.0 xxxxxxxxxxxx 3 days ago 816MB -ghcr.io/idaholab/malcolm/dashboards 24.07.0 xxxxxxxxxxxx 3 days ago 1.02GB -ghcr.io/idaholab/malcolm/dashboards-helper 24.07.0 xxxxxxxxxxxx 3 days ago 184MB -ghcr.io/idaholab/malcolm/file-monitor 24.07.0 xxxxxxxxxxxx 3 days ago 588MB -ghcr.io/idaholab/malcolm/file-upload 24.07.0 xxxxxxxxxxxx 3 days ago 259MB -ghcr.io/idaholab/malcolm/filebeat-oss 24.07.0 xxxxxxxxxxxx 3 days ago 624MB -ghcr.io/idaholab/malcolm/freq 24.07.0 xxxxxxxxxxxx 3 days ago 132MB -ghcr.io/idaholab/malcolm/htadmin 24.07.0 xxxxxxxxxxxx 3 days ago 242MB -ghcr.io/idaholab/malcolm/logstash-oss 24.07.0 xxxxxxxxxxxx 3 days ago 1.35GB -ghcr.io/idaholab/malcolm/netbox 24.07.0 xxxxxxxxxxxx 3 days ago 1.01GB -ghcr.io/idaholab/malcolm/nginx-proxy 24.07.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/opensearch 24.07.0 xxxxxxxxxxxx 3 days ago 1.17GB -ghcr.io/idaholab/malcolm/pcap-capture 24.07.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/pcap-monitor 24.07.0 xxxxxxxxxxxx 3 days ago 213MB -ghcr.io/idaholab/malcolm/postgresql 24.07.0 xxxxxxxxxxxx 3 days ago 268MB -ghcr.io/idaholab/malcolm/redis 24.07.0 xxxxxxxxxxxx 3 days ago 34.2MB -ghcr.io/idaholab/malcolm/suricata 24.07.0 xxxxxxxxxxxx 3 days ago 278MB -ghcr.io/idaholab/malcolm/zeek 24.07.0 xxxxxxxxxxxx 3 days ago 1GB +ghcr.io/idaholab/malcolm/api 24.08.0 xxxxxxxxxxxx 3 days ago 158MB +ghcr.io/idaholab/malcolm/arkime 24.08.0 xxxxxxxxxxxx 3 days ago 816MB +ghcr.io/idaholab/malcolm/dashboards 24.08.0 xxxxxxxxxxxx 3 days ago 1.02GB +ghcr.io/idaholab/malcolm/dashboards-helper 24.08.0 xxxxxxxxxxxx 3 days ago 184MB +ghcr.io/idaholab/malcolm/file-monitor 24.08.0 xxxxxxxxxxxx 3 days ago 588MB +ghcr.io/idaholab/malcolm/file-upload 24.08.0 xxxxxxxxxxxx 3 days ago 259MB +ghcr.io/idaholab/malcolm/filebeat-oss 24.08.0 xxxxxxxxxxxx 3 days ago 624MB +ghcr.io/idaholab/malcolm/freq 24.08.0 xxxxxxxxxxxx 3 days ago 132MB +ghcr.io/idaholab/malcolm/htadmin 24.08.0 xxxxxxxxxxxx 3 days ago 242MB +ghcr.io/idaholab/malcolm/logstash-oss 24.08.0 xxxxxxxxxxxx 3 days ago 1.35GB +ghcr.io/idaholab/malcolm/netbox 24.08.0 xxxxxxxxxxxx 3 days ago 1.01GB +ghcr.io/idaholab/malcolm/nginx-proxy 24.08.0 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/opensearch 24.08.0 xxxxxxxxxxxx 3 days ago 1.17GB +ghcr.io/idaholab/malcolm/pcap-capture 24.08.0 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/pcap-monitor 24.08.0 xxxxxxxxxxxx 3 days ago 213MB +ghcr.io/idaholab/malcolm/postgresql 24.08.0 xxxxxxxxxxxx 3 days ago 268MB +ghcr.io/idaholab/malcolm/redis 24.08.0 xxxxxxxxxxxx 3 days ago 34.2MB +ghcr.io/idaholab/malcolm/suricata 24.08.0 xxxxxxxxxxxx 3 days ago 278MB +ghcr.io/idaholab/malcolm/zeek 24.08.0 xxxxxxxxxxxx 3 days ago 1GB ``` ### Import from pre-packaged tarballs diff --git a/docs/third-party-envs.md b/docs/third-party-envs.md index 8b8ea8214..4382fd76b 100644 --- a/docs/third-party-envs.md +++ b/docs/third-party-envs.md @@ -43,7 +43,7 @@ The files referenced in this section can be found in [scripts/third-party-enviro amazon-ebs: output will be in this color. ==> amazon-ebs: Prevalidating any provided VPC information - ==> amazon-ebs: Prevalidating AMI Name: malcolm-v24.07.0-arm64-2024-05-30T13-57-31Z + ==> amazon-ebs: Prevalidating AMI Name: malcolm-v24.08.0-arm64-2024-05-30T13-57-31Z amazon-ebs: Found Image ID: ami-xxxxxxxxxxxxxxxxx ... @@ -53,7 +53,7 @@ The files referenced in this section can be found in [scripts/third-party-enviro ==> amazon-ebs: Adding tags to AMI (ami-xxxxxxxxxxxxxxxxx)... ==> amazon-ebs: Tagging snapshot: snap-xxxxxxxxxxxxxxxxx ==> amazon-ebs: Creating AMI tags - amazon-ebs: Adding tag: "Malcolm": "idaholab/Malcolm/v24.07.0" + amazon-ebs: Adding tag: "Malcolm": "idaholab/Malcolm/v24.08.0" amazon-ebs: Adding tag: "source_ami_name": "amzn2-ami-kernel-5.10-hvm-2.0.20240521.0-arm64-gp2" ==> amazon-ebs: Creating snapshot tags ==> amazon-ebs: Terminating the source AWS instance... @@ -79,7 +79,7 @@ The files referenced in this section can be found in [scripts/third-party-enviro "Architecture": "arm64", "CreationDate": "2024-05-30T14:02:21.000Z", "ImageId": "ami-xxxxxxxxxxxxxxxxx", - "ImageLocation": "xxxxxxxxxxxx/malcolm-v24.07.0-arm64-2024-05-30T13-57-31Z", + "ImageLocation": "xxxxxxxxxxxx/malcolm-v24.08.0-arm64-2024-05-30T13-57-31Z", "ImageType": "machine", "Public": false, "OwnerId": "xxxxxxxxxxxx", @@ -100,14 +100,14 @@ The files referenced in this section can be found in [scripts/third-party-enviro ], "EnaSupport": true, "Hypervisor": "xen", - "Name": "malcolm-v24.07.0-arm64-2024-05-30T13-57-31Z", + "Name": "malcolm-v24.08.0-arm64-2024-05-30T13-57-31Z", "RootDeviceName": "/dev/xvda", "RootDeviceType": "ebs", "SriovNetSupport": "simple", "Tags": [ { "Key": "Malcolm", - "Value": "idaholab/Malcolm/v24.07.0" + "Value": "idaholab/Malcolm/v24.08.0" }, { "Key": "source_ami_name", diff --git a/docs/ubuntu-install-example.md b/docs/ubuntu-install-example.md index b58141ee8..26fa5ba09 100644 --- a/docs/ubuntu-install-example.md +++ b/docs/ubuntu-install-example.md @@ -257,25 +257,25 @@ Pulling zeek ... done user@host:~/Malcolm$ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -ghcr.io/idaholab/malcolm/api 24.07.0 xxxxxxxxxxxx 3 days ago 158MB -ghcr.io/idaholab/malcolm/arkime 24.07.0 xxxxxxxxxxxx 3 days ago 816MB -ghcr.io/idaholab/malcolm/dashboards 24.07.0 xxxxxxxxxxxx 3 days ago 1.02GB -ghcr.io/idaholab/malcolm/dashboards-helper 24.07.0 xxxxxxxxxxxx 3 days ago 184MB -ghcr.io/idaholab/malcolm/file-monitor 24.07.0 xxxxxxxxxxxx 3 days ago 588MB -ghcr.io/idaholab/malcolm/file-upload 24.07.0 xxxxxxxxxxxx 3 days ago 259MB -ghcr.io/idaholab/malcolm/filebeat-oss 24.07.0 xxxxxxxxxxxx 3 days ago 624MB -ghcr.io/idaholab/malcolm/freq 24.07.0 xxxxxxxxxxxx 3 days ago 132MB -ghcr.io/idaholab/malcolm/htadmin 24.07.0 xxxxxxxxxxxx 3 days ago 242MB -ghcr.io/idaholab/malcolm/logstash-oss 24.07.0 xxxxxxxxxxxx 3 days ago 1.35GB -ghcr.io/idaholab/malcolm/netbox 24.07.0 xxxxxxxxxxxx 3 days ago 1.01GB -ghcr.io/idaholab/malcolm/nginx-proxy 24.07.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/opensearch 24.07.0 xxxxxxxxxxxx 3 days ago 1.17GB -ghcr.io/idaholab/malcolm/pcap-capture 24.07.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/pcap-monitor 24.07.0 xxxxxxxxxxxx 3 days ago 213MB -ghcr.io/idaholab/malcolm/postgresql 24.07.0 xxxxxxxxxxxx 3 days ago 268MB -ghcr.io/idaholab/malcolm/redis 24.07.0 xxxxxxxxxxxx 3 days ago 34.2MB -ghcr.io/idaholab/malcolm/suricata 24.07.0 xxxxxxxxxxxx 3 days ago 278MB -ghcr.io/idaholab/malcolm/zeek 24.07.0 xxxxxxxxxxxx 3 days ago 1GB +ghcr.io/idaholab/malcolm/api 24.08.0 xxxxxxxxxxxx 3 days ago 158MB +ghcr.io/idaholab/malcolm/arkime 24.08.0 xxxxxxxxxxxx 3 days ago 816MB +ghcr.io/idaholab/malcolm/dashboards 24.08.0 xxxxxxxxxxxx 3 days ago 1.02GB +ghcr.io/idaholab/malcolm/dashboards-helper 24.08.0 xxxxxxxxxxxx 3 days ago 184MB +ghcr.io/idaholab/malcolm/file-monitor 24.08.0 xxxxxxxxxxxx 3 days ago 588MB +ghcr.io/idaholab/malcolm/file-upload 24.08.0 xxxxxxxxxxxx 3 days ago 259MB +ghcr.io/idaholab/malcolm/filebeat-oss 24.08.0 xxxxxxxxxxxx 3 days ago 624MB +ghcr.io/idaholab/malcolm/freq 24.08.0 xxxxxxxxxxxx 3 days ago 132MB +ghcr.io/idaholab/malcolm/htadmin 24.08.0 xxxxxxxxxxxx 3 days ago 242MB +ghcr.io/idaholab/malcolm/logstash-oss 24.08.0 xxxxxxxxxxxx 3 days ago 1.35GB +ghcr.io/idaholab/malcolm/netbox 24.08.0 xxxxxxxxxxxx 3 days ago 1.01GB +ghcr.io/idaholab/malcolm/nginx-proxy 24.08.0 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/opensearch 24.08.0 xxxxxxxxxxxx 3 days ago 1.17GB +ghcr.io/idaholab/malcolm/pcap-capture 24.08.0 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/pcap-monitor 24.08.0 xxxxxxxxxxxx 3 days ago 213MB +ghcr.io/idaholab/malcolm/postgresql 24.08.0 xxxxxxxxxxxx 3 days ago 268MB +ghcr.io/idaholab/malcolm/redis 24.08.0 xxxxxxxxxxxx 3 days ago 34.2MB +ghcr.io/idaholab/malcolm/suricata 24.08.0 xxxxxxxxxxxx 3 days ago 278MB +ghcr.io/idaholab/malcolm/zeek 24.08.0 xxxxxxxxxxxx 3 days ago 1GB ``` Finally, start Malcolm. When Malcolm starts it will stream informational and debug messages to the console until it has completed initializing. diff --git a/kubernetes/03-opensearch.yml b/kubernetes/03-opensearch.yml index 7127e2240..680e009db 100644 --- a/kubernetes/03-opensearch.yml +++ b/kubernetes/03-opensearch.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: opensearch-container - image: ghcr.io/idaholab/malcolm/opensearch:24.07.0 + image: ghcr.io/idaholab/malcolm/opensearch:24.08.0 imagePullPolicy: Always stdin: false tty: true @@ -71,7 +71,7 @@ spec: subPath: "opensearch" initContainers: - name: opensearch-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.07.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/04-dashboards.yml b/kubernetes/04-dashboards.yml index 5aa84b4b2..4652718b8 100644 --- a/kubernetes/04-dashboards.yml +++ b/kubernetes/04-dashboards.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: dashboards-container - image: ghcr.io/idaholab/malcolm/dashboards:24.07.0 + image: ghcr.io/idaholab/malcolm/dashboards:24.08.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/05-upload.yml b/kubernetes/05-upload.yml index b8a152c82..f13854396 100644 --- a/kubernetes/05-upload.yml +++ b/kubernetes/05-upload.yml @@ -34,7 +34,7 @@ spec: spec: containers: - name: upload-container - image: ghcr.io/idaholab/malcolm/file-upload:24.07.0 + image: ghcr.io/idaholab/malcolm/file-upload:24.08.0 imagePullPolicy: Always stdin: false tty: true @@ -73,7 +73,7 @@ spec: subPath: "upload" initContainers: - name: upload-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.07.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/06-pcap-monitor.yml b/kubernetes/06-pcap-monitor.yml index 322f6b8ea..8525b6121 100644 --- a/kubernetes/06-pcap-monitor.yml +++ b/kubernetes/06-pcap-monitor.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: pcap-monitor-container - image: ghcr.io/idaholab/malcolm/pcap-monitor:24.07.0 + image: ghcr.io/idaholab/malcolm/pcap-monitor:24.08.0 imagePullPolicy: Always stdin: false tty: true @@ -70,7 +70,7 @@ spec: name: pcap-monitor-zeek-volume initContainers: - name: pcap-monitor-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.07.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/07-arkime.yml b/kubernetes/07-arkime.yml index 9c3b0b2bb..69ca02bf0 100644 --- a/kubernetes/07-arkime.yml +++ b/kubernetes/07-arkime.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: arkime-container - image: ghcr.io/idaholab/malcolm/arkime:24.07.0 + image: ghcr.io/idaholab/malcolm/arkime:24.08.0 imagePullPolicy: Always stdin: false tty: true @@ -79,7 +79,7 @@ spec: name: arkime-pcap-volume initContainers: - name: arkime-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.07.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/08-api.yml b/kubernetes/08-api.yml index 6b05e7b6b..c4cec03dc 100644 --- a/kubernetes/08-api.yml +++ b/kubernetes/08-api.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: api-container - image: ghcr.io/idaholab/malcolm/api:24.07.0 + image: ghcr.io/idaholab/malcolm/api:24.08.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/09-dashboards-helper.yml b/kubernetes/09-dashboards-helper.yml index 7fce38283..d755c8b2e 100644 --- a/kubernetes/09-dashboards-helper.yml +++ b/kubernetes/09-dashboards-helper.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: dashboards-helper-container - image: ghcr.io/idaholab/malcolm/dashboards-helper:24.07.0 + image: ghcr.io/idaholab/malcolm/dashboards-helper:24.08.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/10-zeek.yml b/kubernetes/10-zeek.yml index 342adb9b2..b1fe88b53 100644 --- a/kubernetes/10-zeek.yml +++ b/kubernetes/10-zeek.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: zeek-offline-container - image: ghcr.io/idaholab/malcolm/zeek:24.07.0 + image: ghcr.io/idaholab/malcolm/zeek:24.08.0 imagePullPolicy: Always stdin: false tty: true @@ -64,7 +64,7 @@ spec: subPath: "zeek/intel" initContainers: - name: zeek-offline-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.07.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/11-suricata.yml b/kubernetes/11-suricata.yml index 8ff3dc79a..bff7d6315 100644 --- a/kubernetes/11-suricata.yml +++ b/kubernetes/11-suricata.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: suricata-offline-container - image: ghcr.io/idaholab/malcolm/suricata:24.07.0 + image: ghcr.io/idaholab/malcolm/suricata:24.08.0 imagePullPolicy: Always stdin: false tty: true @@ -55,7 +55,7 @@ spec: name: suricata-offline-custom-configs-volume initContainers: - name: suricata-offline-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.07.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/12-file-monitor.yml b/kubernetes/12-file-monitor.yml index fd54db2bb..7dd841c2d 100644 --- a/kubernetes/12-file-monitor.yml +++ b/kubernetes/12-file-monitor.yml @@ -33,7 +33,7 @@ spec: spec: containers: - name: file-monitor-container - image: ghcr.io/idaholab/malcolm/file-monitor:24.07.0 + image: ghcr.io/idaholab/malcolm/file-monitor:24.08.0 imagePullPolicy: Always stdin: false tty: true @@ -83,7 +83,7 @@ spec: name: file-monitor-yara-rules-custom-volume initContainers: - name: file-monitor-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.07.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/13-filebeat.yml b/kubernetes/13-filebeat.yml index 8c39c06c5..85b28f9f5 100644 --- a/kubernetes/13-filebeat.yml +++ b/kubernetes/13-filebeat.yml @@ -33,7 +33,7 @@ spec: spec: containers: - name: filebeat-container - image: ghcr.io/idaholab/malcolm/filebeat-oss:24.07.0 + image: ghcr.io/idaholab/malcolm/filebeat-oss:24.08.0 imagePullPolicy: Always stdin: false tty: true @@ -83,7 +83,7 @@ spec: subPath: "nginx" initContainers: - name: filebeat-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.07.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/14-logstash.yml b/kubernetes/14-logstash.yml index 4d1f34959..fe980e8e8 100644 --- a/kubernetes/14-logstash.yml +++ b/kubernetes/14-logstash.yml @@ -49,7 +49,7 @@ spec: # topologyKey: "kubernetes.io/hostname" containers: - name: logstash-container - image: ghcr.io/idaholab/malcolm/logstash-oss:24.07.0 + image: ghcr.io/idaholab/malcolm/logstash-oss:24.08.0 imagePullPolicy: Always stdin: false tty: true @@ -115,7 +115,7 @@ spec: subPath: "logstash" initContainers: - name: logstash-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.07.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/15-netbox-redis.yml b/kubernetes/15-netbox-redis.yml index 96b733b87..952a9bd15 100644 --- a/kubernetes/15-netbox-redis.yml +++ b/kubernetes/15-netbox-redis.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-redis-container - image: ghcr.io/idaholab/malcolm/redis:24.07.0 + image: ghcr.io/idaholab/malcolm/redis:24.08.0 imagePullPolicy: Always stdin: false tty: true @@ -83,7 +83,7 @@ spec: subPath: netbox/redis initContainers: - name: netbox-redis-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.07.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/16-netbox-redis-cache.yml b/kubernetes/16-netbox-redis-cache.yml index 3594959da..1e0c54472 100644 --- a/kubernetes/16-netbox-redis-cache.yml +++ b/kubernetes/16-netbox-redis-cache.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-redis-cache-container - image: ghcr.io/idaholab/malcolm/redis:24.07.0 + image: ghcr.io/idaholab/malcolm/redis:24.08.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/17-netbox-postgres.yml b/kubernetes/17-netbox-postgres.yml index e8efc196f..c7c65353f 100644 --- a/kubernetes/17-netbox-postgres.yml +++ b/kubernetes/17-netbox-postgres.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-postgres-container - image: ghcr.io/idaholab/malcolm/postgresql:24.07.0 + image: ghcr.io/idaholab/malcolm/postgresql:24.08.0 imagePullPolicy: Always stdin: false tty: true @@ -74,7 +74,7 @@ spec: subPath: netbox/postgres initContainers: - name: netbox-postgres-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.07.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/18-netbox.yml b/kubernetes/18-netbox.yml index baf2c6060..cdfa0d22e 100644 --- a/kubernetes/18-netbox.yml +++ b/kubernetes/18-netbox.yml @@ -36,7 +36,7 @@ spec: spec: containers: - name: netbox-container - image: ghcr.io/idaholab/malcolm/netbox:24.07.0 + image: ghcr.io/idaholab/malcolm/netbox:24.08.0 imagePullPolicy: Always stdin: false tty: true @@ -88,7 +88,7 @@ spec: subPath: netbox/media initContainers: - name: netbox-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.07.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/19-htadmin.yml b/kubernetes/19-htadmin.yml index f0dbe7d01..1ebabcec4 100644 --- a/kubernetes/19-htadmin.yml +++ b/kubernetes/19-htadmin.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: htadmin-container - image: ghcr.io/idaholab/malcolm/htadmin:24.07.0 + image: ghcr.io/idaholab/malcolm/htadmin:24.08.0 imagePullPolicy: Always stdin: false tty: true @@ -63,7 +63,7 @@ spec: subPath: "htadmin" initContainers: - name: htadmin-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.07.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/20-pcap-capture.yml b/kubernetes/20-pcap-capture.yml index 0ab3cffd5..e23e69de4 100644 --- a/kubernetes/20-pcap-capture.yml +++ b/kubernetes/20-pcap-capture.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: pcap-capture-container - image: ghcr.io/idaholab/malcolm/pcap-capture:24.07.0 + image: ghcr.io/idaholab/malcolm/pcap-capture:24.08.0 imagePullPolicy: Always stdin: false tty: true @@ -50,7 +50,7 @@ spec: subPath: "upload" initContainers: - name: pcap-capture-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.07.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/21-zeek-live.yml b/kubernetes/21-zeek-live.yml index d4c0d6371..7174ff5b8 100644 --- a/kubernetes/21-zeek-live.yml +++ b/kubernetes/21-zeek-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: zeek-live-container - image: ghcr.io/idaholab/malcolm/zeek:24.07.0 + image: ghcr.io/idaholab/malcolm/zeek:24.08.0 imagePullPolicy: Always stdin: false tty: true @@ -61,7 +61,7 @@ spec: subPath: "zeek/intel" initContainers: - name: zeek-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.07.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/22-suricata-live.yml b/kubernetes/22-suricata-live.yml index 5533f1c02..ee5512dc1 100644 --- a/kubernetes/22-suricata-live.yml +++ b/kubernetes/22-suricata-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: suricata-live-container - image: ghcr.io/idaholab/malcolm/suricata:24.07.0 + image: ghcr.io/idaholab/malcolm/suricata:24.08.0 imagePullPolicy: Always stdin: false tty: true @@ -56,7 +56,7 @@ spec: name: suricata-live-custom-configs-volume initContainers: - name: suricata-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.07.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/23-arkime-live.yml b/kubernetes/23-arkime-live.yml index 6d8ae18f9..57c22ba87 100644 --- a/kubernetes/23-arkime-live.yml +++ b/kubernetes/23-arkime-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: arkime-live-container - image: ghcr.io/idaholab/malcolm/arkime:24.07.0 + image: ghcr.io/idaholab/malcolm/arkime:24.08.0 imagePullPolicy: Always stdin: false tty: true @@ -62,7 +62,7 @@ spec: name: arkime-live-pcap-volume initContainers: - name: arkime-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.07.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/24-freq.yml b/kubernetes/24-freq.yml index d29face6f..ee8175e2a 100644 --- a/kubernetes/24-freq.yml +++ b/kubernetes/24-freq.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: freq-container - image: ghcr.io/idaholab/malcolm/freq:24.07.0 + image: ghcr.io/idaholab/malcolm/freq:24.08.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/98-nginx-proxy.yml b/kubernetes/98-nginx-proxy.yml index 9d3630059..1fb13cfff 100644 --- a/kubernetes/98-nginx-proxy.yml +++ b/kubernetes/98-nginx-proxy.yml @@ -39,7 +39,7 @@ spec: spec: containers: - name: nginx-proxy-container - image: ghcr.io/idaholab/malcolm/nginx-proxy:24.07.0 + image: ghcr.io/idaholab/malcolm/nginx-proxy:24.08.0 imagePullPolicy: Always stdin: false tty: true @@ -99,7 +99,7 @@ spec: subPath: "nginx" initContainers: - name: nginx-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.07.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 imagePullPolicy: Always stdin: false tty: true diff --git a/scripts/third-party-environments/aws/ami/packer_vars.json.example b/scripts/third-party-environments/aws/ami/packer_vars.json.example index 6e9eab121..8c853828f 100644 --- a/scripts/third-party-environments/aws/ami/packer_vars.json.example +++ b/scripts/third-party-environments/aws/ami/packer_vars.json.example @@ -3,7 +3,7 @@ "aws_secret_key": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "instance_type": "t2.micro", "instance_arch": "x86_64", - "malcolm_tag": "v24.07.0", + "malcolm_tag": "v24.08.0", "malcolm_repo": "idaholab/Malcolm", "malcolm_uid": "1000", "ssh_username": "ec2-user", diff --git a/scripts/third-party-environments/aws/ami/scripts/Malcolm_AMI_Setup.sh b/scripts/third-party-environments/aws/ami/scripts/Malcolm_AMI_Setup.sh index b10a2368a..9c91840eb 100755 --- a/scripts/third-party-environments/aws/ami/scripts/Malcolm_AMI_Setup.sh +++ b/scripts/third-party-environments/aws/ami/scripts/Malcolm_AMI_Setup.sh @@ -32,7 +32,7 @@ fi # -u UID (user UID, e.g., 1000) VERBOSE_FLAG= MALCOLM_REPO=${MALCOLM_REPO:-idaholab/Malcolm} -MALCOLM_TAG=${MALCOLM_TAG:-v24.07.0} +MALCOLM_TAG=${MALCOLM_TAG:-v24.08.0} [[ -z "$MALCOLM_UID" ]] && ( [[ $EUID -eq 0 ]] && MALCOLM_UID=1000 || MALCOLM_UID="$(id -u)" ) while getopts 'vr:t:u:' OPTION; do case "$OPTION" in From c6e3f32e10a78caba3aae815a04d87ed3bef45cd Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Thu, 1 Aug 2024 11:12:55 -0600 Subject: [PATCH 02/47] windows event log tweaks --- .../53bff390-5027-11ef-b744-23222ad0b42a.json | 131 ++++++++++++++++++ .../79202ee0-d811-11ee-820d-dd9fd73a3921.json | 129 +++++++---------- 2 files changed, 182 insertions(+), 78 deletions(-) create mode 100644 dashboards/dashboards/beats/53bff390-5027-11ef-b744-23222ad0b42a.json diff --git a/dashboards/dashboards/beats/53bff390-5027-11ef-b744-23222ad0b42a.json b/dashboards/dashboards/beats/53bff390-5027-11ef-b744-23222ad0b42a.json new file mode 100644 index 000000000..112a3254d --- /dev/null +++ b/dashboards/dashboards/beats/53bff390-5027-11ef-b744-23222ad0b42a.json @@ -0,0 +1,131 @@ +{ + "version": "2.15.0", + "objects": [ + { + "id": "53bff390-5027-11ef-b744-23222ad0b42a", + "type": "dashboard", + "namespaces": [ + "default" + ], + "updated_at": "2024-08-01T16:58:51.081Z", + "version": "WzEwNTgsMV0=", + "attributes": { + "title": "Windows Resource Utilization", + "hits": 0, + "description": "", + "panelsJSON": "[{\"version\":\"2.15.0\",\"gridData\":{\"x\":0,\"y\":0,\"w\":24,\"h\":15,\"i\":\"8596e887-a23b-4e4f-af00-358b985a2334\"},\"panelIndex\":\"8596e887-a23b-4e4f-af00-358b985a2334\",\"embeddableConfig\":{},\"panelRefName\":\"panel_0\"},{\"version\":\"2.15.0\",\"gridData\":{\"x\":24,\"y\":0,\"w\":24,\"h\":15,\"i\":\"7159396e-9b4d-4f15-9d81-b6deeb894013\"},\"panelIndex\":\"7159396e-9b4d-4f15-9d81-b6deeb894013\",\"embeddableConfig\":{},\"panelRefName\":\"panel_1\"},{\"version\":\"2.15.0\",\"gridData\":{\"x\":0,\"y\":15,\"w\":48,\"h\":20,\"i\":\"a770e839-1f2c-4564-ba56-8f3ace18df48\"},\"panelIndex\":\"a770e839-1f2c-4564-ba56-8f3ace18df48\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"}]", + "optionsJSON": "{\"useMargins\":true,\"hidePanelTitles\":false}", + "version": 1, + "timeRestore": false, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + } + }, + "references": [ + { + "name": "panel_0", + "type": "visualization", + "id": "008fb710-d817-11ee-820d-dd9fd73a3921" + }, + { + "name": "panel_1", + "type": "visualization", + "id": "7e9a8500-d816-11ee-820d-dd9fd73a3921" + }, + { + "name": "panel_2", + "type": "search", + "id": "3770db80-d815-11ee-820d-dd9fd73a3921" + } + ], + "migrationVersion": { + "dashboard": "7.9.3" + } + }, + { + "id": "008fb710-d817-11ee-820d-dd9fd73a3921", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-08-01T14:12:11.552Z", + "version": "WzkzNCwxXQ==", + "attributes": { + "title": "Windows CPU Usage", + "visState": "{\"title\":\"Windows CPU Usage\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"gauge\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#54B399\",\"split_mode\":\"terms\",\"split_color_mode\":\"opensearchDashboards\",\"metrics\":[{\"size\":1,\"agg_with\":\"noop\",\"order\":\"desc\",\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"top_hit\",\"field\":\"miscbeat.winstat.cpu_utilization\",\"order_by\":\"MALCOLM_OTHER_INDEX_TIME_FIELD_REPLACER\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"00\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"CPU Usage\",\"value_template\":\"{{value}}%\",\"filter\":{\"query\":\"event.module:winstat\",\"language\":\"kuery\"},\"terms_field\":\"host.name\"}],\"time_field\":\"MALCOLM_OTHER_INDEX_TIME_FIELD_REPLACER\",\"index_pattern\":\"MALCOLM_OTHER_INDEX_PATTERN_REPLACER\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"MALCOLM_OTHER_INDEX_PATTERN_REPLACER\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"gauge_color_rules\":[{\"id\":\"a50173c0-d816-11ee-a6da-0f56a6d37163\"}],\"gauge_width\":10,\"gauge_inner_width\":10,\"gauge_style\":\"half\",\"gauge_max\":\"100\",\"time_range_mode\":\"entire_time_range\"}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + } + }, + "references": [], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "7e9a8500-d816-11ee-820d-dd9fd73a3921", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2024-08-01T14:12:11.552Z", + "version": "WzkzMywxXQ==", + "attributes": { + "title": "Windows RAM Usage", + "visState": "{\"title\":\"Windows RAM Usage\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"gauge\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#54B399\",\"split_mode\":\"terms\",\"split_color_mode\":\"opensearchDashboards\",\"metrics\":[{\"size\":1,\"agg_with\":\"max\",\"order\":\"desc\",\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"top_hit\",\"field\":\"miscbeat.winstat.physical_used\",\"order_by\":\"MALCOLM_OTHER_INDEX_TIME_FIELD_REPLACER\"},{\"size\":1,\"agg_with\":\"max\",\"order\":\"desc\",\"id\":\"e4b6eb90-d815-11ee-a6da-0f56a6d37163\",\"type\":\"top_hit\",\"field\":\"miscbeat.winstat.physical_total\",\"order_by\":\"MALCOLM_OTHER_INDEX_TIME_FIELD_REPLACER\"},{\"id\":\"fe21e0d0-d815-11ee-a6da-0f56a6d37163\",\"type\":\"math\",\"variables\":[{\"id\":\"01e57880-d816-11ee-a6da-0f56a6d37163\",\"name\":\"used\",\"field\":\"61ca57f2-469d-11e7-af02-69e470af7417\"},{\"id\":\"082756a0-d816-11ee-a6da-0f56a6d37163\",\"name\":\"total\",\"field\":\"e4b6eb90-d815-11ee-a6da-0f56a6d37163\"}],\"script\":\"(params.used/params.total)*100\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"00.\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"terms_field\":\"host.name\",\"label\":\"RAM Usage\",\"terms_order_by\":\"_count\",\"value_template\":\"{{value}}%\"}],\"time_field\":\"MALCOLM_OTHER_INDEX_TIME_FIELD_REPLACER\",\"index_pattern\":\"MALCOLM_OTHER_INDEX_PATTERN_REPLACER\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"MALCOLM_OTHER_INDEX_PATTERN_REPLACER\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"gauge_color_rules\":[{\"id\":\"8db33970-d815-11ee-a6da-0f56a6d37163\"}],\"gauge_width\":10,\"gauge_inner_width\":10,\"gauge_style\":\"half\",\"filter\":{\"query\":\"event.module:winstat\",\"language\":\"kuery\"},\"time_range_mode\":\"entire_time_range\",\"gauge_max\":\"100\"}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + } + }, + "references": [], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "3770db80-d815-11ee-820d-dd9fd73a3921", + "type": "search", + "namespaces": [ + "default" + ], + "updated_at": "2024-08-01T14:12:11.552Z", + "version": "WzkzNiwxXQ==", + "attributes": { + "title": "Windows Resource Utilization", + "description": "", + "hits": 0, + "columns": [ + "host.name", + "miscbeat.winstat.cpu_utilization", + "miscbeat.winstat.physical_total", + "miscbeat.winstat.physical_used", + "miscbeat.winstat.physical_available", + "miscbeat.winstat.processes", + "miscbeat.winstat.threads", + "miscbeat.winstat.handles" + ], + "sort": [], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"event.module:winstat\",\"language\":\"kuery\"},\"highlightAll\":false,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"MALCOLM_OTHER_INDEX_TIME_FIELD_REPLACER\",\"fixed_interval\":\"30s\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + } + }, + "references": [ + { + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern", + "id": "MALCOLM_OTHER_INDEX_PATTERN_REPLACER" + } + ], + "migrationVersion": { + "search": "7.9.3" + } + } + ] +} \ No newline at end of file diff --git a/dashboards/dashboards/beats/79202ee0-d811-11ee-820d-dd9fd73a3921.json b/dashboards/dashboards/beats/79202ee0-d811-11ee-820d-dd9fd73a3921.json index c84824522..41d05342d 100644 --- a/dashboards/dashboards/beats/79202ee0-d811-11ee-820d-dd9fd73a3921.json +++ b/dashboards/dashboards/beats/79202ee0-d811-11ee-820d-dd9fd73a3921.json @@ -1,5 +1,5 @@ { - "version": "2.14.0", + "version": "2.15.0", "objects": [ { "id": "79202ee0-d811-11ee-820d-dd9fd73a3921", @@ -7,13 +7,13 @@ "namespaces": [ "default" ], - "updated_at": "2024-06-10T19:58:41.326Z", - "version": "WzEwMTIsMV0=", + "updated_at": "2024-08-01T15:32:35.352Z", + "version": "WzEwNDksMV0=", "attributes": { "title": "Windows Events", "hits": 0, - "description": "Windows events and resource statistics sent via Fluent Bit", - "panelsJSON": "[{\"version\":\"2.14.0\",\"gridData\":{\"x\":0,\"y\":0,\"w\":18,\"h\":15,\"i\":\"346bb696-5fa2-4504-a1d8-5a6f51244c7b\"},\"panelIndex\":\"346bb696-5fa2-4504-a1d8-5a6f51244c7b\",\"embeddableConfig\":{\"hidePanelTitles\":true,\"vis\":{\"sortColumn\":{\"colIndex\":2,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_0\"},{\"version\":\"2.14.0\",\"gridData\":{\"x\":18,\"y\":0,\"w\":30,\"h\":15,\"i\":\"9c39d8b3-ad8c-4247-b97f-9736e469c988\"},\"panelIndex\":\"9c39d8b3-ad8c-4247-b97f-9736e469c988\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_1\"},{\"version\":\"2.14.0\",\"gridData\":{\"x\":0,\"y\":15,\"w\":30,\"h\":36,\"i\":\"bccfb126-a864-4c11-a8c7-a9a1286c8f0f\"},\"panelIndex\":\"bccfb126-a864-4c11-a8c7-a9a1286c8f0f\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"2.14.0\",\"gridData\":{\"x\":30,\"y\":15,\"w\":18,\"h\":15,\"i\":\"bc116b54-f251-4e77-833c-c557b5d5c1d7\"},\"panelIndex\":\"bc116b54-f251-4e77-833c-c557b5d5c1d7\",\"embeddableConfig\":{\"hidePanelTitles\":true,\"table\":null,\"vis\":{\"sortColumn\":{\"colIndex\":1,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_3\"},{\"version\":\"2.14.0\",\"gridData\":{\"x\":30,\"y\":30,\"w\":18,\"h\":21,\"i\":\"1372927d-8b1d-4531-94fb-377dbccfff6b\"},\"panelIndex\":\"1372927d-8b1d-4531-94fb-377dbccfff6b\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_4\"},{\"version\":\"2.14.0\",\"gridData\":{\"x\":0,\"y\":51,\"w\":24,\"h\":15,\"i\":\"65476182-297e-4bbf-a7af-b0ff0584eeb0\"},\"panelIndex\":\"65476182-297e-4bbf-a7af-b0ff0584eeb0\",\"embeddableConfig\":{\"hidePanelTitles\":false},\"panelRefName\":\"panel_5\"},{\"version\":\"2.14.0\",\"gridData\":{\"x\":24,\"y\":51,\"w\":24,\"h\":15,\"i\":\"0bf80b0d-91f3-47ca-b2a3-ea6844c9727a\"},\"panelIndex\":\"0bf80b0d-91f3-47ca-b2a3-ea6844c9727a\",\"embeddableConfig\":{\"hidePanelTitles\":false},\"panelRefName\":\"panel_6\"},{\"version\":\"2.14.0\",\"gridData\":{\"x\":0,\"y\":66,\"w\":48,\"h\":30,\"i\":\"49f38efc-1ab3-4e38-96e5-b0458c026491\"},\"panelIndex\":\"49f38efc-1ab3-4e38-96e5-b0458c026491\",\"embeddableConfig\":{},\"panelRefName\":\"panel_7\"},{\"version\":\"2.14.0\",\"gridData\":{\"x\":0,\"y\":96,\"w\":48,\"h\":25,\"i\":\"0249a576-f162-4bf9-aa2d-bf85552665c7\"},\"panelIndex\":\"0249a576-f162-4bf9-aa2d-bf85552665c7\",\"embeddableConfig\":{},\"panelRefName\":\"panel_8\"}]", + "description": "Windows event logs", + "panelsJSON": "[{\"version\":\"2.15.0\",\"gridData\":{\"x\":0,\"y\":0,\"w\":18,\"h\":19,\"i\":\"346bb696-5fa2-4504-a1d8-5a6f51244c7b\"},\"panelIndex\":\"346bb696-5fa2-4504-a1d8-5a6f51244c7b\",\"embeddableConfig\":{\"hidePanelTitles\":true,\"vis\":{\"sortColumn\":{\"colIndex\":3,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_0\"},{\"version\":\"2.15.0\",\"gridData\":{\"x\":18,\"y\":0,\"w\":30,\"h\":19,\"i\":\"9c39d8b3-ad8c-4247-b97f-9736e469c988\"},\"panelIndex\":\"9c39d8b3-ad8c-4247-b97f-9736e469c988\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_1\"},{\"version\":\"2.15.0\",\"gridData\":{\"x\":0,\"y\":19,\"w\":30,\"h\":36,\"i\":\"bccfb126-a864-4c11-a8c7-a9a1286c8f0f\"},\"panelIndex\":\"bccfb126-a864-4c11-a8c7-a9a1286c8f0f\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"2.15.0\",\"gridData\":{\"x\":30,\"y\":19,\"w\":18,\"h\":15,\"i\":\"bc116b54-f251-4e77-833c-c557b5d5c1d7\"},\"panelIndex\":\"bc116b54-f251-4e77-833c-c557b5d5c1d7\",\"embeddableConfig\":{\"hidePanelTitles\":true,\"table\":null,\"vis\":{\"sortColumn\":{\"colIndex\":1,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_3\"},{\"version\":\"2.15.0\",\"gridData\":{\"x\":30,\"y\":34,\"w\":18,\"h\":21,\"i\":\"1372927d-8b1d-4531-94fb-377dbccfff6b\"},\"panelIndex\":\"1372927d-8b1d-4531-94fb-377dbccfff6b\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_4\"},{\"version\":\"2.15.0\",\"gridData\":{\"x\":0,\"y\":55,\"w\":16,\"h\":39,\"i\":\"3ce8c85f-ded2-4ff2-9a91-e85523bd2516\"},\"panelIndex\":\"3ce8c85f-ded2-4ff2-9a91-e85523bd2516\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_5\"},{\"version\":\"2.15.0\",\"gridData\":{\"x\":16,\"y\":55,\"w\":32,\"h\":39,\"i\":\"197395c9-4133-47ad-9290-1cb15f09e1ce\"},\"panelIndex\":\"197395c9-4133-47ad-9290-1cb15f09e1ce\",\"embeddableConfig\":{\"sort\":[[\"MALCOLM_OTHER_INDEX_TIME_FIELD_REPLACER\",\"desc\"]]},\"panelRefName\":\"panel_6\"},{\"version\":\"2.15.0\",\"gridData\":{\"x\":0,\"y\":94,\"w\":48,\"h\":30,\"i\":\"49f38efc-1ab3-4e38-96e5-b0458c026491\"},\"panelIndex\":\"49f38efc-1ab3-4e38-96e5-b0458c026491\",\"embeddableConfig\":{},\"panelRefName\":\"panel_7\"}]", "optionsJSON": "{\"hidePanelTitles\":false,\"useMargins\":true}", "version": 1, "timeRestore": false, @@ -50,22 +50,17 @@ { "name": "panel_5", "type": "visualization", - "id": "7e9a8500-d816-11ee-820d-dd9fd73a3921" + "id": "853142d0-5018-11ef-b744-23222ad0b42a" }, { "name": "panel_6", - "type": "visualization", - "id": "008fb710-d817-11ee-820d-dd9fd73a3921" + "type": "search", + "id": "af406720-501a-11ef-b744-23222ad0b42a" }, { "name": "panel_7", "type": "search", "id": "be2f24d0-d809-11ee-820d-dd9fd73a3921" - }, - { - "name": "panel_8", - "type": "search", - "id": "3770db80-d815-11ee-820d-dd9fd73a3921" } ], "migrationVersion": { @@ -78,11 +73,11 @@ "namespaces": [ "default" ], - "updated_at": "2024-06-10T19:11:30.906Z", - "version": "WzkwMiwxXQ==", + "updated_at": "2024-08-01T14:12:11.552Z", + "version": "WzkyOCwxXQ==", "attributes": { "title": "Windows Events by Host", - "visState": "{\"title\":\"Windows Events by Host\",\"type\":\"table\",\"aggs\":[{\"id\":\"4\",\"enabled\":true,\"type\":\"top_hits\",\"params\":{\"field\":\"@timestamp\",\"aggregate\":\"concat\",\"size\":1,\"sortField\":\"@timestamp\",\"sortOrder\":\"desc\",\"customLabel\":\"Last Log\"},\"schema\":\"metric\"},{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"host.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Origin\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.winlog.Computer\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Computer Name\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", + "visState": "{\"title\":\"Windows Events by Host\",\"type\":\"table\",\"aggs\":[{\"id\":\"4\",\"enabled\":true,\"type\":\"top_hits\",\"params\":{\"field\":\"MALCOLM_OTHER_INDEX_TIME_FIELD_REPLACER\",\"aggregate\":\"concat\",\"size\":1,\"sortField\":\"MALCOLM_OTHER_INDEX_TIME_FIELD_REPLACER\",\"sortOrder\":\"desc\",\"customLabel\":\"Last Log\"},\"schema\":\"metric\"},{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"host.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Origin\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"host.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Computer Name\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", "uiStateJSON": "{}", "description": "", "version": 1, @@ -108,11 +103,11 @@ "namespaces": [ "default" ], - "updated_at": "2024-06-10T19:11:30.906Z", - "version": "WzkwMywxXQ==", + "updated_at": "2024-08-01T14:12:11.552Z", + "version": "WzkyOSwxXQ==", "attributes": { "title": "Windows Events over Time", - "visState": "{\"title\":\"Windows Events over Time\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"now/d\",\"to\":\"now/d\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"event.dataset\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":5,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"square root\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}", + "visState": "{\"title\":\"Windows Events over Time\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"MALCOLM_OTHER_INDEX_TIME_FIELD_REPLACER\",\"timeRange\":{\"from\":\"now/d\",\"to\":\"now/d\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"event.dataset\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":5,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"square root\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}", "uiStateJSON": "{\"vis\":{\"legendOpen\":true}}", "description": "", "version": 1, @@ -138,8 +133,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-06-10T19:11:30.906Z", - "version": "WzkwNCwxXQ==", + "updated_at": "2024-08-01T14:12:11.552Z", + "version": "WzkzMCwxXQ==", "attributes": { "title": "Windows Event Provider", "visState": "{\"title\":\"Windows Event Provider\",\"type\":\"horizontal_bar\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.winlog.ProviderName\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":25,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Event Provider\"},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.winlog.ProviderName\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":25,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Event Provider\"},\"schema\":\"group\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":50},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"square root\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":75,\"filter\":true,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":true},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}", @@ -168,8 +163,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-06-10T19:11:30.906Z", - "version": "WzkwNSwxXQ==", + "updated_at": "2024-08-01T14:12:11.552Z", + "version": "WzkzMSwxXQ==", "attributes": { "title": "Windows Event Results", "visState": "{\"title\":\"Windows Event Results\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"event.result\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Event Result\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", @@ -198,8 +193,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-06-10T19:57:25.326Z", - "version": "WzEwMDksMV0=", + "updated_at": "2024-08-01T14:12:11.552Z", + "version": "WzkzMiwxXQ==", "attributes": { "title": "Windows Event Users", "visState": "{\"title\":\"Windows Event Users\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"related.user\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Username\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", @@ -223,76 +218,56 @@ } }, { - "id": "7e9a8500-d816-11ee-820d-dd9fd73a3921", + "id": "853142d0-5018-11ef-b744-23222ad0b42a", "type": "visualization", "namespaces": [ "default" ], - "updated_at": "2024-06-10T19:11:30.906Z", - "version": "WzkwOSwxXQ==", + "updated_at": "2024-08-01T15:30:06.733Z", + "version": "WzEwNDcsMV0=", "attributes": { - "title": "Windows RAM Usage", - "visState": "{\"title\":\"Windows RAM Usage\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"gauge\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#54B399\",\"split_mode\":\"terms\",\"split_color_mode\":\"opensearchDashboards\",\"metrics\":[{\"size\":1,\"agg_with\":\"max\",\"order\":\"desc\",\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"top_hit\",\"field\":\"miscbeat.winstat.physical_used\",\"order_by\":\"@timestamp\"},{\"size\":1,\"agg_with\":\"max\",\"order\":\"desc\",\"id\":\"e4b6eb90-d815-11ee-a6da-0f56a6d37163\",\"type\":\"top_hit\",\"field\":\"miscbeat.winstat.physical_total\",\"order_by\":\"@timestamp\"},{\"id\":\"fe21e0d0-d815-11ee-a6da-0f56a6d37163\",\"type\":\"math\",\"variables\":[{\"id\":\"01e57880-d816-11ee-a6da-0f56a6d37163\",\"name\":\"used\",\"field\":\"61ca57f2-469d-11e7-af02-69e470af7417\"},{\"id\":\"082756a0-d816-11ee-a6da-0f56a6d37163\",\"name\":\"total\",\"field\":\"e4b6eb90-d815-11ee-a6da-0f56a6d37163\"}],\"script\":\"(params.used/params.total)*100\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"00.\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"terms_field\":\"host.name\",\"label\":\"RAM Usage\",\"terms_order_by\":\"_count\",\"value_template\":\"{{value}}%\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"MALCOLM_OTHER_INDEX_PATTERN_REPLACER\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"MALCOLM_NETWORK_INDEX_PATTERN_REPLACER\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"gauge_color_rules\":[{\"id\":\"8db33970-d815-11ee-a6da-0f56a6d37163\"}],\"gauge_width\":10,\"gauge_inner_width\":10,\"gauge_style\":\"half\",\"filter\":{\"query\":\"event.module:winstat\",\"language\":\"kuery\"},\"time_range_mode\":\"entire_time_range\",\"gauge_max\":\"100\"}}", - "uiStateJSON": "{}", + "title": "Windows Process Executable", + "visState": "{\"title\":\"Windows Process Executable\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"process.executable\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Process Executable\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":25,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", + "uiStateJSON": "{\"vis\":{\"sortColumn\":{\"colIndex\":1,\"direction\":\"desc\"}}}", "description": "", "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" - } + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" + }, + "savedSearchRefName": "search_0" }, - "references": [], - "migrationVersion": { - "visualization": "7.10.0" - } - }, - { - "id": "008fb710-d817-11ee-820d-dd9fd73a3921", - "type": "visualization", - "namespaces": [ - "default" - ], - "updated_at": "2024-06-10T19:11:30.906Z", - "version": "WzkxMCwxXQ==", - "attributes": { - "title": "Windows CPU Usage", - "visState": "{\"title\":\"Windows CPU Usage\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"gauge\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#54B399\",\"split_mode\":\"terms\",\"split_color_mode\":\"opensearchDashboards\",\"metrics\":[{\"size\":1,\"agg_with\":\"noop\",\"order\":\"desc\",\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"top_hit\",\"field\":\"miscbeat.winstat.cpu_utilization\",\"order_by\":\"@timestamp\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"00\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"CPU Usage\",\"value_template\":\"{{value}}%\",\"filter\":{\"query\":\"event.module:winstat\",\"language\":\"kuery\"},\"terms_field\":\"host.name\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"MALCOLM_OTHER_INDEX_PATTERN_REPLACER\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"MALCOLM_NETWORK_INDEX_PATTERN_REPLACER\",\"default_timefield\":\"firstPacket\",\"isModelInvalid\":false,\"gauge_color_rules\":[{\"id\":\"a50173c0-d816-11ee-a6da-0f56a6d37163\"}],\"gauge_width\":10,\"gauge_inner_width\":10,\"gauge_style\":\"half\",\"gauge_max\":\"100\",\"time_range_mode\":\"entire_time_range\"}}", - "uiStateJSON": "{}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + "references": [ + { + "name": "search_0", + "type": "search", + "id": "be2f24d0-d809-11ee-820d-dd9fd73a3921" } - }, - "references": [], + ], "migrationVersion": { "visualization": "7.10.0" } }, { - "id": "be2f24d0-d809-11ee-820d-dd9fd73a3921", + "id": "af406720-501a-11ef-b744-23222ad0b42a", "type": "search", "namespaces": [ "default" ], - "updated_at": "2024-06-10T19:11:30.906Z", - "version": "WzkwNywxXQ==", + "updated_at": "2024-08-01T15:28:21.137Z", + "version": "WzEwNDYsMV0=", "attributes": { - "title": "Windows Event Logs", + "title": "Windows Event Logs - Process Command Line", "description": "", "hits": 0, "columns": [ "host.name", - "miscbeat.winlog.Computer", - "event.dataset", - "event.id", - "miscbeat.winlog.ProviderName", "related.user", - "event.result" + "process.command_line" ], "sort": [], "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"event.module:(winlog OR winevtlog)\",\"language\":\"lucene\"},\"highlightAll\":false,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + "searchSourceJSON": "{\"query\":{\"query\":\"event.module:(winlog OR winevtlog) AND process.command_line:*\",\"language\":\"lucene\"},\"highlightAll\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"MALCOLM_OTHER_INDEX_TIME_FIELD_REPLACER\",\"fixed_interval\":\"30d\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" } }, "references": [ @@ -307,31 +282,29 @@ } }, { - "id": "3770db80-d815-11ee-820d-dd9fd73a3921", + "id": "be2f24d0-d809-11ee-820d-dd9fd73a3921", "type": "search", "namespaces": [ "default" ], - "updated_at": "2024-06-10T19:11:30.906Z", - "version": "WzkwOCwxXQ==", + "updated_at": "2024-08-01T14:12:11.552Z", + "version": "WzkzNSwxXQ==", "attributes": { - "title": "Windows Resource Utilization", + "title": "Windows Event Logs", "description": "", "hits": 0, "columns": [ "host.name", - "miscbeat.winstat.cpu_utilization", - "miscbeat.winstat.physical_total", - "miscbeat.winstat.physical_used", - "miscbeat.winstat.physical_available", - "miscbeat.winstat.processes", - "miscbeat.winstat.threads", - "miscbeat.winstat.handles" + "event.dataset", + "event.id", + "miscbeat.winlog.ProviderName", + "related.user", + "event.result" ], "sort": [], "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"event.module:winstat\",\"language\":\"kuery\"},\"highlightAll\":false,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"fixed_interval\":\"30s\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + "searchSourceJSON": "{\"query\":{\"query\":\"event.module:(winlog OR winevtlog)\",\"language\":\"lucene\"},\"highlightAll\":false,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"MALCOLM_OTHER_INDEX_TIME_FIELD_REPLACER\",\"fixed_interval\":\"30s\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" } }, "references": [ From b0dfc47e344232dea242c91406303ee42e7abd8b Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Thu, 1 Aug 2024 15:32:56 -0600 Subject: [PATCH 03/47] don't make user confirm netbox autopopulation twice --- docs/kubernetes.md | 2 +- docs/malcolm-hedgehog-e2e-iso-install.md | 4 ++-- scripts/install.py | 24 +++++++----------------- 3 files changed, 10 insertions(+), 20 deletions(-) diff --git a/docs/kubernetes.md b/docs/kubernetes.md index 2d71d2b49..beeeccc66 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -436,7 +436,7 @@ Should Malcolm automatically populate NetBox inventory based on observed network Specify default NetBox site name: Malcolm -Should Malcolm create "catch-all" prefixes for private IP address space? (y / N): n +Should Malcolm automatically create missing NetBox subnet prefixes based on observed network traffic? (y / N): n Enable dark mode for OpenSearch Dashboards? (Y / n): y diff --git a/docs/malcolm-hedgehog-e2e-iso-install.md b/docs/malcolm-hedgehog-e2e-iso-install.md index 9a50c8d74..bda9cb861 100644 --- a/docs/malcolm-hedgehog-e2e-iso-install.md +++ b/docs/malcolm-hedgehog-e2e-iso-install.md @@ -278,8 +278,8 @@ The [configuration and tuning](malcolm-config.md#ConfigAndTuning) wizard's quest - Answer **Y** to [populate the NetBox inventory](asset-interaction-analysis.md#NetBoxPopPassive) based on observed network traffic. Autopopulation is **not** recommended: [manual inventory population](asset-interaction-analysis.md#NetBoxPopManual) is the preferred method to create an accurate representation of the intended network design. * **Specify default NetBox site name** - NetBox has the concept of [sites](https://demo.netbox.dev/static/docs/core-functionality/sites-and-racks/); this default site name will be used for NetBox enrichment lookups performed by Logstash and will be associated with traffic Malcolm itself [captures on local interfaces](live-analysis.md#LocalPCAP). -* **Should Malcolm create "catch-all" prefixes for private IP address space?** - - Answer **Y** to automatically create "catch-all" NetBox prefixes for private IP address space (i.e., one each for `10.0.0.0/8`, `172.16.0.0/12`, and `192.168.0.0/16`, respectively). This is not recommended for networks with more than one subnet. +* **Should Malcolm automatically create missing NetBox subnet prefixes based on observed network traffic?** + - Answer **Y** to automatically create NetBox IP prefixes for private address space based on observed traffic. * **Should Malcolm capture live network traffic?** - Malcolm itself can perform [live analysis](live-analysis.md#LocalPCAP) of traffic it sees on another network interface (ideally not the same one used for its management). Answer **no** to this question in installations where Hedgehog Linux will be handling all network traffic capture. If users want Malcolm to observe and capture traffic instead of, or in addition to, a sensor running Hedgehog Linux, they should answer **yes** enable life traffic analysis using default settings, or select **customize** to proceed to answer the following related questions individually. - **Should Malcolm capture live network traffic to PCAP files for analysis with Arkime?** diff --git a/scripts/install.py b/scripts/install.py index 7c19023fe..068970c0e 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -1481,19 +1481,13 @@ def tweak_malcolm_runtime(self, malcolm_install_path): 'Should Malcolm enrich network traffic using NetBox?', default=args.netboxLogstashEnrich, ) - netboxAutoPopulate = ( - netboxEnabled - and InstallerYesOrNo( - 'Should Malcolm automatically populate NetBox inventory based on observed network traffic?', - default=args.netboxAutoPopulate, - ) - and ( - args.acceptDefaultsNonInteractive - or InstallerYesOrNo( - "Autopopulating NetBox's inventory is not recommended. Are you sure?", - default=args.netboxAutoPopulate, - ) - ) + netboxAutoPopulate = netboxEnabled and InstallerYesOrNo( + 'Should Malcolm automatically populate NetBox inventory based on observed network traffic?', + default=args.netboxAutoPopulate, + ) + netboxLogstashAutoSubnets = netboxLogstashEnrich and InstallerYesOrNo( + 'Should Malcolm automatically create missing NetBox subnet prefixes based on observed network traffic?', + default=args.netboxLogstashAutoSubnets, ) netboxSiteName = ( InstallerAskForString( @@ -1505,10 +1499,6 @@ def tweak_malcolm_runtime(self, malcolm_install_path): ) if len(netboxSiteName) == 0: netboxSiteName = 'Malcolm' - netboxLogstashAutoSubnets = netboxLogstashEnrich and InstallerYesOrNo( - 'Should Malcolm automatically create missing NetBox subnet prefixes based on observed network traffic?', - default=args.netboxLogstashAutoSubnets, - ) # input packet capture parameters pcapNetSniff = False From 456e5cfa6210944d571ef681abdbcc6d9c2b1b0e Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Thu, 1 Aug 2024 20:44:02 -0600 Subject: [PATCH 04/47] do netbox-initializers even if we are doing a netbox preload restore first --- netbox/scripts/netbox_init.py | 53 +++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/netbox/scripts/netbox_init.py b/netbox/scripts/netbox_init.py index f1337ca90..c9b64af17 100755 --- a/netbox/scripts/netbox_init.py +++ b/netbox/scripts/netbox_init.py @@ -491,7 +491,7 @@ def main(): except Exception as e: logging.error(f"{type(e).__name__} restoring {os.path.basename(preloadDatabaseFile)}: {e}") - # only proceed to do the regular population/preload if if we didn't preload a database backup, or + # only proceed to do the regular population if if we didn't preload a database backup, or # if we attempted (and failed) but they didn't explicitly specify a backup file if not preloadDatabaseSuccess and (not args.preloadBackupFile): # create connection to netbox API @@ -715,31 +715,36 @@ def main(): except Exception as e: logging.error(f"{type(e).__name__} migrating prefix VRF to prefix description: {e}") - # ###### Netbox-Initializers ################################################################################### - if os.path.isfile(netboxVenvPy) and os.path.isfile(manageScript) and os.path.isdir(args.preloadDir): - try: - with malcolm_utils.pushd(os.path.dirname(manageScript)): - # make a local copy of the YMLs to preload - with tempfile.TemporaryDirectory() as tmpPreloadDir: - copy_tree(args.preloadDir, tmpPreloadDir) - retcode, output = malcolm_utils.run_process( - [ - netboxVenvPy, - os.path.basename(manageScript), - "load_initializer_data", - "--path", - tmpPreloadDir, - ], - logger=logging, - ) - if retcode == 0: - logging.debug(f"netbox-initializers: {retcode} {output}") - else: - logging.error(f"Error processing netbox-initializers: {retcode} {output}") + # ###### Netbox-Initializers ################################################################################### + # We're doing the netbox initializers even if we did a database backup preload, as we always want to + # honor there being something in here. If things already got loaded upas part of the backup preload, + # no big deal, those things will just fail for already existing. + if os.path.isfile(netboxVenvPy) and os.path.isfile(manageScript) and os.path.isdir(args.preloadDir): + try: + with malcolm_utils.pushd(os.path.dirname(manageScript)): + # make a local copy of the YMLs to preload + with tempfile.TemporaryDirectory() as tmpPreloadDir: + copy_tree(args.preloadDir, tmpPreloadDir) + retcode, output = malcolm_utils.run_process( + [ + netboxVenvPy, + os.path.basename(manageScript), + "load_initializer_data", + "--path", + tmpPreloadDir, + ], + logger=logging, + ) + if retcode == 0: + logging.debug(f"netbox-initializers: {retcode} {output}") + else: + logging.error(f"Error processing netbox-initializers: {retcode} {output}") - except Exception as e: - logging.error(f"{type(e).__name__} processing netbox-initializers: {e}") + except Exception as e: + logging.error(f"{type(e).__name__} processing netbox-initializers: {e}") + # this if statement is the same check we did above after the preload backup restore finished + if not preloadDatabaseSuccess and (not args.preloadBackupFile): # ###### Device-Type-Library-Import ########################################################################### if os.path.isdir(args.libraryDir): try: From 3ca65de946d0dc0c5c1350a7f61b95128ab8c2a4 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Thu, 1 Aug 2024 20:47:44 -0600 Subject: [PATCH 05/47] documentation tweak --- docs/asset-interaction-analysis.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/asset-interaction-analysis.md b/docs/asset-interaction-analysis.md index 3ec81e89a..db0caea89 100644 --- a/docs/asset-interaction-analysis.md +++ b/docs/asset-interaction-analysis.md @@ -127,8 +127,6 @@ See [idaholab/Malcolm#134](https://github.com/idaholab/Malcolm/issues/134). YML files in [`./netbox/preload`]({{ site.github.repository_url }}/tree/{{ site.github.build_revision }}/netbox/preload/) under the Malcolm installation directory will be preloaded upon startup using the third-party [netbox-initializers](https://github.com/tobiasge/netbox-initializers) plugin. Examples illustrating the format of these YML files can be found at its [GitHub repository](https://github.com/tobiasge/netbox-initializers/tree/main/src/netbox_initializers/initializers/yaml). -[workflow files] - ## Backup and Restore The NetBox database may be backed up and restored using `./scripts/netbox-backup` and `./scripts/netbox-restore`, respectively. While Malcolm is running, run the following command from within the Malcolm installation directory to backup the entire NetBox database: From 8422826257a77494b91f7be7b90e12b5d91b7d8d Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 5 Aug 2024 12:23:12 -0600 Subject: [PATCH 06/47] Bump Arkime to v5.4.0 --- Dockerfiles/arkime.Dockerfile | 2 +- hedgehog-iso/build.sh | 2 +- hedgehog-iso/config/hooks/normal/0910-sensor-build.hook.chroot | 2 +- hedgehog-raspi/sensor_install.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index 95fd9e16d..d2a6413d2 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -34,7 +34,7 @@ ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONUNBUFFERED 1 ENV ARKIME_DIR "/opt/arkime" -ENV ARKIME_VERSION "5.3.0" +ENV ARKIME_VERSION "5.4.0" ENV ARKIME_DEB_URL "https://github.com/arkime/arkime/releases/download/v${ARKIME_VERSION}/arkime_${ARKIME_VERSION}-1.debian12_XXX.deb" ENV ARKIME_JA4_SO_URL "https://github.com/arkime/arkime/releases/download/v${ARKIME_VERSION}/ja4plus.XXX.so" ENV ARKIME_LOCALELASTICSEARCH no diff --git a/hedgehog-iso/build.sh b/hedgehog-iso/build.sh index 1d9cf6645..998f2cdec 100755 --- a/hedgehog-iso/build.sh +++ b/hedgehog-iso/build.sh @@ -8,7 +8,7 @@ IMAGE_DISTRIBUTION=bookworm BEATS_VER="8.14.3" BEATS_OSS="-oss" -ARKIME_VER="5.3.0" +ARKIME_VER="5.4.0" BUILD_ERROR_CODE=1 diff --git a/hedgehog-iso/config/hooks/normal/0910-sensor-build.hook.chroot b/hedgehog-iso/config/hooks/normal/0910-sensor-build.hook.chroot index 0425f5409..50f9e89af 100755 --- a/hedgehog-iso/config/hooks/normal/0910-sensor-build.hook.chroot +++ b/hedgehog-iso/config/hooks/normal/0910-sensor-build.hook.chroot @@ -64,7 +64,7 @@ rm -Rf zeek* spicy* ### # get Arkime JA4+ plugin -ARKIME_VERSION="5.3.0" +ARKIME_VERSION="5.4.0" curl "${GITHUB_API_CURL_ARGS[@]}" \ -o "/opt/arkime/plugins/ja4plus.${ARCH}.so" \ "https://github.com/arkime/arkime/releases/download/v${ARKIME_VERSION}/ja4plus.${ARCH}.so" diff --git a/hedgehog-raspi/sensor_install.sh b/hedgehog-raspi/sensor_install.sh index dea0d2379..8c73c7c1e 100644 --- a/hedgehog-raspi/sensor_install.sh +++ b/hedgehog-raspi/sensor_install.sh @@ -32,7 +32,7 @@ SHARED_DIR='/opt/buildshared' WORK_DIR="$(mktemp -d -t hedgehog-XXXXXX)" SENSOR_DIR='/opt/sensor' -ARKIME_VERSION="5.3.0" +ARKIME_VERSION="5.4.0" BEATS_VER="8.14.3" BEATS_OSS="-oss" From 051d7477547933f6d03c07e1322a9bfcac1a8a1e Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 5 Aug 2024 12:24:49 -0600 Subject: [PATCH 07/47] bump yq to v4.44.3 --- Dockerfiles/filebeat.Dockerfile | 2 +- Dockerfiles/netbox.Dockerfile | 2 +- Dockerfiles/suricata.Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfiles/filebeat.Dockerfile b/Dockerfiles/filebeat.Dockerfile index e4751d52a..a1c44d7f7 100644 --- a/Dockerfiles/filebeat.Dockerfile +++ b/Dockerfiles/filebeat.Dockerfile @@ -66,7 +66,7 @@ ENV SUPERCRONIC_VERSION "0.2.30" ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-" ENV SUPERCRONIC_CRONTAB "/etc/crontab" -ENV YQ_VERSION "4.44.2" +ENV YQ_VERSION "4.44.3" ENV YQ_URL "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_" ENV EVTX_VERSION "0.8.2" diff --git a/Dockerfiles/netbox.Dockerfile b/Dockerfiles/netbox.Dockerfile index 07939abe8..758cef153 100644 --- a/Dockerfiles/netbox.Dockerfile +++ b/Dockerfiles/netbox.Dockerfile @@ -32,7 +32,7 @@ ENV SUPERCRONIC_CRONTAB "/etc/crontab" ENV NETBOX_INITIALIZERS_VERSION "50d077d" -ENV YQ_VERSION "4.44.2" +ENV YQ_VERSION "4.44.3" ENV YQ_URL "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_" ENV NETBOX_DEVICETYPE_LIBRARY_IMPORT_URL "https://codeload.github.com/netbox-community/Device-Type-Library-Import/tar.gz/develop" diff --git a/Dockerfiles/suricata.Dockerfile b/Dockerfiles/suricata.Dockerfile index 0404b010f..50ae17df7 100644 --- a/Dockerfiles/suricata.Dockerfile +++ b/Dockerfiles/suricata.Dockerfile @@ -37,7 +37,7 @@ ENV SUPERCRONIC_VERSION "0.2.30" ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-" ENV SUPERCRONIC_CRONTAB "/etc/crontab" -ENV YQ_VERSION "4.44.2" +ENV YQ_VERSION "4.44.3" ENV YQ_URL "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_" ENV SURICATA_VERSION_PATTERN "1:7.0.*" From 1d574e777f71f9a92e6f2db00435adc2b9305147 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 5 Aug 2024 12:25:23 -0600 Subject: [PATCH 08/47] bump evtx to v0.8.3 --- Dockerfiles/filebeat.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfiles/filebeat.Dockerfile b/Dockerfiles/filebeat.Dockerfile index a1c44d7f7..e5fd24cf4 100644 --- a/Dockerfiles/filebeat.Dockerfile +++ b/Dockerfiles/filebeat.Dockerfile @@ -69,7 +69,7 @@ ENV SUPERCRONIC_CRONTAB "/etc/crontab" ENV YQ_VERSION "4.44.3" ENV YQ_URL "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_" -ENV EVTX_VERSION "0.8.2" +ENV EVTX_VERSION "0.8.3" ENV EVTX_URL "https://github.com/omerbenamram/evtx/releases/download/v${EVTX_VERSION}/evtx_dump-v${EVTX_VERSION}-XXX-unknown-linux-gnu" USER root From 6beb8ad2cd9aa9cf260ae7601e5cdd90f2c04088 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 7 Aug 2024 07:36:48 -0600 Subject: [PATCH 09/47] Bump Zeek to v7.0.0, idaholab/Malcolm#535 --- Dockerfiles/zeek.Dockerfile | 2 +- hedgehog-raspi/sensor_install.sh | 2 +- shared/bin/zeek-deb-download.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile index 974266849..54f9882ae 100644 --- a/Dockerfiles/zeek.Dockerfile +++ b/Dockerfiles/zeek.Dockerfile @@ -33,7 +33,7 @@ ENV PGROUP "zeeker" ENV PUSER_PRIV_DROP false # for download and install -ARG ZEEK_VERSION=6.2.1-0 +ARG ZEEK_VERSION=7.0.0-0 ENV ZEEK_VERSION $ZEEK_VERSION # put Zeek and Spicy in PATH diff --git a/hedgehog-raspi/sensor_install.sh b/hedgehog-raspi/sensor_install.sh index 8c73c7c1e..6ab11d9f3 100644 --- a/hedgehog-raspi/sensor_install.sh +++ b/hedgehog-raspi/sensor_install.sh @@ -214,7 +214,7 @@ build_zeek_src() { export PYTHONUNBUFFERED=1 zeek_url=https://github.com/zeek/zeek.git - zeek_version=6.2.1 + zeek_version=7.0.0 zeek_release=1 zeek_dir=/opt/zeek # Zeek's build eats a ton of resources; prevent OOM from the killing build process diff --git a/shared/bin/zeek-deb-download.sh b/shared/bin/zeek-deb-download.sh index 18a1f7bbe..d2586b333 100755 --- a/shared/bin/zeek-deb-download.sh +++ b/shared/bin/zeek-deb-download.sh @@ -6,7 +6,7 @@ unset VERBOSE command -v dpkg >/dev/null 2>&1 && ARCH="$(dpkg --print-architecture)" || ARCH=amd64 DISTRO=Debian_12 OUTPUT_DIR=/tmp -ZEEK_VERSION=6.2.1-0 +ZEEK_VERSION=7.0.0-0 while getopts a:d:o:vz: opts; do case ${opts} in From 8f7ee29195018cb11b603d43813bc2111dd69c87 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 12 Aug 2024 13:09:24 -0600 Subject: [PATCH 10/47] bump kubernetes/ingress-nginx to controller-v1.11.1 for example --- kubernetes/vagrant/deploy_ingress_nginx.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/vagrant/deploy_ingress_nginx.sh b/kubernetes/vagrant/deploy_ingress_nginx.sh index 7ba1b0dcc..9963d1742 100755 --- a/kubernetes/vagrant/deploy_ingress_nginx.sh +++ b/kubernetes/vagrant/deploy_ingress_nginx.sh @@ -24,7 +24,7 @@ AWS_EXPOSE_ANNOTATIONS=( "{\"service.beta.kubernetes.io/aws-load-balancer-nlb-target-type\":\"instance\"}" "{\"service.beta.kubernetes.io/aws-load-balancer-scheme\":\"internet-facing\"}" ) -INGRESS_NGINX_CONTROLLER_VERSION=1.8.0 +INGRESS_NGINX_CONTROLLER_VERSION=1.11.1 KUBECONFIG= WORKDIR= DRY_RUN=none From d7aae7bf579f9a27fa12cbec2a28b1e9ed7f3273 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 12 Aug 2024 13:11:07 -0600 Subject: [PATCH 11/47] bump opensearch and opensearch dashboards to v2.16.0 --- Dockerfiles/dashboards.Dockerfile | 10 +++++----- Dockerfiles/opensearch.Dockerfile | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfiles/dashboards.Dockerfile b/Dockerfiles/dashboards.Dockerfile index 4d472d7a0..78a187822 100644 --- a/Dockerfiles/dashboards.Dockerfile +++ b/Dockerfiles/dashboards.Dockerfile @@ -1,6 +1,6 @@ ARG TARGETPLATFORM=linux/amd64 -FROM --platform=${TARGETPLATFORM} opensearchproject/opensearch-dashboards:2.15.0 +FROM --platform=${TARGETPLATFORM} opensearchproject/opensearch-dashboards:2.16.0 LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' @@ -43,10 +43,10 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') # Malcolm manages authentication and encryption via NGINX reverse proxy /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove securityDashboards --allow-root && \ cd /tmp && \ - # unzip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \ - # sed -i "s/2\.13\.0/2\.15\.0/g" opensearch-dashboards/transformVis/opensearch_dashboards.json && \ - # sed -i "s/2\.13\.0/2\.15\.0/g" opensearch-dashboards/transformVis/package.json && \ - # zip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \ + unzip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \ + sed -i "s/2\.15\.0/2\.16\.0/g" opensearch-dashboards/transformVis/opensearch_dashboards.json && \ + sed -i "s/2\.15\.0/2\.16\.0/g" opensearch-dashboards/transformVis/package.json && \ + zip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \ cd /usr/share/opensearch-dashboards/plugins && \ /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin install file:///tmp/transformVis.zip --allow-root && \ rm -rf /tmp/transformVis /tmp/opensearch-dashboards && \ diff --git a/Dockerfiles/opensearch.Dockerfile b/Dockerfiles/opensearch.Dockerfile index a88077e5e..e9ad48fcb 100644 --- a/Dockerfiles/opensearch.Dockerfile +++ b/Dockerfiles/opensearch.Dockerfile @@ -1,6 +1,6 @@ ARG TARGETPLATFORM=linux/amd64 -FROM --platform=${TARGETPLATFORM} opensearchproject/opensearch:2.15.0 +FROM --platform=${TARGETPLATFORM} opensearchproject/opensearch:2.16.0 # Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" From 6405cbd72189708acb4fb2cf62e3853c2a93c5e1 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 12 Aug 2024 13:12:57 -0600 Subject: [PATCH 12/47] bump logstash, filebeat to v8.15.0 --- Dockerfiles/filebeat.Dockerfile | 2 +- Dockerfiles/logstash.Dockerfile | 2 +- api/requirements.txt | 2 +- hedgehog-iso/build.sh | 2 +- hedgehog-raspi/sensor_install.sh | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfiles/filebeat.Dockerfile b/Dockerfiles/filebeat.Dockerfile index e5fd24cf4..2df341307 100644 --- a/Dockerfiles/filebeat.Dockerfile +++ b/Dockerfiles/filebeat.Dockerfile @@ -1,6 +1,6 @@ ARG TARGETPLATFORM=linux/amd64 -FROM --platform=${TARGETPLATFORM} docker.elastic.co/beats/filebeat-oss:8.14.3 +FROM --platform=${TARGETPLATFORM} docker.elastic.co/beats/filebeat-oss:8.15.0 # Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" diff --git a/Dockerfiles/logstash.Dockerfile b/Dockerfiles/logstash.Dockerfile index 7d24431f3..7e630740f 100644 --- a/Dockerfiles/logstash.Dockerfile +++ b/Dockerfiles/logstash.Dockerfile @@ -1,6 +1,6 @@ ARG TARGETPLATFORM=linux/amd64 -FROM --platform=${TARGETPLATFORM} docker.elastic.co/logstash/logstash-oss:8.14.3 +FROM --platform=${TARGETPLATFORM} docker.elastic.co/logstash/logstash-oss:8.15.0 LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' diff --git a/api/requirements.txt b/api/requirements.txt index 61f63958b..ea7e4ed48 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -6,5 +6,5 @@ requests==2.32.0 regex==2022.3.2 dateparser==1.1.1 elasticsearch==8.14.0 -elasticsearch-dsl==8.14.0 +elasticsearch-dsl==8.15.0 psutil==5.9.8 \ No newline at end of file diff --git a/hedgehog-iso/build.sh b/hedgehog-iso/build.sh index 998f2cdec..673592f6d 100755 --- a/hedgehog-iso/build.sh +++ b/hedgehog-iso/build.sh @@ -5,7 +5,7 @@ IMAGE_PUBLISHER=idaholab IMAGE_VERSION=1.0.0 IMAGE_DISTRIBUTION=bookworm -BEATS_VER="8.14.3" +BEATS_VER="8.15.0" BEATS_OSS="-oss" ARKIME_VER="5.4.0" diff --git a/hedgehog-raspi/sensor_install.sh b/hedgehog-raspi/sensor_install.sh index 6ab11d9f3..0388d562b 100644 --- a/hedgehog-raspi/sensor_install.sh +++ b/hedgehog-raspi/sensor_install.sh @@ -34,7 +34,7 @@ SENSOR_DIR='/opt/sensor' ARKIME_VERSION="5.4.0" -BEATS_VER="8.14.3" +BEATS_VER="8.15.0" BEATS_OSS="-oss" # Option to build from sources if desired From 1cf975cfe7287a68a9c0098b505adc4600f4b16a Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 12 Aug 2024 13:13:35 -0600 Subject: [PATCH 13/47] bump fluent-bit to v3.1.5 --- scripts/third-party-logs/fluent-bit-setup.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/third-party-logs/fluent-bit-setup.ps1 b/scripts/third-party-logs/fluent-bit-setup.ps1 index 14a20247b..84b603a01 100644 --- a/scripts/third-party-logs/fluent-bit-setup.ps1 +++ b/scripts/third-party-logs/fluent-bit-setup.ps1 @@ -9,7 +9,7 @@ ############################################################################### $fluent_bit_version = '3.1' -$fluent_bit_full_version = '3.1.4' +$fluent_bit_full_version = '3.1.5' ############################################################################### # select an item from a menu provided in an array From 9418da076156eb2d906c303d9214cfe061e50c8b Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 13 Aug 2024 08:37:04 -0600 Subject: [PATCH 14/47] better health/liveness check for zeek-offline container --- Dockerfiles/zeek.Dockerfile | 8 +++++--- docker-compose-dev.yml | 2 +- docker-compose.yml | 2 +- kubernetes/10-zeek.yml | 4 +--- scripts/third-party-environments/aws/ami/.gitignore | 2 +- zeek/scripts/container_health.sh | 13 +++++++++++++ 6 files changed, 22 insertions(+), 9 deletions(-) create mode 100755 zeek/scripts/container_health.sh diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile index 54f9882ae..0eb6e3341 100644 --- a/Dockerfiles/zeek.Dockerfile +++ b/Dockerfiles/zeek.Dockerfile @@ -144,6 +144,10 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') # add configuration and scripts COPY --chmod=755 shared/bin/docker-uid-gid-setup.sh /usr/local/bin/ COPY --chmod=755 shared/bin/service_check_passthrough.sh /usr/local/bin/ +COPY --chmod=755 shared/bin/zeek_intel_setup.sh ${ZEEK_DIR}/bin/ +COPY --chmod=755 shared/bin/zeekdeploy.sh ${ZEEK_DIR}/bin/ +COPY --chmod=755 zeek/scripts/container_health.sh /usr/local/bin/ +COPY --chmod=755 zeek/scripts/docker_entrypoint.sh /usr/local/bin/ COPY --from=ghcr.io/mmguero-dev/gostatic --chmod=755 /goStatic /usr/bin/goStatic ADD shared/bin/pcap_processor.py /usr/local/bin/ ADD shared/bin/pcap_utils.py /usr/local/bin/ @@ -153,9 +157,7 @@ ADD shared/pcaps /tmp/pcaps ADD zeek/supervisord.conf /etc/supervisord.conf ADD zeek/config/*.zeek ${ZEEK_DIR}/share/zeek/site/ ADD zeek/config/*.txt ${ZEEK_DIR}/share/zeek/site/ -ADD zeek/scripts/docker_entrypoint.sh /usr/local/bin/ -ADD shared/bin/zeek_intel_setup.sh ${ZEEK_DIR}/bin/ -ADD shared/bin/zeekdeploy.sh ${ZEEK_DIR}/bin/ + RUN groupadd --gid ${DEFAULT_GID} ${PUSER} && \ useradd -M --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} --home /nonexistant ${PUSER} && \ diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index cb0706bd9..15533c5f1 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -591,7 +591,7 @@ services: target: /opt/zeek/share/zeek/site/local.zeek read_only: true healthcheck: - test: ["CMD", "supervisorctl", "status", "pcap-zeek"] + test: ["CMD", "/usr/local/bin/container_health.sh"] interval: 30s timeout: 15s retries: 3 diff --git a/docker-compose.yml b/docker-compose.yml index 68d4d5ad5..67b3140aa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -501,7 +501,7 @@ services: target: /opt/zeek/share/zeek/site/custom read_only: true healthcheck: - test: ["CMD", "supervisorctl", "status", "pcap-zeek"] + test: ["CMD", "/usr/local/bin/container_health.sh"] interval: 30s timeout: 15s retries: 3 diff --git a/kubernetes/10-zeek.yml b/kubernetes/10-zeek.yml index b1fe88b53..de874a2ae 100644 --- a/kubernetes/10-zeek.yml +++ b/kubernetes/10-zeek.yml @@ -36,9 +36,7 @@ spec: livenessProbe: exec: command: - - supervisorctl - - status - - pcap-zeek + - /usr/local/bin/container_health.sh initialDelaySeconds: 60 periodSeconds: 30 timeoutSeconds: 15 diff --git a/scripts/third-party-environments/aws/ami/.gitignore b/scripts/third-party-environments/aws/ami/.gitignore index 3aa158c59..0ab735314 100644 --- a/scripts/third-party-environments/aws/ami/.gitignore +++ b/scripts/third-party-environments/aws/ami/.gitignore @@ -1 +1 @@ -packer_vars.json +packer_vars*.json diff --git a/zeek/scripts/container_health.sh b/zeek/scripts/container_health.sh new file mode 100755 index 000000000..2fa834f4c --- /dev/null +++ b/zeek/scripts/container_health.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# the zeek container is considered "healthy" if the intel population script OR the (live|pcap)-zeek process is running + +if [[ "${ZEEK_LIVE_CAPTURE:-false}" == "true" ]]; then + supervisorctl status live-zeek >/dev/null 2>&1 +else + if [[ "${ZEEK_INTEL_REFRESH_ON_ENTRYPOINT:-false}" == "true" ]]; then + ( ps a 2>/dev/null | grep -q '[z]eek_intel_from_threat_feed.py' ) || supervisorctl status pcap-zeek >/dev/null 2>&1 + else + supervisorctl status pcap-zeek >/dev/null 2>&1 + fi +fi From cd305bb9a7d9dc39b1cb48aeb2020d9c9a6fa3d6 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 13 Aug 2024 11:17:45 -0600 Subject: [PATCH 15/47] for idaholab/Malcolm#539, some containers need resource request specified for Kubernetes. parse out opensearch and logstash JAVA_OPTS and create kubernetes resource requests for memory basedon those values to ensure those containers don't start on pods with insufficient memory --- scripts/malcolm_common.py | 26 ++++ scripts/malcolm_kubernetes.py | 263 ++++++++++++++++++++++++---------- 2 files changed, 215 insertions(+), 74 deletions(-) diff --git a/scripts/malcolm_common.py b/scripts/malcolm_common.py index 5bcd82aee..fdf001cfc 100644 --- a/scripts/malcolm_common.py +++ b/scripts/malcolm_common.py @@ -7,6 +7,7 @@ import importlib import json import os +import math import platform import re import string @@ -172,6 +173,31 @@ def LocalPathForContainerBindMount(service, dockerComposeContents, containerPath return localPath +################################################################################################## +def GetMemMegabytesFromJavaOptsLine(val): + resultStr = None + resultMB = 0 + for opt in ('Xmx', 'Xms'): + if resultStr is not None: + break + if match := re.search(fr'-{opt}(\d+[kmg])', val, re.IGNORECASE): + resultStr = match.group(1) + if resultStr is not None: + value = int(resultStr[:-1]) + unit = resultStr[-1].lower() + if unit == 'g': + resultMB = value * 1024 + elif unit == 'm': + resultMB = value + elif unit == 'k': + resultMB = math.ceil(value / 1024) + else: + resultMB = 0 + if resultMB < 1: + resultMB = 0 + return resultMB + + ################################################################################################## def GetUidGidFromEnv(configDir=None): configDirToCheck = configDir if configDir and os.path.isdir(configDir) else os.path.join(MalcolmPath, 'config') diff --git a/scripts/malcolm_kubernetes.py b/scripts/malcolm_kubernetes.py index 43e9d38a5..eb1121047 100644 --- a/scripts/malcolm_kubernetes.py +++ b/scripts/malcolm_kubernetes.py @@ -8,6 +8,7 @@ import os from concurrent.futures import ThreadPoolExecutor, as_completed +from contextlib import nullcontext from collections import defaultdict from itertools import chain from io import StringIO @@ -15,11 +16,14 @@ from malcolm_common import ( DotEnvDynamic, + GetMemMegabytesFromJavaOptsLine, KubernetesDynamic, MalcolmPath, + NullRepresenter, PROFILE_HEDGEHOG, PROFILE_MALCOLM, YAMLDynamic, + YAML_VERSION, ) from malcolm_utils import ( deep_get, @@ -29,6 +33,7 @@ remove_suffix, tablify, LoadStrIfJson, + temporary_filename, val2bool, ) @@ -233,6 +238,11 @@ 'zeek-offline', ] +CONTAINER_JAVA_OPTS_VARS = { + 'opensearch': 'OPENSEARCH_JAVA_OPTS', + 'logstash': 'LS_JAVA_OPTS', +} + ################################################################################################### def _nanocore_to_millicore(n): @@ -694,7 +704,7 @@ def DeleteNamespace(namespace, deleteRetPerVol=False): return results_dict -def StartMalcolm(namespace, malcolmPath, configPath, profile=PROFILE_MALCOLM): +def StartMalcolm(namespace, malcolmPath, configPath, profile=PROFILE_MALCOLM, dryrun=False): if not namespace: namespace = 'malcolm' @@ -710,21 +720,23 @@ def StartMalcolm(namespace, malcolmPath, configPath, profile=PROFILE_MALCOLM): and (apiClient := kubeImported.client.ApiClient()) ): # create the namespace - try: - results_dict['create_namespace']['result'] = client.create_namespace( - kubeImported.client.V1Namespace(metadata=kubeImported.client.V1ObjectMeta(name=namespace)) - ).metadata - except kubeImported.client.rest.ApiException as x: - if x.status != 409: - results_dict['create_namespace']['error'] = LoadStrIfJson(str(x)) - if not results_dict['create_namespace']['error']: - results_dict['create_namespace']['error'] = str(x) + if not dryrun: + try: + results_dict['create_namespace']['result'] = client.create_namespace( + kubeImported.client.V1Namespace(metadata=kubeImported.client.V1ObjectMeta(name=namespace)) + ).metadata + except kubeImported.client.rest.ApiException as x: + if x.status != 409: + results_dict['create_namespace']['error'] = LoadStrIfJson(str(x)) + if not results_dict['create_namespace']['error']: + results_dict['create_namespace']['error'] = str(x) # create configmaps from files # files in nested directories will be created with a name like foo_MALDIR_bar_MALDIR_baz.txt # and then renamed to foo/bar/baz.txt during container start up by docker-uid-gid-setup.sh - results_dict['create_namespaced_config_map']['result'] = dict() - results_dict['create_namespaced_secret']['result'] = dict() + if not dryrun: + results_dict['create_namespaced_config_map']['result'] = dict() + results_dict['create_namespaced_secret']['result'] = dict() for configMapName, configMapFiles in MALCOLM_CONFIGMAPS.items(): for isSecret in (True, False): resultsEntry = 'create_namespaced_secret' if isSecret else 'create_namespaced_config_map' @@ -763,24 +775,27 @@ def StartMalcolm(namespace, malcolmPath, configPath, profile=PROFILE_MALCOLM): name=configMapName, namespace=namespace, ) - if isSecret: - results_dict[resultsEntry]['result'][configMapName] = client.create_namespaced_secret( - namespace=namespace, - body=kubeImported.client.V1Secret( - metadata=metadata, - string_data=dataMap if dataMap else {}, - data=binaryDataMap if binaryDataMap else {}, - ), - ).metadata - else: - results_dict[resultsEntry]['result'][configMapName] = client.create_namespaced_config_map( - namespace=namespace, - body=kubeImported.client.V1ConfigMap( - metadata=metadata, - data=dataMap if dataMap else {}, - binary_data=binaryDataMap if binaryDataMap else {}, - ), - ).metadata + if not dryrun: + if isSecret: + results_dict[resultsEntry]['result'][configMapName] = client.create_namespaced_secret( + namespace=namespace, + body=kubeImported.client.V1Secret( + metadata=metadata, + string_data=dataMap if dataMap else {}, + data=binaryDataMap if binaryDataMap else {}, + ), + ).metadata + else: + results_dict[resultsEntry]['result'][configMapName] = ( + client.create_namespaced_config_map( + namespace=namespace, + body=kubeImported.client.V1ConfigMap( + metadata=metadata, + data=dataMap if dataMap else {}, + binary_data=binaryDataMap if binaryDataMap else {}, + ), + ).metadata + ) except kubeImported.client.rest.ApiException as x: if x.status != 409: if 'error' not in results_dict[resultsEntry]: @@ -790,8 +805,10 @@ def StartMalcolm(namespace, malcolmPath, configPath, profile=PROFILE_MALCOLM): results_dict[resultsEntry]['error'][os.path.basename(configMapName)] = str(x) # create configmaps (or secrets, given a K8S_SECRET key) from .env files - results_dict['create_namespaced_config_map_from_env_file']['result'] = dict() - results_dict['create_namespaced_secret_from_env_file']['result'] = dict() + namedEnvs = defaultdict(dict) + if not dryrun: + results_dict['create_namespaced_config_map_from_env_file']['result'] = dict() + results_dict['create_namespaced_secret_from_env_file']['result'] = dict() for envFileName in glob.iglob(os.path.join(configPath, '*.env'), recursive=False): if os.path.isfile(envFileName): try: @@ -800,24 +817,26 @@ def StartMalcolm(namespace, malcolmPath, configPath, profile=PROFILE_MALCOLM): metadata = kubeImported.client.V1ObjectMeta( name=remove_suffix(os.path.basename(envFileName), '.env') + '-env' ) - if isSecret: - resultsEntry = 'create_namespaced_secret_from_env_file' - results_dict[resultsEntry]['result'][metadata.name] = client.create_namespaced_secret( - namespace=namespace, - body=kubeImported.client.V1Secret( - metadata=metadata, - string_data=values if values else {}, - ), - ).metadata - else: - resultsEntry = 'create_namespaced_config_map_from_env_file' - results_dict[resultsEntry]['result'][metadata.name] = client.create_namespaced_config_map( - namespace=namespace, - body=kubeImported.client.V1ConfigMap( - metadata=metadata, - data=values if values else {}, - ), - ).metadata + namedEnvs[metadata.name] = values if values else {} + if not dryrun: + if isSecret: + resultsEntry = 'create_namespaced_secret_from_env_file' + results_dict[resultsEntry]['result'][metadata.name] = client.create_namespaced_secret( + namespace=namespace, + body=kubeImported.client.V1Secret( + metadata=metadata, + string_data=values if values else {}, + ), + ).metadata + else: + resultsEntry = 'create_namespaced_config_map_from_env_file' + results_dict[resultsEntry]['result'][metadata.name] = client.create_namespaced_config_map( + namespace=namespace, + body=kubeImported.client.V1ConfigMap( + metadata=metadata, + data=values if values else {}, + ), + ).metadata except kubeImported.client.rest.ApiException as x: if x.status != 409: @@ -828,7 +847,8 @@ def StartMalcolm(namespace, malcolmPath, configPath, profile=PROFILE_MALCOLM): results_dict[resultsEntry]['error'][os.path.basename(envFileName)] = str(x) # apply manifests - results_dict['create_from_yaml']['result'] = dict() + if not dryrun: + results_dict['create_from_yaml']['result'] = dict() yamlFiles = sorted( list( chain( @@ -842,6 +862,7 @@ def StartMalcolm(namespace, malcolmPath, configPath, profile=PROFILE_MALCOLM): for yamlName in yamlFiles: # check to make sure the container in this YAML file belongs to this profile containerBelongsInProfile = True + manYamlFileContents = None with open(yamlName, 'r') as manYamlFile: if manYamlFileContents := list(yamlImported.YAML(typ='safe', pure=True).load_all(manYamlFile)): for doc in manYamlFileContents: @@ -856,28 +877,122 @@ def StartMalcolm(namespace, malcolmPath, configPath, profile=PROFILE_MALCOLM): # apply the manifests in this YAML file, otherwise skip it if containerBelongsInProfile: - try: - results_dict['create_from_yaml']['result'][os.path.basename(yamlName)] = ( - kubeImported.utils.create_from_yaml( - apiClient, - yamlName, - namespace=namespace, - ) - ) - except kubeImported.client.rest.ApiException as x: - if x.status != 409: - if 'error' not in results_dict['create_from_yaml']: - results_dict['create_from_yaml']['error'] = dict() - results_dict['create_from_yaml']['error'][os.path.basename(yamlName)] = LoadStrIfJson(str(x)) - if not results_dict['create_from_yaml']['error'][os.path.basename(yamlName)]: - results_dict['create_from_yaml']['error'][os.path.basename(yamlName)] = str(x) - except kubeImported.utils.FailToCreateError as fe: - if [exc for exc in fe.api_exceptions if exc.status != 409]: - if 'error' not in results_dict['create_from_yaml']: - results_dict['create_from_yaml']['error'] = dict() - results_dict['create_from_yaml']['error'][os.path.basename(yamlName)] = LoadStrIfJson(str(fe)) - if not results_dict['create_from_yaml']['error'][os.path.basename(yamlName)]: - results_dict['create_from_yaml']['error'][os.path.basename(yamlName)] = str(fe) + + # Some containers need to have resource requests created for them on the fly (idaholab/Malcolm#539). + # For now the only ones I'm doing this for are ones that have JAVA_OPTS specified (see CONTAINER_JAVA_OPTS_VARS) + # which we retrieve from the container's environment variables we created earlier as configMapRefs. + modified = False + if manYamlFileContents: + for docIdx, doc in enumerate(manYamlFileContents): + if ( + ('spec' in manYamlFileContents[docIdx]) + and ('template' in manYamlFileContents[docIdx]['spec']) + and ('spec' in manYamlFileContents[docIdx]['spec']['template']) + and ('containers' in manYamlFileContents[docIdx]['spec']['template']['spec']) + ): + # loop over each container defined in this document (by index since we're modifying in-place) + for containerIdx, container in enumerate( + manYamlFileContents[docIdx]['spec']['template']['spec']['containers'] + ): + # we're only concerned about containters we've defined by name in CONTAINER_JAVA_OPTS_VARS + containerName = remove_suffix( + manYamlFileContents[docIdx]['spec']['template']['spec']['containers'][ + containerIdx + ].get('name', ''), + '-container', + ) + if containerName in CONTAINER_JAVA_OPTS_VARS: + # load up a list of environment variable sets (configMapRefs) defined in the container's envFrom + containerEnvs = {} + if ( + 'envFrom' + in manYamlFileContents[docIdx]['spec']['template']['spec']['containers'][ + containerIdx + ] + ): + for env in manYamlFileContents[docIdx]['spec']['template']['spec'][ + 'containers' + ][containerIdx]['envFrom']: + if ('configMapRef' in env) and ('name' in env['configMapRef']): + containerEnvs.update(namedEnvs.get(env['configMapRef']['name'], {})) + # proceed if the environment variable in CONTAINER_JAVA_OPTS_VARS for this container + # is defined in this container's runtime environment variables + if CONTAINER_JAVA_OPTS_VARS[containerName] in containerEnvs: + # calculate mebibytes for resources request from JAVA_OPTS line and proceed if it's > 0 + if requestMib := GetMemMegabytesFromJavaOptsLine( + containerEnvs.get(CONTAINER_JAVA_OPTS_VARS[containerName], '') + ): + if ( + 'resources' + not in manYamlFileContents[docIdx]['spec']['template']['spec'][ + 'containers' + ][containerIdx] + ): + manYamlFileContents[docIdx]['spec']['template']['spec']['containers'][ + containerIdx + ]['resources'] = {} + if ( + 'requests' + not in manYamlFileContents[docIdx]['spec']['template']['spec'][ + 'containers' + ][containerIdx]['resources'] + ): + manYamlFileContents[docIdx]['spec']['template']['spec']['containers'][ + containerIdx + ]['resources']['requests'] = {} + if ( + 'memory' + not in manYamlFileContents[docIdx]['spec']['template']['spec'][ + 'containers' + ][containerIdx]['resources']['requests'] + ): + manYamlFileContents[docIdx]['spec']['template']['spec']['containers'][ + containerIdx + ]['resources']['requests']['memory'] = f'{requestMib}Mi' + modified = True + + # if we added a resource request, write out the modified YAML to a temporary file + with temporary_filename(suffix='.yml') if modified else nullcontext() as tmpYmlFileName: + if modified: + with open(tmpYmlFileName, 'w') as tmpYmlFile: + outYaml = yamlImported.YAML(typ='rt') + outYaml.preserve_quotes = True + outYaml.allow_duplicate_keys = True + outYaml.representer.ignore_aliases = lambda *args: True + outYaml.representer.add_representer(type(None), NullRepresenter()) + outYaml.boolean_representation = ['false', 'true'] + outYaml.version = YAML_VERSION + outYaml.width = 4096 + outYaml.dump_all(manYamlFileContents, tmpYmlFile) + + if not dryrun: + try: + # load from the temporary file if we made modifications, otherwise load from the original + results_dict['create_from_yaml']['result'][os.path.basename(yamlName)] = ( + kubeImported.utils.create_from_yaml( + apiClient, + tmpYmlFileName if modified else yamlName, + namespace=namespace, + ) + ) + except kubeImported.client.rest.ApiException as x: + if x.status != 409: + if 'error' not in results_dict['create_from_yaml']: + results_dict['create_from_yaml']['error'] = dict() + results_dict['create_from_yaml']['error'][os.path.basename(yamlName)] = LoadStrIfJson( + str(x) + ) + if not results_dict['create_from_yaml']['error'][os.path.basename(yamlName)]: + results_dict['create_from_yaml']['error'][os.path.basename(yamlName)] = str(x) + except kubeImported.utils.FailToCreateError as fe: + if [exc for exc in fe.api_exceptions if exc.status != 409]: + if 'error' not in results_dict['create_from_yaml']: + results_dict['create_from_yaml']['error'] = dict() + results_dict['create_from_yaml']['error'][os.path.basename(yamlName)] = LoadStrIfJson( + str(fe) + ) + if not results_dict['create_from_yaml']['error'][os.path.basename(yamlName)]: + results_dict['create_from_yaml']['error'][os.path.basename(yamlName)] = str(fe) return results_dict @@ -899,7 +1014,7 @@ def CheckPersistentStorageDefs(namespace, malcolmPath, profile=PROFILE_MALCOLM): ) for yamlName in yamlFiles: with open(yamlName, 'r') as cf: - allYamlContents.extend(list(yamlImported.YAML(typ='safe', pure=True).safe_load_all(cf))) + allYamlContents.extend(list(yamlImported.YAML(typ='safe', pure=True).load_all(cf))) for name, kind in REQUIRED_VOLUME_OBJECTS[profile].items(): for doc in allYamlContents: if ( From 761d811d0029b3e2270b45091e11e9785e1e257c Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 13 Aug 2024 11:24:45 -0600 Subject: [PATCH 16/47] bump elasticsearch-py to 8.15.0 --- api/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/requirements.txt b/api/requirements.txt index ea7e4ed48..516642bfa 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -5,6 +5,6 @@ opensearch-py==2.6.0 requests==2.32.0 regex==2022.3.2 dateparser==1.1.1 -elasticsearch==8.14.0 +elasticsearch==8.15.0 elasticsearch-dsl==8.15.0 psutil==5.9.8 \ No newline at end of file From 1fa2364b726995afcd7f4507724cbb2b73b6f4af Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 13 Aug 2024 11:44:22 -0600 Subject: [PATCH 17/47] fix idaholab/Malcolm#540, kubernetes manifest for netbox refers to netbox-netmap-json configmap which no longer exists --- kubernetes/18-netbox.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/kubernetes/18-netbox.yml b/kubernetes/18-netbox.yml index cdfa0d22e..03fb934d7 100644 --- a/kubernetes/18-netbox.yml +++ b/kubernetes/18-netbox.yml @@ -77,8 +77,6 @@ spec: volumeMounts: - mountPath: /var/local/ca-trust/configmap name: netbox-var-local-catrust-volume - - mountPath: /usr/local/share/configmap - name: netbox-netmap-json-volume - mountPath: /etc/netbox/config/configmap name: netbox-config-volume - mountPath: /opt/netbox-preload/configmap @@ -105,9 +103,6 @@ spec: - name: netbox-var-local-catrust-volume configMap: name: var-local-catrust - - name: netbox-netmap-json-volume - configMap: - name: netbox-netmap-json - name: netbox-config-volume configMap: name: netbox-config From f83e88b206cf961e3f634e350271aacae8957bbd Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 14 Aug 2024 07:48:32 -0600 Subject: [PATCH 18/47] idaholab/Malcolm#529, in ISO installer, prompt to format other drives for artifact storage rather than just doing it automatically --- shared/bin/preseed_late_user_config.sh | 27 ++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/shared/bin/preseed_late_user_config.sh b/shared/bin/preseed_late_user_config.sh index 439c01a72..61b81a022 100755 --- a/shared/bin/preseed_late_user_config.sh +++ b/shared/bin/preseed_late_user_config.sh @@ -63,11 +63,38 @@ Description: Template: malcolm/ssh_password_auth_title Type: text Description: SSH Password Authentication + +Template: malcolm/format_large_storage +Type: boolean +Default: false +Description: + Format non-OS drive(s) for artifact storage? + +Template: malcolm/format_large_storage_title +Type: text +Description: Format Non-OS Drive(s)? !EOF! # load template db_x_loadtemplatefile /tmp/malcolm.template malcolm +# set title +db_settitle malcolm/format_large_storage_title + +# prompt +db_input critical malcolm/format_large_storage +db_go + +# get answer to $RET +db_get malcolm/format_large_storage + +if [ "$RET" = false ]; then + # capture_storage_format file was already created in preseed, remove it + rm -f /etc/capture_storage_format* +fi + +echo "malcolm/format_large_storage=$RET" >> /tmp/malcolm.answer + # set title db_settitle malcolm/disable_ipv6_title From 8133de608c610d88112fffc8a3ed0aa9ecc0514a Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 14 Aug 2024 07:50:52 -0600 Subject: [PATCH 19/47] idaholab/Malcolm#529, documentation changes --- docs/hedgehog-installation.md | 1 + docs/malcolm-hedgehog-e2e-iso-install.md | 2 ++ docs/malcolm-iso.md | 1 + 3 files changed, 4 insertions(+) diff --git a/docs/hedgehog-installation.md b/docs/hedgehog-installation.md index 1ca0cfd4f..659048803 100644 --- a/docs/hedgehog-installation.md +++ b/docs/hedgehog-installation.md @@ -33,6 +33,7 @@ After the passwords have been entered, the installer will proceed to format the At the end of the installation process, the installer will prompt users with few self-explanatory yes/no questions: +* **Format non-OS drive(s) for artifact storage?** * **Disable IPv6?** * **Automatically login to the GUI session?** * **Should the GUI session be locked due to inactivity?** diff --git a/docs/malcolm-hedgehog-e2e-iso-install.md b/docs/malcolm-hedgehog-e2e-iso-install.md index bda9cb861..6942fa743 100644 --- a/docs/malcolm-hedgehog-e2e-iso-install.md +++ b/docs/malcolm-hedgehog-e2e-iso-install.md @@ -110,6 +110,7 @@ After the passwords have been entered, the installer will proceed to format the At the end of the installation process, users will be prompted with a few self-explanatory yes/no questions: +* **Format non-OS drive(s) for artifact storage?** * **Disable IPv6?** * **Automatically login to the GUI session?** * **Should the GUI session be locked due to inactivity?** @@ -364,6 +365,7 @@ The installer will ask for a few pieces of information prior to installing Hedge At the end of the installation process, users will be prompted with a few self-explanatory yes/no questions: +* **Format non-OS drive(s) for artifact storage?** * **Disable IPv6?** * **Automatically login to the GUI session?** * **Should the GUI session be locked due to inactivity?** diff --git a/docs/malcolm-iso.md b/docs/malcolm-iso.md index 3edfa5033..e93d2a0eb 100644 --- a/docs/malcolm-iso.md +++ b/docs/malcolm-iso.md @@ -71,6 +71,7 @@ The installer will ask for several pieces of information prior to installing the At the end of the installation process, users will be prompted with the following self-explanatory yes/no questions: +* **Format non-OS drive(s) for artifact storage?** * **Disable IPv6?** * **Automatically login to the GUI session?** * **Should the GUI session be locked due to inactivity?** From 29ae24086c2fde478403b564052ba387eb566b22 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 14 Aug 2024 14:42:05 -0600 Subject: [PATCH 20/47] idaholab/Malcolm#530, work in progress on netbox plugin installation --- .dockerignore | 2 + Dockerfiles/netbox.Dockerfile | 1 + docker-compose-dev.yml | 8 +- docker-compose.yml | 8 +- docs/contributing-local-modifications.md | 6 + docs/custom-rules.md | 7 +- malcolm-iso/build.sh | 1 + netbox/custom-plugins/requirements/.gitignore | 3 + netbox/scripts/netbox_init.py | 282 +++++++++++++++--- netbox/supervisord.conf | 1 + scripts/malcolm_appliance_packager.sh | 1 + scripts/malcolm_utils.py | 6 +- 12 files changed, 281 insertions(+), 45 deletions(-) create mode 100644 netbox/custom-plugins/requirements/.gitignore diff --git a/.dockerignore b/.dockerignore index 4172fe569..b8dc8dc67 100644 --- a/.dockerignore +++ b/.dockerignore @@ -42,3 +42,5 @@ suricata-logs netbox/netbox/media netbox/netbox/postgres netbox/netbox/redis +netbox/custom-plugins +zeek/custom \ No newline at end of file diff --git a/Dockerfiles/netbox.Dockerfile b/Dockerfiles/netbox.Dockerfile index 758cef153..01e93ba27 100644 --- a/Dockerfiles/netbox.Dockerfile +++ b/Dockerfiles/netbox.Dockerfile @@ -76,6 +76,7 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') procps \ psmisc \ python3-dev \ + ripgrep \ rsync \ supervisor \ tini && \ diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 15533c5f1..a8989ea3e 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -1105,7 +1105,7 @@ services: bind: create_host_path: false source: ./netbox/config - target: /etc/netbox/config + target: /etc/netbox/config/configmap read_only: true - type: bind bind: @@ -1118,6 +1118,12 @@ services: source: ./netbox/preload target: /opt/netbox-preload/configmap read_only: true + - type: bind + bind: + create_host_path: false + source: ./netbox/custom-plugins + target: /opt/netbox-custom-plugins + read_only: true healthcheck: test: ["CMD", "curl", "--silent", "http://localhost:8080/netbox/api/"] interval: 60s diff --git a/docker-compose.yml b/docker-compose.yml index 67b3140aa..705f3587a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -979,7 +979,7 @@ services: bind: create_host_path: false source: ./netbox/config - target: /etc/netbox/config + target: /etc/netbox/config/configmap read_only: true - type: bind bind: @@ -992,6 +992,12 @@ services: source: ./netbox/preload target: /opt/netbox-preload/configmap read_only: true + - type: bind + bind: + create_host_path: false + source: ./netbox/custom-plugins + target: /opt/netbox-custom-plugins + read_only: true healthcheck: test: ["CMD", "curl", "--silent", "http://localhost:8080/netbox/api/"] interval: 60s diff --git a/docs/contributing-local-modifications.md b/docs/contributing-local-modifications.md index c58be6d16..2cd99546a 100644 --- a/docs/contributing-local-modifications.md +++ b/docs/contributing-local-modifications.md @@ -463,6 +463,12 @@ services: source: ./netbox/preload target: /opt/netbox-preload/configmap read_only: true + - type: bind + bind: + create_host_path: false + source: ./netbox/custom-plugins + target: /opt/netbox-custom-plugins + read_only: true netbox-postgres: volumes: - type: bind diff --git a/docs/custom-rules.md b/docs/custom-rules.md index 2a69b77f0..64f62ed4c 100644 --- a/docs/custom-rules.md +++ b/docs/custom-rules.md @@ -1,8 +1,9 @@ -# Custom Rules and Scripts +# Custom Rules, Scripts and Plugins * [Suricata](#Suricata) * [Zeek](#Zeek) * [YARA](#YARA) +* [NetBox](#NetBox) * [Other Customizations](#Other) Much of Malcolm's behavior can be adjusted through [environment variable files](malcolm-config.md#MalcolmConfigEnvVars). However, some components allow further customization through the use of custom scripts, configuration files, and rules. @@ -73,6 +74,10 @@ docker compose exec file-monitor supervisorctl restart yara If the `EXTRACTED_FILE_YARA_CUSTOM_ONLY` [environment variable](malcolm-config.md#MalcolmConfigEnvVars) is set to `true`, Malcolm will bypass the default Yara rulesets ([Neo23x0/signature-base](https://github.com/Neo23x0/signature-base), [reversinglabs/reversinglabs-yara-rules](https://github.com/reversinglabs/reversinglabs-yara-rules), and [bartblaze/Yara-rules](https://github.com/bartblaze/Yara-rules)) and use only user-defined rules in `./yara/rules`. +## NetBox + +TODO documentation + ## Other Customizations There are other areas of Malcolm that can be modified and customized to fit users' needs. Please see these other sections of the documentation for more information. diff --git a/malcolm-iso/build.sh b/malcolm-iso/build.sh index dc325fcb2..00954d792 100755 --- a/malcolm-iso/build.sh +++ b/malcolm-iso/build.sh @@ -100,6 +100,7 @@ if [ -d "$WORKDIR" ]; then mkdir -p "$MALCOLM_DEST_DIR/htadmin/" mkdir -p "$MALCOLM_DEST_DIR/logstash/certs/" mkdir -p "$MALCOLM_DEST_DIR/logstash/maps/" + mkdir -p "$MALCOLM_DEST_DIR/netbox/custom-plugins/requirements/" mkdir -p "$MALCOLM_DEST_DIR/netbox/media/" mkdir -p "$MALCOLM_DEST_DIR/netbox/postgres/" mkdir -p "$MALCOLM_DEST_DIR/netbox/redis/" diff --git a/netbox/custom-plugins/requirements/.gitignore b/netbox/custom-plugins/requirements/.gitignore new file mode 100644 index 000000000..a5baada18 --- /dev/null +++ b/netbox/custom-plugins/requirements/.gitignore @@ -0,0 +1,3 @@ +* +!.gitignore + diff --git a/netbox/scripts/netbox_init.py b/netbox/scripts/netbox_init.py index c9b64af17..6b836df40 100755 --- a/netbox/scripts/netbox_init.py +++ b/netbox/scripts/netbox_init.py @@ -4,6 +4,7 @@ # Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. import argparse +import ast import glob import gzip import ipaddress @@ -26,6 +27,7 @@ from collections.abc import Iterable from distutils.dir_util import copy_tree from datetime import datetime +from packaging.version import Version from slugify import slugify ################################################################################################### @@ -36,45 +38,6 @@ ################################################################################################### -def get_iterable(x): - if isinstance(x, Iterable) and not isinstance(x, str): - return x - else: - return (x,) - - -def is_ip_address(x): - try: - ipaddress.ip_address(x) - return True - except Exception: - return False - - -def is_ip_v4_address(x): - try: - ipaddress.IPv4Address(x) - return True - except Exception: - return False - - -def is_ip_v6_address(x): - try: - ipaddress.IPv6Address(x) - return True - except Exception: - return False - - -def is_ip_network(x): - try: - ipaddress.ip_network(x) - return True - except Exception: - return False - - def min_hash_value_by_value(x): return next( iter(list({k: v for k, v in sorted(x.items(), key=lambda item: item[1])}.values())), @@ -105,6 +68,104 @@ def max_hash_value_by_key(x): return last +def GetInstalledPackages(venvPy): + packagesInstalled = {} + cmd = [ + venvPy, + "-m", + "pip", + "--no-color", + "--no-input", + "--disable-pip-version-check", + "list", + "--local", + "--format", + "json", + "--verbose", + ] + err, results = malcolm_utils.run_process(cmd, stderr=False, logger=logging) + if (err == 0) and results and (len(results) > 0): + try: + packagesInstalled = {item['name']: item for item in malcolm_utils.LoadStrIfJson(results[0])} + except Exception as e: + logging.error(f"{type(e).__name__} getting list of installed Python packages: {e}") + + return packagesInstalled + + +def InstallPackageDirIfNeeded( + packageDir, + venvPy, + preinstalledPackagesDict={}, +): + installResult = False + + # First do a "dry run" install to determine what would happen. The report from this will + # help us determine if the package actually needs installed or not, as pip always treats + # installations from local directories as "new installs" and would uninstall/reinstall + # no matter what, which we want to avoid if we don't need it. + pluginNeedsInstall = False + with malcolm_utils.temporary_filename(suffix='.json') as dryRunInstallReportFileName: + cmd = [ + venvPy, + "-m", + "pip", + "--no-color", + "--no-input", + "--disable-pip-version-check", + "install", + "--upgrade", + "--dry-run", + "--progress-bar", + "off", + "--report", + dryRunInstallReportFileName, + packageDir, + ] + err, results = malcolm_utils.run_process(cmd, logger=logging) + if (err == 0) and os.path.isfile(dryRunInstallReportFileName): + with open(dryRunInstallReportFileName, 'r') as f: + dryRunReport = malcolm_utils.LoadFileIfJson(f) + wouldInstallInfo = { + malcolm_utils.deep_get(installItem, ['metadata', 'name']): malcolm_utils.deep_get( + installItem, ['metadata', 'version'] + ) + for installItem in dryRunReport.get('install', []) + } + pluginNeedsInstall = any( + [ + package_name + for package_name, new_version in wouldInstallInfo.items() + if (package_name not in preinstalledPackagesDict) + or (Version(new_version) > Version(preinstalledPackagesDict[package_name]['version'])) + ] + ) + else: + pluginNeedsInstall = True + + if pluginNeedsInstall: + with malcolm_utils.temporary_filename(suffix='.json') as installReportFileName: + cmd = [ + venvPy, + "-m", + "pip", + "--no-color", + "--no-input", + "--disable-pip-version-check", + "install", + "--upgrade", + "--progress-bar", + "off", + "--report", + installReportFileName, + packageDir, + ] + err, results = malcolm_utils.run_process(cmd, logger=logging) + installResult = err == 0 + + return installResult + + ################################################################################################### # main def main(): @@ -219,6 +280,14 @@ def main(): required=False, help="NetBox installation directory", ) + parser.add_argument( + '--netbox-config', + dest='netboxConfigDir', + type=str, + default=os.getenv('NETBOX_CONFIG_PATH', '/etc/netbox/config'), + required=False, + help="NetBox config directory (containing plugins.py, etc.)", + ) parser.add_argument( '-l', '--library', @@ -228,6 +297,15 @@ def main(): required=False, help="Directory containing NetBox Device-Type-Library-Import project and library repo", ) + parser.add_argument( + '-c', + '--custom-plugins', + dest='customPluginsDir', + type=str, + default=os.getenv('NETBOX_CUSTOM_PLUGINS_PATH', '/opt/netbox-custom-plugins'), + required=False, + help="Parent directory containing custom NetBox plugins to install", + ) parser.add_argument( '-p', '--preload', @@ -297,6 +375,132 @@ def main(): netboxVenvPy = os.path.join(os.path.join(os.path.join(args.netboxDir, 'venv'), 'bin'), 'python') manageScript = os.path.join(os.path.join(args.netboxDir, 'netbox'), 'manage.py') + # CUSTOM PLUGIN INSTALLATION ################################################################################# + if os.path.isdir(args.customPluginsDir) and os.path.isfile(os.path.join(args.netboxConfigDir, 'plugins.py')): + + # get a list of what packages/plugins already installed (package names and versions in a dict) + packagesInstalled = GetInstalledPackages(netboxVenvPy) + + # if there is a "requirements" subdirectory, handle that first as it contains dependencies + if os.path.isdir(os.path.join(args.customPluginsDir, 'requirements')): + requirementsSubDirs = [ + malcolm_utils.remove_suffix(f.path, '/') + for f in os.scandir(os.path.join(args.customPluginsDir, 'requirements')) + if f.is_dir() + ] + for packageDir in requirementsSubDirs: + packageInstalled = InstallPackageDirIfNeeded(packageDir, netboxVenvPy, packagesInstalled) + logging.info( + f"{os.path.basename(packageDir)} (dependency): {'' if packageInstalled else 'not ' }installed" + ) + + # now install the plugins directories + installedOrUpdatedPlugins = [] + customPluginSubdirs = [ + malcolm_utils.remove_suffix(f.path, '/') + for f in os.scandir(args.customPluginsDir) + if f.is_dir() and (os.path.basename(f) != 'requirements') + ] + for pluginDir in customPluginSubdirs: + if pluginInstalled := InstallPackageDirIfNeeded(pluginDir, netboxVenvPy, packagesInstalled): + installedOrUpdatedPlugins.append(pluginDir) + logging.info(f"{os.path.basename(pluginDir)}: {'' if pluginInstalled else 'not ' }installed") + + # for any packages that were newly installed (or updated, we'll be thorough) we need to make + # sure the package name is in the plugins.py + logging.info(f"Plugins installed or updated: {installedOrUpdatedPlugins}") + if installedOrUpdatedPlugins: + # get updated list of installed packages + packagesInstalled = GetInstalledPackages(netboxVenvPy) + + # now get the names of the NetBox plugins installed + pluginNames = [] + + # first get a list of __init__.py files for potential plugins installed in the package location(s) + cmd = [ + '/usr/bin/rg', + '--files-with-matches', + '--iglob', + '__init__.py', + r'\bPluginConfig\b', + list({package['location'] for package in packagesInstalled.values() if 'location' in package}), + ] + err, results = malcolm_utils.run_process(cmd, stderr=False, logger=logging) + if results: + # process each of those potential plugin __init__.py files + for pluginInitFileName in results: + try: + if os.path.isfile(pluginInitFileName): + # parse the Python of the __init__.py into an abstract syntax tree + with open(pluginInitFileName, 'r') as f: + node = ast.parse(f.read()) + # look at each Class defined in this code + for c in [n for n in node.body if isinstance(n, ast.ClassDef)]: + # plugins are classes with "PluginConfig" for a parent + if any([baseClass.id == 'PluginConfig' for baseClass in c.bases]): + # this ia a plugin class, so iterate over its members (functions, + # variables, etc.) to find its name + for item in c.body: + # the name is defined as an assignment (ast.Assign) + if isinstance(item, ast.Assign): + # does this assignment have a target called 'name'? + for target in item.targets: + if isinstance(target, ast.Name) and target.id == 'name': + # check if the value assigned to 'name' is a constant + if isinstance(item.value, ast.Constant): + pluginNames.append(item.value.value) + except Exception as e: + logging.error(f"{type(e).__name__} identifying NetBox plugin names: {e}") + + if pluginNames: + pluginNames = list(set(pluginNames)) + # at this point we have a list of plugin names for all of the plugin classes! + # we need to make sure they exist in plugins.py + + # Load and parse the plugins.py file + pluginsListFound = False + with open(os.path.join(args.netboxConfigDir, 'plugins.py'), 'r') as pluginFile: + code = pluginFile.read() + tree = ast.parse(code) + + # Walk the AST to find the PLUGINS assignment + class PluginListModifier(ast.NodeTransformer): + def visit_Assign(self, node): + global pluginsListFound + if isinstance(node.targets[0], ast.Name) and node.targets[0].id == 'PLUGINS': + pluginsListFound = True + # Check if the node's value is a list + if isinstance(node.value, ast.List): + # Get the existing plugin names in the list + existingPlugins = {elt.s for elt in node.value.elts if isinstance(elt, ast.Str)} + # Add new plugins if they aren't already in the list + for plugin in pluginNames: + if plugin not in existingPlugins: + node.value.elts.append(ast.Constant(value=plugin)) + return node + + # Modify the AST + modifier = PluginListModifier() + modifiedTree = modifier.visit(tree) + + # # If PLUGINS was not found, add it at the end of the module + if not pluginsListFound: + modifiedTree.body.append( + ast.Assign( + targets=[ast.Name(id='PLUGINS', ctx=ast.Store())], + value=ast.List(elts=[ast.Constant(value=plugin) for plugin in pluginNames], ctx=ast.Load()), + ) + ) + + # Unparse the modified AST back into code + modifiedCode = ast.unparse(ast.fix_missing_locations(modifiedTree)) + + # Write the modified code back to the file + with open(os.path.join(args.netboxConfigDir, 'plugins.py'), 'w') as pluginFile: + pluginFile.write(modifiedCode) + + # END CUSTOM PLUGIN INSTALLATION ############################################################################# + # if there is a database backup .gz in the preload directory, load it up (preferring the newest) # if there are multiple) instead of populating via API preloadDatabaseFile = args.preloadBackupFile diff --git a/netbox/supervisord.conf b/netbox/supervisord.conf index be80d8736..0764e5865 100644 --- a/netbox/supervisord.conf +++ b/netbox/supervisord.conf @@ -39,6 +39,7 @@ command=/opt/netbox/venv/bin/python /usr/local/bin/netbox_init.py --token "%(ENV_SUPERUSER_API_TOKEN)s" --library "%(ENV_NETBOX_DEVICETYPE_LIBRARY_IMPORT_PATH)s" --preload "%(ENV_NETBOX_PRELOAD_PATH)s" + --custom-plugins "%(ENV_NETBOX_CUSTOM_PLUGINS_PATH)s" --postgres-host "%(ENV_DB_HOST)s" --postgres-db "%(ENV_DB_NAME)s" --postgres-user "%(ENV_DB_USER)s" diff --git a/scripts/malcolm_appliance_packager.sh b/scripts/malcolm_appliance_packager.sh index 2c824c099..bd4634b96 100755 --- a/scripts/malcolm_appliance_packager.sh +++ b/scripts/malcolm_appliance_packager.sh @@ -66,6 +66,7 @@ if mkdir "$DESTDIR"; then mkdir $VERBOSE -p "$DESTDIR/htadmin/" mkdir $VERBOSE -p "$DESTDIR/logstash/certs/" mkdir $VERBOSE -p "$DESTDIR/logstash/maps/" + mkdir $VERBOSE -p "$DESTDIR/netbox/custom-plugins/requirements/" mkdir $VERBOSE -p "$DESTDIR/netbox/media/" mkdir $VERBOSE -p "$DESTDIR/netbox/postgres/" mkdir $VERBOSE -p "$DESTDIR/netbox/redis/" diff --git a/scripts/malcolm_utils.py b/scripts/malcolm_utils.py index 208ef2d2e..80d6ccf68 100644 --- a/scripts/malcolm_utils.py +++ b/scripts/malcolm_utils.py @@ -176,9 +176,9 @@ def decapitalize(s): # # Example: # d = {'meta': {'status': 'OK', 'status_code': 200}} -# DeepGet(d, ['meta', 'status_code']) # => 200 -# DeepGet(d, ['garbage', 'status_code']) # => None -# DeepGet(d, ['meta', 'garbage'], default='-') # => '-' +# deep_get(d, ['meta', 'status_code']) # => 200 +# deep_get(d, ['garbage', 'status_code']) # => None +# deep_get(d, ['meta', 'garbage'], default='-') # => '-' def deep_get(d, keys, default=None): k = get_iterable(keys) if d is None: From 890ab6115bd744a8469015f1620bbfe0a49ea590 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 14 Aug 2024 14:53:23 -0600 Subject: [PATCH 21/47] idaholab/Malcolm#530, work in progress on netbox plugin installation --- Dockerfiles/netbox.Dockerfile | 4 ++++ kubernetes/18-netbox.yml | 1 + netbox/scripts/netbox_init.py | 8 ++++++++ 3 files changed, 13 insertions(+) diff --git a/Dockerfiles/netbox.Dockerfile b/Dockerfiles/netbox.Dockerfile index 01e93ba27..2087189a4 100644 --- a/Dockerfiles/netbox.Dockerfile +++ b/Dockerfiles/netbox.Dockerfile @@ -43,6 +43,8 @@ ARG NETBOX_DEVICETYPE_LIBRARY_IMPORT_PATH="/opt/netbox-devicetype-library-import ARG NETBOX_DEFAULT_SITE=Malcolm ARG NETBOX_CRON=true ARG NETBOX_PRELOAD_PATH="/opt/netbox-preload" +ARG NETBOX_CUSTOM_PLUGINS_PATH="/opt/netbox-custom-plugins" +ARG NETBOX_CONFIG_PATH="/etc/netbox/config" ENV NETBOX_PATH /opt/netbox ENV BASE_PATH netbox @@ -50,6 +52,8 @@ ENV NETBOX_DEVICETYPE_LIBRARY_IMPORT_PATH $NETBOX_DEVICETYPE_LIBRARY_IMPORT_PATH ENV NETBOX_DEFAULT_SITE $NETBOX_DEFAULT_SITE ENV NETBOX_CRON $NETBOX_CRON ENV NETBOX_PRELOAD_PATH $NETBOX_PRELOAD_PATH +ENV NETBOX_CUSTOM_PLUGINS_PATH $NETBOX_CUSTOM_PLUGINS_PATH +ENV NETBOX_CONFIG_PATH $NETBOX_CONFIG_PATH ADD netbox/patch/* /tmp/netbox-patches/ diff --git a/kubernetes/18-netbox.yml b/kubernetes/18-netbox.yml index 03fb934d7..af9b5de48 100644 --- a/kubernetes/18-netbox.yml +++ b/kubernetes/18-netbox.yml @@ -79,6 +79,7 @@ spec: name: netbox-var-local-catrust-volume - mountPath: /etc/netbox/config/configmap name: netbox-config-volume + # TODO: custom-plugins - mountPath: /opt/netbox-preload/configmap name: netbox-preload-volume - mountPath: /opt/netbox/netbox/media diff --git a/netbox/scripts/netbox_init.py b/netbox/scripts/netbox_init.py index 6b836df40..59dd93834 100755 --- a/netbox/scripts/netbox_init.py +++ b/netbox/scripts/netbox_init.py @@ -396,6 +396,7 @@ def main(): # now install the plugins directories installedOrUpdatedPlugins = [] + pluginsListModified = False customPluginSubdirs = [ malcolm_utils.remove_suffix(f.path, '/') for f in os.scandir(args.customPluginsDir) @@ -467,6 +468,7 @@ def main(): class PluginListModifier(ast.NodeTransformer): def visit_Assign(self, node): global pluginsListFound + global pluginsListModified if isinstance(node.targets[0], ast.Name) and node.targets[0].id == 'PLUGINS': pluginsListFound = True # Check if the node's value is a list @@ -477,6 +479,7 @@ def visit_Assign(self, node): for plugin in pluginNames: if plugin not in existingPlugins: node.value.elts.append(ast.Constant(value=plugin)) + pluginsListModified = True return node # Modify the AST @@ -491,6 +494,7 @@ def visit_Assign(self, node): value=ast.List(elts=[ast.Constant(value=plugin) for plugin in pluginNames], ctx=ast.Load()), ) ) + pluginsListModified = True # Unparse the modified AST back into code modifiedCode = ast.unparse(ast.fix_missing_locations(modifiedTree)) @@ -499,6 +503,10 @@ def visit_Assign(self, node): with open(os.path.join(args.netboxConfigDir, 'plugins.py'), 'w') as pluginFile: pluginFile.write(modifiedCode) + if installedOrUpdatedPlugins or pluginsListModified: + # TODO: migrate? restart things? + pass + # END CUSTOM PLUGIN INSTALLATION ############################################################################# # if there is a database backup .gz in the preload directory, load it up (preferring the newest) From 86ef1134496976b09ab5d6d55e01cdec8f5397b0 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 14 Aug 2024 16:33:37 -0600 Subject: [PATCH 22/47] idaholab/Malcolm#529, in ISO installer, prompt to format other drives for artifact storage rather than just doing it automatically --- shared/bin/preseed_late_user_config.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/bin/preseed_late_user_config.sh b/shared/bin/preseed_late_user_config.sh index 61b81a022..3fe94bf3b 100755 --- a/shared/bin/preseed_late_user_config.sh +++ b/shared/bin/preseed_late_user_config.sh @@ -66,7 +66,7 @@ Description: SSH Password Authentication Template: malcolm/format_large_storage Type: boolean -Default: false +Default: true Description: Format non-OS drive(s) for artifact storage? From cf92f73e48fb2e1efcab23ab4c03c392a0204e99 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Thu, 15 Aug 2024 08:18:13 -0600 Subject: [PATCH 23/47] idaholab/Malcolm#530, work in progress on netbox plugin installation --- Dockerfiles/netbox.Dockerfile | 7 +- netbox/scripts/netbox_init.py | 290 +++----------------- netbox/scripts/netbox_install_plugins.py | 326 +++++++++++++++++++++++ netbox/supervisord.conf | 1 - 4 files changed, 369 insertions(+), 255 deletions(-) create mode 100755 netbox/scripts/netbox_install_plugins.py diff --git a/Dockerfiles/netbox.Dockerfile b/Dockerfiles/netbox.Dockerfile index 2087189a4..7b5aa802b 100644 --- a/Dockerfiles/netbox.Dockerfile +++ b/Dockerfiles/netbox.Dockerfile @@ -1,6 +1,6 @@ ARG TARGETPLATFORM=linux/amd64 -FROM --platform=${TARGETPLATFORM} netboxcommunity/netbox:v4.0.8 +FROM --platform=${TARGETPLATFORM} netboxcommunity/netbox:v4.0.9 # Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" @@ -120,7 +120,7 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') "Django>=4.2.10,<5" \ paramiko \ pillow && \ - mkdir -p "${NETBOX_PATH}/netbox/${BASE_PATH}" && \ + mkdir -p "${NETBOX_PATH}/netbox/${BASE_PATH}" "${NETBOX_CUSTOM_PLUGINS_PATH}/requirements" && \ mv "${NETBOX_PATH}/netbox/static" "${NETBOX_PATH}/netbox/${BASE_PATH}/static" && \ jq '. += { "settings": { "http": { "discard_unsafe_fields": false } } }' /etc/unit/nginx-unit.json | jq 'del(.listeners."[::]:8080")' | jq 'del(.listeners."[::]:8081")' | jq ".routes.main[0].match.uri = \"/${BASE_PATH}/static/*\"" > /etc/unit/nginx-unit-new.json && \ mv /etc/unit/nginx-unit-new.json /etc/unit/nginx-unit.json && \ @@ -128,7 +128,8 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') tr -cd '\11\12\15\40-\176' < "${NETBOX_PATH}/netbox/${BASE_PATH}/configuration.py" > "${NETBOX_PATH}/netbox/${BASE_PATH}/configuration_ascii.py" && \ mv "${NETBOX_PATH}/netbox/${BASE_PATH}/configuration_ascii.py" "${NETBOX_PATH}/netbox/${BASE_PATH}/configuration.py" && \ sed -i "s/\('CENSUS_REPORTING_ENABLED',[[:space:]]*\)True/\1False/" "${NETBOX_PATH}/netbox/${BASE_PATH}/settings.py" && \ - sed -i -E 's@^([[:space:]]*\-\-(state|tmp))([[:space:]])@\1dir\3@g' "${NETBOX_PATH}/launch-netbox.sh" + sed -i -E 's@^([[:space:]]*\-\-(state|tmp))([[:space:]])@\1dir\3@g' "${NETBOX_PATH}/launch-netbox.sh" && \ + sed -i '/\/opt\/netbox\/venv\/bin\/activate/a \\n# Install custom plugins \npython3 /usr/local/bin/netbox_install_plugins.py' /opt/netbox/docker-entrypoint.sh COPY --chmod=755 shared/bin/docker-uid-gid-setup.sh /usr/local/bin/ COPY --chmod=755 shared/bin/service_check_passthrough.sh /usr/local/bin/ diff --git a/netbox/scripts/netbox_init.py b/netbox/scripts/netbox_init.py index 59dd93834..c9b64af17 100755 --- a/netbox/scripts/netbox_init.py +++ b/netbox/scripts/netbox_init.py @@ -4,7 +4,6 @@ # Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. import argparse -import ast import glob import gzip import ipaddress @@ -27,7 +26,6 @@ from collections.abc import Iterable from distutils.dir_util import copy_tree from datetime import datetime -from packaging.version import Version from slugify import slugify ################################################################################################### @@ -38,6 +36,45 @@ ################################################################################################### +def get_iterable(x): + if isinstance(x, Iterable) and not isinstance(x, str): + return x + else: + return (x,) + + +def is_ip_address(x): + try: + ipaddress.ip_address(x) + return True + except Exception: + return False + + +def is_ip_v4_address(x): + try: + ipaddress.IPv4Address(x) + return True + except Exception: + return False + + +def is_ip_v6_address(x): + try: + ipaddress.IPv6Address(x) + return True + except Exception: + return False + + +def is_ip_network(x): + try: + ipaddress.ip_network(x) + return True + except Exception: + return False + + def min_hash_value_by_value(x): return next( iter(list({k: v for k, v in sorted(x.items(), key=lambda item: item[1])}.values())), @@ -68,104 +105,6 @@ def max_hash_value_by_key(x): return last -def GetInstalledPackages(venvPy): - packagesInstalled = {} - cmd = [ - venvPy, - "-m", - "pip", - "--no-color", - "--no-input", - "--disable-pip-version-check", - "list", - "--local", - "--format", - "json", - "--verbose", - ] - err, results = malcolm_utils.run_process(cmd, stderr=False, logger=logging) - if (err == 0) and results and (len(results) > 0): - try: - packagesInstalled = {item['name']: item for item in malcolm_utils.LoadStrIfJson(results[0])} - except Exception as e: - logging.error(f"{type(e).__name__} getting list of installed Python packages: {e}") - - return packagesInstalled - - -def InstallPackageDirIfNeeded( - packageDir, - venvPy, - preinstalledPackagesDict={}, -): - installResult = False - - # First do a "dry run" install to determine what would happen. The report from this will - # help us determine if the package actually needs installed or not, as pip always treats - # installations from local directories as "new installs" and would uninstall/reinstall - # no matter what, which we want to avoid if we don't need it. - pluginNeedsInstall = False - with malcolm_utils.temporary_filename(suffix='.json') as dryRunInstallReportFileName: - cmd = [ - venvPy, - "-m", - "pip", - "--no-color", - "--no-input", - "--disable-pip-version-check", - "install", - "--upgrade", - "--dry-run", - "--progress-bar", - "off", - "--report", - dryRunInstallReportFileName, - packageDir, - ] - err, results = malcolm_utils.run_process(cmd, logger=logging) - if (err == 0) and os.path.isfile(dryRunInstallReportFileName): - with open(dryRunInstallReportFileName, 'r') as f: - dryRunReport = malcolm_utils.LoadFileIfJson(f) - wouldInstallInfo = { - malcolm_utils.deep_get(installItem, ['metadata', 'name']): malcolm_utils.deep_get( - installItem, ['metadata', 'version'] - ) - for installItem in dryRunReport.get('install', []) - } - pluginNeedsInstall = any( - [ - package_name - for package_name, new_version in wouldInstallInfo.items() - if (package_name not in preinstalledPackagesDict) - or (Version(new_version) > Version(preinstalledPackagesDict[package_name]['version'])) - ] - ) - else: - pluginNeedsInstall = True - - if pluginNeedsInstall: - with malcolm_utils.temporary_filename(suffix='.json') as installReportFileName: - cmd = [ - venvPy, - "-m", - "pip", - "--no-color", - "--no-input", - "--disable-pip-version-check", - "install", - "--upgrade", - "--progress-bar", - "off", - "--report", - installReportFileName, - packageDir, - ] - err, results = malcolm_utils.run_process(cmd, logger=logging) - installResult = err == 0 - - return installResult - - ################################################################################################### # main def main(): @@ -280,14 +219,6 @@ def main(): required=False, help="NetBox installation directory", ) - parser.add_argument( - '--netbox-config', - dest='netboxConfigDir', - type=str, - default=os.getenv('NETBOX_CONFIG_PATH', '/etc/netbox/config'), - required=False, - help="NetBox config directory (containing plugins.py, etc.)", - ) parser.add_argument( '-l', '--library', @@ -297,15 +228,6 @@ def main(): required=False, help="Directory containing NetBox Device-Type-Library-Import project and library repo", ) - parser.add_argument( - '-c', - '--custom-plugins', - dest='customPluginsDir', - type=str, - default=os.getenv('NETBOX_CUSTOM_PLUGINS_PATH', '/opt/netbox-custom-plugins'), - required=False, - help="Parent directory containing custom NetBox plugins to install", - ) parser.add_argument( '-p', '--preload', @@ -375,140 +297,6 @@ def main(): netboxVenvPy = os.path.join(os.path.join(os.path.join(args.netboxDir, 'venv'), 'bin'), 'python') manageScript = os.path.join(os.path.join(args.netboxDir, 'netbox'), 'manage.py') - # CUSTOM PLUGIN INSTALLATION ################################################################################# - if os.path.isdir(args.customPluginsDir) and os.path.isfile(os.path.join(args.netboxConfigDir, 'plugins.py')): - - # get a list of what packages/plugins already installed (package names and versions in a dict) - packagesInstalled = GetInstalledPackages(netboxVenvPy) - - # if there is a "requirements" subdirectory, handle that first as it contains dependencies - if os.path.isdir(os.path.join(args.customPluginsDir, 'requirements')): - requirementsSubDirs = [ - malcolm_utils.remove_suffix(f.path, '/') - for f in os.scandir(os.path.join(args.customPluginsDir, 'requirements')) - if f.is_dir() - ] - for packageDir in requirementsSubDirs: - packageInstalled = InstallPackageDirIfNeeded(packageDir, netboxVenvPy, packagesInstalled) - logging.info( - f"{os.path.basename(packageDir)} (dependency): {'' if packageInstalled else 'not ' }installed" - ) - - # now install the plugins directories - installedOrUpdatedPlugins = [] - pluginsListModified = False - customPluginSubdirs = [ - malcolm_utils.remove_suffix(f.path, '/') - for f in os.scandir(args.customPluginsDir) - if f.is_dir() and (os.path.basename(f) != 'requirements') - ] - for pluginDir in customPluginSubdirs: - if pluginInstalled := InstallPackageDirIfNeeded(pluginDir, netboxVenvPy, packagesInstalled): - installedOrUpdatedPlugins.append(pluginDir) - logging.info(f"{os.path.basename(pluginDir)}: {'' if pluginInstalled else 'not ' }installed") - - # for any packages that were newly installed (or updated, we'll be thorough) we need to make - # sure the package name is in the plugins.py - logging.info(f"Plugins installed or updated: {installedOrUpdatedPlugins}") - if installedOrUpdatedPlugins: - # get updated list of installed packages - packagesInstalled = GetInstalledPackages(netboxVenvPy) - - # now get the names of the NetBox plugins installed - pluginNames = [] - - # first get a list of __init__.py files for potential plugins installed in the package location(s) - cmd = [ - '/usr/bin/rg', - '--files-with-matches', - '--iglob', - '__init__.py', - r'\bPluginConfig\b', - list({package['location'] for package in packagesInstalled.values() if 'location' in package}), - ] - err, results = malcolm_utils.run_process(cmd, stderr=False, logger=logging) - if results: - # process each of those potential plugin __init__.py files - for pluginInitFileName in results: - try: - if os.path.isfile(pluginInitFileName): - # parse the Python of the __init__.py into an abstract syntax tree - with open(pluginInitFileName, 'r') as f: - node = ast.parse(f.read()) - # look at each Class defined in this code - for c in [n for n in node.body if isinstance(n, ast.ClassDef)]: - # plugins are classes with "PluginConfig" for a parent - if any([baseClass.id == 'PluginConfig' for baseClass in c.bases]): - # this ia a plugin class, so iterate over its members (functions, - # variables, etc.) to find its name - for item in c.body: - # the name is defined as an assignment (ast.Assign) - if isinstance(item, ast.Assign): - # does this assignment have a target called 'name'? - for target in item.targets: - if isinstance(target, ast.Name) and target.id == 'name': - # check if the value assigned to 'name' is a constant - if isinstance(item.value, ast.Constant): - pluginNames.append(item.value.value) - except Exception as e: - logging.error(f"{type(e).__name__} identifying NetBox plugin names: {e}") - - if pluginNames: - pluginNames = list(set(pluginNames)) - # at this point we have a list of plugin names for all of the plugin classes! - # we need to make sure they exist in plugins.py - - # Load and parse the plugins.py file - pluginsListFound = False - with open(os.path.join(args.netboxConfigDir, 'plugins.py'), 'r') as pluginFile: - code = pluginFile.read() - tree = ast.parse(code) - - # Walk the AST to find the PLUGINS assignment - class PluginListModifier(ast.NodeTransformer): - def visit_Assign(self, node): - global pluginsListFound - global pluginsListModified - if isinstance(node.targets[0], ast.Name) and node.targets[0].id == 'PLUGINS': - pluginsListFound = True - # Check if the node's value is a list - if isinstance(node.value, ast.List): - # Get the existing plugin names in the list - existingPlugins = {elt.s for elt in node.value.elts if isinstance(elt, ast.Str)} - # Add new plugins if they aren't already in the list - for plugin in pluginNames: - if plugin not in existingPlugins: - node.value.elts.append(ast.Constant(value=plugin)) - pluginsListModified = True - return node - - # Modify the AST - modifier = PluginListModifier() - modifiedTree = modifier.visit(tree) - - # # If PLUGINS was not found, add it at the end of the module - if not pluginsListFound: - modifiedTree.body.append( - ast.Assign( - targets=[ast.Name(id='PLUGINS', ctx=ast.Store())], - value=ast.List(elts=[ast.Constant(value=plugin) for plugin in pluginNames], ctx=ast.Load()), - ) - ) - pluginsListModified = True - - # Unparse the modified AST back into code - modifiedCode = ast.unparse(ast.fix_missing_locations(modifiedTree)) - - # Write the modified code back to the file - with open(os.path.join(args.netboxConfigDir, 'plugins.py'), 'w') as pluginFile: - pluginFile.write(modifiedCode) - - if installedOrUpdatedPlugins or pluginsListModified: - # TODO: migrate? restart things? - pass - - # END CUSTOM PLUGIN INSTALLATION ############################################################################# - # if there is a database backup .gz in the preload directory, load it up (preferring the newest) # if there are multiple) instead of populating via API preloadDatabaseFile = args.preloadBackupFile diff --git a/netbox/scripts/netbox_install_plugins.py b/netbox/scripts/netbox_install_plugins.py new file mode 100755 index 000000000..f796fa3f4 --- /dev/null +++ b/netbox/scripts/netbox_install_plugins.py @@ -0,0 +1,326 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. + +import argparse +import ast +import json +import logging +import os +import sys +import malcolm_utils + +from datetime import datetime +from packaging.version import Version + +################################################################################################### +args = None +script_name = os.path.basename(__file__) +script_path = os.path.dirname(os.path.realpath(__file__)) +orig_path = os.getcwd() + + +################################################################################################### +def GetInstalledPackages(venvPy): + packagesInstalled = {} + cmd = [ + venvPy, + "-m", + "pip", + "--no-color", + "--no-input", + "--disable-pip-version-check", + "list", + "--local", + "--format", + "json", + "--verbose", + ] + err, results = malcolm_utils.run_process(cmd, stderr=False, logger=logging) + if (err == 0) and results and (len(results) > 0): + try: + packagesInstalled = {item['name']: item for item in malcolm_utils.LoadStrIfJson(results[0])} + except Exception as e: + logging.error(f"{type(e).__name__} getting list of installed Python packages: {e}") + + return packagesInstalled + + +################################################################################################### +def InstallPackageDirIfNeeded( + packageDir, + venvPy, + preinstalledPackagesDict={}, +): + installResult = False + + # First do a "dry run" install to determine what would happen. The report from this will + # help us determine if the package actually needs installed or not, as pip always treats + # installations from local directories as "new installs" and would uninstall/reinstall + # no matter what, which we want to avoid if we don't need it. + pluginNeedsInstall = False + with malcolm_utils.temporary_filename(suffix='.json') as dryRunInstallReportFileName: + cmd = [ + venvPy, + "-m", + "pip", + "--no-color", + "--no-input", + "--disable-pip-version-check", + "install", + "--upgrade", + "--dry-run", + "--progress-bar", + "off", + "--report", + dryRunInstallReportFileName, + packageDir, + ] + err, results = malcolm_utils.run_process(cmd, logger=logging) + if (err == 0) and os.path.isfile(dryRunInstallReportFileName): + with open(dryRunInstallReportFileName, 'r') as f: + dryRunReport = malcolm_utils.LoadFileIfJson(f) + wouldInstallInfo = { + malcolm_utils.deep_get(installItem, ['metadata', 'name']): malcolm_utils.deep_get( + installItem, ['metadata', 'version'] + ) + for installItem in dryRunReport.get('install', []) + } + pluginNeedsInstall = any( + [ + package_name + for package_name, new_version in wouldInstallInfo.items() + if (package_name not in preinstalledPackagesDict) + or (Version(new_version) > Version(preinstalledPackagesDict[package_name]['version'])) + ] + ) + else: + pluginNeedsInstall = True + + if pluginNeedsInstall: + with malcolm_utils.temporary_filename(suffix='.json') as installReportFileName: + cmd = [ + venvPy, + "-m", + "pip", + "--no-color", + "--no-input", + "--disable-pip-version-check", + "install", + "--upgrade", + "--progress-bar", + "off", + "--report", + installReportFileName, + packageDir, + ] + err, results = malcolm_utils.run_process(cmd, logger=logging) + installResult = err == 0 + + return installResult + + +################################################################################################### +# main +def main(): + global args + + parser = argparse.ArgumentParser( + description='\n'.join([]), + formatter_class=argparse.RawTextHelpFormatter, + add_help=False, + usage='{} '.format(script_name), + ) + parser.add_argument( + '--verbose', + '-v', + action='count', + default=1, + help='Increase verbosity (e.g., -v, -vv, etc.)', + ) + parser.add_argument( + '-n', + '--netbox', + dest='netboxDir', + type=str, + default=os.getenv('NETBOX_PATH', '/opt/netbox'), + required=False, + help="NetBox installation directory", + ) + parser.add_argument( + '--netbox-config', + dest='netboxConfigDir', + type=str, + default=os.getenv('NETBOX_CONFIG_PATH', '/etc/netbox/config'), + required=False, + help="NetBox config directory (containing plugins.py, etc.)", + ) + parser.add_argument( + '-c', + '--custom-plugins', + dest='customPluginsDir', + type=str, + default=os.getenv('NETBOX_CUSTOM_PLUGINS_PATH', '/opt/netbox-custom-plugins'), + required=False, + help="Parent directory containing custom NetBox plugins to install", + ) + try: + parser.error = parser.exit + args = parser.parse_args() + except SystemExit: + parser.print_help() + exit(2) + + args.verbose = logging.ERROR - (10 * args.verbose) if args.verbose > 0 else 0 + logging.basicConfig( + level=args.verbose, format='%(asctime)s %(levelname)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S' + ) + logging.debug(os.path.join(script_path, script_name)) + logging.debug("Arguments: {}".format(sys.argv[1:])) + logging.debug("Arguments: {}".format(args)) + if args.verbose > logging.DEBUG: + sys.tracebacklimit = 0 + + netboxVenvPy = os.path.join(os.path.join(os.path.join(args.netboxDir, 'venv'), 'bin'), 'python') + manageScript = os.path.join(os.path.join(args.netboxDir, 'netbox'), 'manage.py') + + if os.path.isdir(args.customPluginsDir) and os.path.isfile(os.path.join(args.netboxConfigDir, 'plugins.py')): + + # get a list of what packages/plugins already installed (package names and versions in a dict) + packagesInstalled = GetInstalledPackages(netboxVenvPy) + + # if there is a "requirements" subdirectory, handle that first as it contains dependencies + if os.path.isdir(os.path.join(args.customPluginsDir, 'requirements')): + requirementsSubDirs = [ + malcolm_utils.remove_suffix(f.path, '/') + for f in os.scandir(os.path.join(args.customPluginsDir, 'requirements')) + if f.is_dir() + ] + for packageDir in requirementsSubDirs: + packageInstalled = InstallPackageDirIfNeeded(packageDir, netboxVenvPy, packagesInstalled) + logging.info( + f"{os.path.basename(packageDir)} (dependency): {'' if packageInstalled else 'not ' }installed" + ) + + # now install the plugins directories + installedOrUpdatedPlugins = [] + global pluginsListModified + global pluginsListFound + pluginsListModified = False + pluginsListFound = False + customPluginSubdirs = [ + malcolm_utils.remove_suffix(f.path, '/') + for f in os.scandir(args.customPluginsDir) + if f.is_dir() and (os.path.basename(f) != 'requirements') + ] + for pluginDir in customPluginSubdirs: + if pluginInstalled := InstallPackageDirIfNeeded(pluginDir, netboxVenvPy, packagesInstalled): + installedOrUpdatedPlugins.append(pluginDir) + logging.info(f"{os.path.basename(pluginDir)}: {'' if pluginInstalled else 'not ' }installed") + + # for any packages that were newly installed (or updated, we'll be thorough) we need to make + # sure the package name is in the plugins.py + logging.info(f"Plugins installed or updated: {installedOrUpdatedPlugins}") + if installedOrUpdatedPlugins: + # get updated list of installed packages + packagesInstalled = GetInstalledPackages(netboxVenvPy) + + # now get the names of the NetBox plugins installed + pluginNames = [] + + # first get a list of __init__.py files for potential plugins installed in the package location(s) + cmd = [ + '/usr/bin/rg', + '--files-with-matches', + '--iglob', + '__init__.py', + r'\bPluginConfig\b', + list({package['location'] for package in packagesInstalled.values() if 'location' in package}), + ] + err, results = malcolm_utils.run_process(cmd, stderr=False, logger=logging) + if results: + # process each of those potential plugin __init__.py files + for pluginInitFileName in results: + try: + if os.path.isfile(pluginInitFileName): + # parse the Python of the __init__.py into an abstract syntax tree + with open(pluginInitFileName, 'r') as f: + node = ast.parse(f.read()) + # look at each Class defined in this code + for c in [n for n in node.body if isinstance(n, ast.ClassDef)]: + # plugins are classes with "PluginConfig" for a parent + if any([baseClass.id == 'PluginConfig' for baseClass in c.bases]): + # this ia a plugin class, so iterate over its members (functions, + # variables, etc.) to find its name + for item in c.body: + # the name is defined as an assignment (ast.Assign) + if isinstance(item, ast.Assign): + # does this assignment have a target called 'name'? + for target in item.targets: + if isinstance(target, ast.Name) and target.id == 'name': + # check if the value assigned to 'name' is a constant + if isinstance(item.value, ast.Constant): + pluginNames.append(item.value.value) + except Exception as e: + logging.error(f"{type(e).__name__} identifying NetBox plugin names: {e}") + + if pluginNames: + pluginNames = list(set(pluginNames)) + # at this point we have a list of plugin names for all of the plugin classes! + # we need to make sure they exist in plugins.py + + # Load and parse the plugins.py file + pluginsListFound = False + with open(os.path.join(args.netboxConfigDir, 'plugins.py'), 'r') as pluginFile: + code = pluginFile.read() + tree = ast.parse(code) + + # Walk the AST to find the PLUGINS assignment + class PluginListModifier(ast.NodeTransformer): + def visit_Assign(self, node): + global pluginsListFound + global pluginsListModified + if isinstance(node.targets[0], ast.Name) and node.targets[0].id == 'PLUGINS': + pluginsListFound = True + # Check if the node's value is a list + if isinstance(node.value, ast.List): + # Get the existing plugin names in the list + existingPlugins = {elt.s for elt in node.value.elts if isinstance(elt, ast.Str)} + # Add new plugins if they aren't already in the list + for plugin in pluginNames: + if plugin not in existingPlugins: + node.value.elts.append(ast.Constant(value=plugin)) + pluginsListModified = True + return node + + # Modify the AST + modifier = PluginListModifier() + modifiedTree = modifier.visit(tree) + + # # If PLUGINS was not found, add it at the end of the module + if not pluginsListFound: + logging.debug('here') + modifiedTree.body.append( + ast.Assign( + targets=[ast.Name(id='PLUGINS', ctx=ast.Store())], + value=ast.List(elts=[ast.Constant(value=plugin) for plugin in pluginNames], ctx=ast.Load()), + ) + ) + pluginsListModified = True + + # Unparse the modified AST back into code + modifiedCode = ast.unparse(ast.fix_missing_locations(modifiedTree)) + + # Write the modified code back to the file + with open(os.path.join(args.netboxConfigDir, 'plugins.py'), 'w') as pluginFile: + pluginFile.write(modifiedCode) + + if installedOrUpdatedPlugins or pluginsListModified: + # TODO: migrate? restart things? + pass + + +################################################################################################### +if __name__ == '__main__': + main() diff --git a/netbox/supervisord.conf b/netbox/supervisord.conf index 0764e5865..be80d8736 100644 --- a/netbox/supervisord.conf +++ b/netbox/supervisord.conf @@ -39,7 +39,6 @@ command=/opt/netbox/venv/bin/python /usr/local/bin/netbox_init.py --token "%(ENV_SUPERUSER_API_TOKEN)s" --library "%(ENV_NETBOX_DEVICETYPE_LIBRARY_IMPORT_PATH)s" --preload "%(ENV_NETBOX_PRELOAD_PATH)s" - --custom-plugins "%(ENV_NETBOX_CUSTOM_PLUGINS_PATH)s" --postgres-host "%(ENV_DB_HOST)s" --postgres-db "%(ENV_DB_NAME)s" --postgres-user "%(ENV_DB_USER)s" From bd7aab6f828569b838e6781720c84af5c8f87454 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Thu, 15 Aug 2024 08:23:01 -0600 Subject: [PATCH 24/47] idaholab/Malcolm#530, work in progress on netbox plugin installation --- netbox/config/plugins.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/netbox/config/plugins.py b/netbox/config/plugins.py index f170eef4e..e67df2e5d 100644 --- a/netbox/config/plugins.py +++ b/netbox/config/plugins.py @@ -1,15 +1 @@ -# Add your plugins and plugin settings here. -# Of course uncomment this file out. - -# To learn how to build images with your required plugins -# See https://github.com/netbox-community/netbox-docker/wiki/Using-Netbox-Plugins - -PLUGINS = [ - 'netbox_initializers', -] - -# PLUGINS_CONFIG = { -# "netbox_bgp": { -# ADD YOUR SETTINGS HERE -# } -# } +PLUGINS = [] From c66aa9db348a7378a0d2dd6e2c3138eb23ea612b Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Thu, 15 Aug 2024 08:37:41 -0600 Subject: [PATCH 25/47] idaholab/Malcolm#530, documentation for netbox plugin installation --- docs/README.md | 3 ++- docs/custom-rules.md | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/README.md b/docs/README.md index 3fd33c8ce..cbff488d8 100644 --- a/docs/README.md +++ b/docs/README.md @@ -88,10 +88,11 @@ Malcolm can also easily be deployed locally on an ordinary consumer workstation + [Email Sender Accounts](alerting.md#AlertingEmail) * [Search Queries in Arkime and OpenSearch Dashboards](queries-cheat-sheet.md#SearchCheatSheet) * Other Malcolm features - - [Custom Rules and Scripts](custom-rules.md#CustomRulesAndScripts) + - [Custom Rules, Scripts and Plugins](custom-rules.md#CustomRulesAndScripts) + [Suricata](custom-rules.md#Suricata) + [Zeek](custom-rules.md#Zeek) + [YARA](custom-rules.md#YARA) + + [NetBox Plugins](custom-rules.md#NetBox) + [Other Customizations](custom-rules.md#Other) - [Automatic file extraction and scanning](file-scanning.md#ZeekFileExtraction) + [User interface](file-scanning.md#ZeekFileExtractionUI) diff --git a/docs/custom-rules.md b/docs/custom-rules.md index 64f62ed4c..d2ac8f0ea 100644 --- a/docs/custom-rules.md +++ b/docs/custom-rules.md @@ -3,7 +3,7 @@ * [Suricata](#Suricata) * [Zeek](#Zeek) * [YARA](#YARA) -* [NetBox](#NetBox) +* [NetBox Plugins](#NetBox) * [Other Customizations](#Other) Much of Malcolm's behavior can be adjusted through [environment variable files](malcolm-config.md#MalcolmConfigEnvVars). However, some components allow further customization through the use of custom scripts, configuration files, and rules. @@ -74,9 +74,11 @@ docker compose exec file-monitor supervisorctl restart yara If the `EXTRACTED_FILE_YARA_CUSTOM_ONLY` [environment variable](malcolm-config.md#MalcolmConfigEnvVars) is set to `true`, Malcolm will bypass the default Yara rulesets ([Neo23x0/signature-base](https://github.com/Neo23x0/signature-base), [reversinglabs/reversinglabs-yara-rules](https://github.com/reversinglabs/reversinglabs-yara-rules), and [bartblaze/Yara-rules](https://github.com/bartblaze/Yara-rules)) and use only user-defined rules in `./yara/rules`. -## NetBox +## NetBox Plugins -TODO documentation +NetBox's functionality can be extended with plugins that can provide "[new data models, integrations, and more](https://netboxlabs.com/netbox-plugins/)" (see also the [NetBox Wiki](https://github.com/netbox-community/netbox/wiki/Plugins)). + +When Malcolm's NetBox container [starts up]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/netbox/scripts/netbox_install_plugins.py), it installs (using [pip](https://packaging.python.org/en/latest/guides/tool-recommendations/#installing-packages)) any NetBox plugins that have [cloned](https://docs.github.com/en/repositories/creating-and-managing-repositories/cloning-a-repository) or [downloaded and extracted](https://docs.github.com/en/repositories/working-with-files/using-files/downloading-source-code-archives) into subdirectories in `./netbox/custom-plugins/` in the Malcolm installation directory. In instances where Malcolm is being run in an offline/airgapped configuration, the plugins' additional dependencies must also be present under `./netbox/custom-plugins/requirements/`, where they will be automatically installed first. ## Other Customizations From 87ca32a47cbe7c325ce02491545719c0c44032ee Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Thu, 15 Aug 2024 08:48:22 -0600 Subject: [PATCH 26/47] update URLs for fluent-bit-setup.ps1 helper script (idaholab/Malcolm#541) --- scripts/third-party-logs/fluent-bit-setup.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/third-party-logs/fluent-bit-setup.ps1 b/scripts/third-party-logs/fluent-bit-setup.ps1 index 84b603a01..f901ea0a6 100644 --- a/scripts/third-party-logs/fluent-bit-setup.ps1 +++ b/scripts/third-party-logs/fluent-bit-setup.ps1 @@ -2,14 +2,14 @@ # fluent-bit-setup.ps1 # # Interactive PowerShell script to aid in the installation and -# configuration of fluent-bit (https://fluentbit.io/) for forwarding logs to +# configuration of fluent-bit (https://packages.fluentbit.io) for forwarding logs to # an instance of Malcolm (https://github.com/idaholab/malcolm). # # Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ############################################################################### $fluent_bit_version = '3.1' -$fluent_bit_full_version = '3.1.5' +$fluent_bit_full_version = '3.1.6' ############################################################################### # select an item from a menu provided in an array @@ -107,8 +107,8 @@ if (-Not $fluentbit_installed) { $choices = '&Yes', '&No' $decision = $Host.UI.PromptForChoice($title, $question, $choices, 0) if ($decision -eq 0) { - $fluentbit_sha_url = "https://fluentbit.io/releases/$fluent_bit_version/$fluent_bit_shafile" - $fluentbit_url = "https://fluentbit.io/releases/$fluent_bit_version/$fluent_bit_zip" + $fluentbit_sha_url = "https://packages.fluentbit.io/windows/$fluent_bit_shafile" + $fluentbit_url = "https://packages.fluentbit.io/windows/$fluent_bit_zip" Invoke-WebRequest -Uri "$fluentbit_sha_url" -OutFile "$fluent_bit_shafile" Invoke-WebRequest -Uri "$fluentbit_url" -OutFile "$fluent_bit_zip" } From 41274d158f7bb45f7973cd660e7e5360a98901c8 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Thu, 15 Aug 2024 09:22:37 -0600 Subject: [PATCH 27/47] idaholab/Malcolm#530, work in progress on netbox plugin installation --- Dockerfiles/netbox.Dockerfile | 2 ++ netbox/scripts/netbox_install_plugins.py | 40 +++++++++++++----------- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/Dockerfiles/netbox.Dockerfile b/Dockerfiles/netbox.Dockerfile index 7b5aa802b..dcba61abc 100644 --- a/Dockerfiles/netbox.Dockerfile +++ b/Dockerfiles/netbox.Dockerfile @@ -139,6 +139,8 @@ COPY --chmod=644 scripts/malcolm_utils.py /usr/local/bin/ COPY --chmod=644 netbox/supervisord.conf /etc/supervisord.conf COPY --chmod=644 netbox/preload/*.yml $NETBOX_PRELOAD_PATH/ +VOLUME ["/opt/netbox/venv/lib/python3.11/site-packages"] + EXPOSE 9001 ENTRYPOINT ["/usr/bin/tini", \ diff --git a/netbox/scripts/netbox_install_plugins.py b/netbox/scripts/netbox_install_plugins.py index f796fa3f4..82733035d 100755 --- a/netbox/scripts/netbox_install_plugins.py +++ b/netbox/scripts/netbox_install_plugins.py @@ -10,9 +10,11 @@ import os import sys import malcolm_utils +from tempfile import TemporaryDirectory from datetime import datetime from packaging.version import Version +from distutils import dir_util ################################################################################################### args = None @@ -99,24 +101,26 @@ def InstallPackageDirIfNeeded( pluginNeedsInstall = True if pluginNeedsInstall: - with malcolm_utils.temporary_filename(suffix='.json') as installReportFileName: - cmd = [ - venvPy, - "-m", - "pip", - "--no-color", - "--no-input", - "--disable-pip-version-check", - "install", - "--upgrade", - "--progress-bar", - "off", - "--report", - installReportFileName, - packageDir, - ] - err, results = malcolm_utils.run_process(cmd, logger=logging) - installResult = err == 0 + with TemporaryDirectory() as tmpPackageDir: + dir_util.copy_tree(packageDir, tmpPackageDir, preserve_symlinks=True) + with malcolm_utils.temporary_filename(suffix='.json') as installReportFileName: + cmd = [ + venvPy, + "-m", + "pip", + "--no-color", + "--no-input", + "--disable-pip-version-check", + "install", + "--upgrade", + "--progress-bar", + "off", + "--report", + installReportFileName, + tmpPackageDir, + ] + err, results = malcolm_utils.run_process(cmd, logger=logging) + installResult = err == 0 return installResult From 06e0684f8c06499aab4fb30a674bac40cede6729 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Thu, 15 Aug 2024 11:31:35 -0600 Subject: [PATCH 28/47] idaholab/Malcolm#530, work in progress on netbox plugin installation --- netbox/scripts/netbox_install_plugins.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/netbox/scripts/netbox_install_plugins.py b/netbox/scripts/netbox_install_plugins.py index 82733035d..1a35eae46 100755 --- a/netbox/scripts/netbox_install_plugins.py +++ b/netbox/scripts/netbox_install_plugins.py @@ -321,8 +321,18 @@ def visit_Assign(self, node): pluginFile.write(modifiedCode) if installedOrUpdatedPlugins or pluginsListModified: - # TODO: migrate? restart things? - pass + # collect static files + with malcolm_utils.pushd(os.path.dirname(manageScript)): + # migrations if needed + cmd = [ + netboxVenvPy, + os.path.basename(manageScript), + "collectstatic", + "--no-input", + ] + err, results = malcolm_utils.run_process(cmd, logger=logging) + if err != 0: + logging.error(f'{err} collecting static files: {results}') ################################################################################################### From 4f03919a331a6bbe8d6de8822ac9d4ce21af0ef6 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Thu, 15 Aug 2024 11:53:03 -0600 Subject: [PATCH 29/47] idaholab/Malcolm#530, work in progress on netbox plugin installation --- Dockerfiles/netbox.Dockerfile | 2 -- netbox/scripts/netbox_install_plugins.py | 18 +++++++++++++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/Dockerfiles/netbox.Dockerfile b/Dockerfiles/netbox.Dockerfile index dcba61abc..7b5aa802b 100644 --- a/Dockerfiles/netbox.Dockerfile +++ b/Dockerfiles/netbox.Dockerfile @@ -139,8 +139,6 @@ COPY --chmod=644 scripts/malcolm_utils.py /usr/local/bin/ COPY --chmod=644 netbox/supervisord.conf /etc/supervisord.conf COPY --chmod=644 netbox/preload/*.yml $NETBOX_PRELOAD_PATH/ -VOLUME ["/opt/netbox/venv/lib/python3.11/site-packages"] - EXPOSE 9001 ENTRYPOINT ["/usr/bin/tini", \ diff --git a/netbox/scripts/netbox_install_plugins.py b/netbox/scripts/netbox_install_plugins.py index 1a35eae46..02524ad71 100755 --- a/netbox/scripts/netbox_install_plugins.py +++ b/netbox/scripts/netbox_install_plugins.py @@ -324,6 +324,22 @@ def visit_Assign(self, node): # collect static files with malcolm_utils.pushd(os.path.dirname(manageScript)): # migrations if needed + for plugin in pluginNames: + cmd = [ + netboxVenvPy, + os.path.basename(manageScript), + "makemigrations", + plugin, + ] + err, results = malcolm_utils.run_process(cmd, logger=logging) + if err != 0: + logging.warning(f'{err} making migrations for {plugin}: {results}') + + cmd = [netboxVenvPy, os.path.basename(manageScript), "migrate"] + err, results = malcolm_utils.run_process(cmd, logger=logging) + if err != 0: + logging.warning(f'{err} migrating: {results}') + cmd = [ netboxVenvPy, os.path.basename(manageScript), @@ -332,7 +348,7 @@ def visit_Assign(self, node): ] err, results = malcolm_utils.run_process(cmd, logger=logging) if err != 0: - logging.error(f'{err} collecting static files: {results}') + logging.warning(f'{err} collecting static files: {results}') ################################################################################################### From f76f2c1e83e0ab00b9b839e974e7ac5130286ccc Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Thu, 15 Aug 2024 11:57:19 -0600 Subject: [PATCH 30/47] idaholab/Malcolm#530, work in progress on netbox plugin installation --- docs/custom-rules.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/custom-rules.md b/docs/custom-rules.md index d2ac8f0ea..59ce9b7f0 100644 --- a/docs/custom-rules.md +++ b/docs/custom-rules.md @@ -80,6 +80,10 @@ NetBox's functionality can be extended with plugins that can provide "[new data When Malcolm's NetBox container [starts up]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/netbox/scripts/netbox_install_plugins.py), it installs (using [pip](https://packaging.python.org/en/latest/guides/tool-recommendations/#installing-packages)) any NetBox plugins that have [cloned](https://docs.github.com/en/repositories/creating-and-managing-repositories/cloning-a-repository) or [downloaded and extracted](https://docs.github.com/en/repositories/working-with-files/using-files/downloading-source-code-archives) into subdirectories in `./netbox/custom-plugins/` in the Malcolm installation directory. In instances where Malcolm is being run in an offline/airgapped configuration, the plugins' additional dependencies must also be present under `./netbox/custom-plugins/requirements/`, where they will be automatically installed first. +The following warning is quoted from the [NetBox documentation](https://netboxlabs.com/docs/netbox/en/stable/configuration/plugins/): + +> Plugins extend NetBox by allowing external code to run with the same access and privileges as NetBox itself. Only install plugins from trusted sources. The NetBox maintainers make absolutely no guarantees about the integrity or security of your installation with plugins enabled. + ## Other Customizations There are other areas of Malcolm that can be modified and customized to fit users' needs. Please see these other sections of the documentation for more information. From dda57187b588663d356c80f53e03dc552b044432 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Thu, 15 Aug 2024 13:01:51 -0600 Subject: [PATCH 31/47] handle BASE_PATH better in netbox thanks to this comment on github: https://github.com/netbox-community/netbox-docker/issues/650#issuecomment-2286358806 --- Dockerfiles/netbox.Dockerfile | 12 +++++------- config/netbox.env.example | 1 - netbox/config/configuration.py | 4 +--- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/Dockerfiles/netbox.Dockerfile b/Dockerfiles/netbox.Dockerfile index 7b5aa802b..51722f9ce 100644 --- a/Dockerfiles/netbox.Dockerfile +++ b/Dockerfiles/netbox.Dockerfile @@ -47,7 +47,6 @@ ARG NETBOX_CUSTOM_PLUGINS_PATH="/opt/netbox-custom-plugins" ARG NETBOX_CONFIG_PATH="/etc/netbox/config" ENV NETBOX_PATH /opt/netbox -ENV BASE_PATH netbox ENV NETBOX_DEVICETYPE_LIBRARY_IMPORT_PATH $NETBOX_DEVICETYPE_LIBRARY_IMPORT_PATH ENV NETBOX_DEFAULT_SITE $NETBOX_DEFAULT_SITE ENV NETBOX_CRON $NETBOX_CRON @@ -120,14 +119,13 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') "Django>=4.2.10,<5" \ paramiko \ pillow && \ - mkdir -p "${NETBOX_PATH}/netbox/${BASE_PATH}" "${NETBOX_CUSTOM_PLUGINS_PATH}/requirements" && \ - mv "${NETBOX_PATH}/netbox/static" "${NETBOX_PATH}/netbox/${BASE_PATH}/static" && \ - jq '. += { "settings": { "http": { "discard_unsafe_fields": false } } }' /etc/unit/nginx-unit.json | jq 'del(.listeners."[::]:8080")' | jq 'del(.listeners."[::]:8081")' | jq ".routes.main[0].match.uri = \"/${BASE_PATH}/static/*\"" > /etc/unit/nginx-unit-new.json && \ + mkdir -p "${NETBOX_PATH}/netbox/netbox" "${NETBOX_CUSTOM_PLUGINS_PATH}/requirements" && \ + jq '. += { "settings": { "http": { "discard_unsafe_fields": false } } }' /etc/unit/nginx-unit.json | jq 'del(.listeners."[::]:8080")' | jq 'del(.listeners."[::]:8081")' | jq '.routes.main[0].action.share = "`/opt/netbox/netbox${uri.substring(7)}`"' | jq '.routes.main[0].match.uri = "/netbox/static/*"' | jq '.routes.status[0].match.uri = "/netbox/status/*"' > /etc/unit/nginx-unit-new.json && \ mv /etc/unit/nginx-unit-new.json /etc/unit/nginx-unit.json && \ chmod 644 /etc/unit/nginx-unit.json && \ - tr -cd '\11\12\15\40-\176' < "${NETBOX_PATH}/netbox/${BASE_PATH}/configuration.py" > "${NETBOX_PATH}/netbox/${BASE_PATH}/configuration_ascii.py" && \ - mv "${NETBOX_PATH}/netbox/${BASE_PATH}/configuration_ascii.py" "${NETBOX_PATH}/netbox/${BASE_PATH}/configuration.py" && \ - sed -i "s/\('CENSUS_REPORTING_ENABLED',[[:space:]]*\)True/\1False/" "${NETBOX_PATH}/netbox/${BASE_PATH}/settings.py" && \ + tr -cd '\11\12\15\40-\176' < "${NETBOX_PATH}/netbox/netbox/configuration.py" > "${NETBOX_PATH}/netbox/netbox/configuration_ascii.py" && \ + mv "${NETBOX_PATH}/netbox/netbox/configuration_ascii.py" "${NETBOX_PATH}/netbox/netbox/configuration.py" && \ + sed -i "s/\('CENSUS_REPORTING_ENABLED',[[:space:]]*\)True/\1False/" "${NETBOX_PATH}/netbox/netbox/settings.py" && \ sed -i -E 's@^([[:space:]]*\-\-(state|tmp))([[:space:]])@\1dir\3@g' "${NETBOX_PATH}/launch-netbox.sh" && \ sed -i '/\/opt\/netbox\/venv\/bin\/activate/a \\n# Install custom plugins \npython3 /usr/local/bin/netbox_install_plugins.py' /opt/netbox/docker-entrypoint.sh diff --git a/config/netbox.env.example b/config/netbox.env.example index 11b0d3622..e6cce09ff 100644 --- a/config/netbox.env.example +++ b/config/netbox.env.example @@ -3,7 +3,6 @@ CORS_ORIGIN_ALLOW_ALL=True # (see https://docs.djangoproject.com/en/4.1/ref/settings/#csrf-trusted-origins) # CSRF_TRUSTED_ORIGINS=https://malcolm.example.org CSRF_TRUSTED_ORIGINS=http://* https://* -BASE_PATH=netbox REMOTE_AUTH_ENABLED=True REMOTE_AUTH_BACKEND=netbox.authentication.RemoteUserBackend REMOTE_AUTH_HEADER=HTTP_X_REMOTE_AUTH diff --git a/netbox/config/configuration.py b/netbox/config/configuration.py index 36cb63758..b1c462204 100644 --- a/netbox/config/configuration.py +++ b/netbox/config/configuration.py @@ -142,9 +142,7 @@ def _environ_get_and_map(variable_name: str, default: str | None = None, map_fn: BANNER_LOGIN = environ.get('BANNER_LOGIN', None) # Base URL path if accessing NetBox within a directory. For example, if installed at http://example.com/netbox/, set: -# BASE_PATH = 'netbox/' -if 'BASE_PATH' in environ: - BASE_PATH = environ.get('BASE_PATH', '') +BASE_PATH = 'netbox/' # Maximum number of days to retain logged changes. Set to 0 to retain changes indefinitely. (Default: 90) if 'CHANGELOG_RETENTION' in environ: From 9dee43bd276abd9bad2b64e2f843a4538e7e5640 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Thu, 15 Aug 2024 15:03:22 -0600 Subject: [PATCH 32/47] idaholab/Malcolm#542 add 'public' pseudo-segment for source/destination public IP addresses --- .../dashboards/677ee170-809e-11ed-8d5b-07069f823b6f.json | 2 +- logstash/pipelines/enrichment/11_lookups.conf | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/dashboards/dashboards/677ee170-809e-11ed-8d5b-07069f823b6f.json b/dashboards/dashboards/677ee170-809e-11ed-8d5b-07069f823b6f.json index 8a589ef6e..434969cba 100644 --- a/dashboards/dashboards/677ee170-809e-11ed-8d5b-07069f823b6f.json +++ b/dashboards/dashboards/677ee170-809e-11ed-8d5b-07069f823b6f.json @@ -150,7 +150,7 @@ "description": "", "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"source.segment.id:* OR destination.segment.id:*\",\"language\":\"lucene\"},\"filter\":[]}" + "searchSourceJSON": "{\"query\":{\"query\":\"source.segment:* OR destination.segment:*\",\"language\":\"lucene\"},\"filter\":[]}" }, "savedSearchRefName": "search_0" }, diff --git a/logstash/pipelines/enrichment/11_lookups.conf b/logstash/pipelines/enrichment/11_lookups.conf index c757561b8..042285e29 100644 --- a/logstash/pipelines/enrichment/11_lookups.conf +++ b/logstash/pipelines/enrichment/11_lookups.conf @@ -260,12 +260,19 @@ filter { } else if ("internal_source" in [tags]) and ("external_destination" in [tags]) { mutate { id => "mutate_add_field_metadata_network_direction_outbound" add_field => { "[network][direction]" => "outbound" } } + mutate { id => "mutate_add_field_direction_outbound_destination_segment" + add_field => { "[destination][segment][name]" => "public" } } } else if ("external_source" in [tags]) and ("internal_destination" in [tags]) { mutate { id => "mutate_add_field_metadata_network_direction_inbound" add_field => { "[network][direction]" => "inbound" } } + mutate { id => "mutate_add_field_direction_inbound_source_segment" + add_field => { "[source][segment][name]" => "public" } } } else if ("external_source" in [tags]) and ("external_destination" in [tags]) { mutate { id => "mutate_add_field_metadata_network_direction_external" add_field => { "[network][direction]" => "external" } } + mutate { id => "mutate_add_field_direction_external_segments" + add_field => { "[source][segment][name]" => "public" + "[destination][segment][name]" => "public" } } } else if ("external_source" in [tags]) or ("external_destination" in [tags]) { mutate { id => "mutate_add_field_metadata_network_direction_external_one_sided" add_field => { "[network][direction]" => "external" } } From ea014c253017a1bb1186779969f296b7056bf02f Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Thu, 15 Aug 2024 16:05:58 -0600 Subject: [PATCH 33/47] idaholab/Malcolm#542 add 'public' pseudo-segment for source/destination public IP addresses --- .../677ee170-809e-11ed-8d5b-07069f823b6f.json | 4 ++-- docs/asset-interaction-analysis.md | 2 ++ logstash/pipelines/enrichment/11_lookups.conf | 11 ++++------- logstash/pipelines/enrichment/21_netbox.conf | 10 ++++++++++ 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/dashboards/dashboards/677ee170-809e-11ed-8d5b-07069f823b6f.json b/dashboards/dashboards/677ee170-809e-11ed-8d5b-07069f823b6f.json index 434969cba..13919e684 100644 --- a/dashboards/dashboards/677ee170-809e-11ed-8d5b-07069f823b6f.json +++ b/dashboards/dashboards/677ee170-809e-11ed-8d5b-07069f823b6f.json @@ -641,7 +641,7 @@ "updated_at": "2024-06-12T16:50:47.700Z", "version": "WzQzMiwxXQ==", "attributes": { - "title": "NetBox Enrichment - Candidate Logs", + "title": "NetBox Enrichment - Logs", "description": "", "hits": 0, "columns": [ @@ -650,7 +650,7 @@ "sort": [], "version": 1, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"query\":\"(NOT event.provider:arkime) AND ((NOT event.provider:zeek) OR event.dataset:(conn OR notice OR weird OR signatures OR software OR known*)) AND (network.direction:(internal OR inbound OR outbound))\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + "searchSourceJSON": "{\"highlightAll\":false,\"version\":true,\"query\":{\"query\":\"tags:netbox\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" } }, "references": [ diff --git a/docs/asset-interaction-analysis.md b/docs/asset-interaction-analysis.md index db0caea89..b133e9632 100644 --- a/docs/asset-interaction-analysis.md +++ b/docs/asset-interaction-analysis.md @@ -53,6 +53,8 @@ For Malcolm's purposes, both physical devices and virtualized hosts will be stor NetBox has the concept of [sites](https://demo.netbox.dev/static/docs/core-functionality/sites-and-racks/). Sites can have overlapping IP address ranges. The site to associate with network traffic can be specified when [PCAP is uploaded](upload.md#Upload), when configuring [live analysis](live-analysis.md#LiveAnalysis), and when [configuring forwarding from Hedgehog Linux](malcolm-hedgehog-e2e-iso-install.md#Hedgehogfilebeat). If not otherwise specified, the value of the `NETBOX_DEFAULT_SITE` variable in [environment variable in `netbox-common.env`](malcolm-config.md#MalcolmConfigEnvVars) will be used for these enrichment lookups. +When NetBox enrichment is attempted for a log, the value `netbox` is automatically added to its `tags` field. + ## Compare and highlight discrepancies between NetBox inventory and observed network traffic As Malcolm cross-checks network traffic with NetBox's model (as described [above](#NetBoxEnrichment)), the resulting enrichment data (or lack thereof) can highlight devices and services observed in network traffic for which there is no corresponding entry in the list of inventoried assets. diff --git a/logstash/pipelines/enrichment/11_lookups.conf b/logstash/pipelines/enrichment/11_lookups.conf index 042285e29..337148f92 100644 --- a/logstash/pipelines/enrichment/11_lookups.conf +++ b/logstash/pipelines/enrichment/11_lookups.conf @@ -135,6 +135,8 @@ filter { mutate { id => "mutate_add_field_srcASN" add_field => { "[source][as][full]" => "AS%{[@metadata][orig_asn][asn]} %{[@metadata][orig_asn][as_org]}" } } } + mutate { id => "mutate_add_field_external_source_segment_public" + add_field => { "[source][segment][name]" => "public" } } # if reverse DNS for public IP addresses is enabled (via environment variable) do it mutate { @@ -208,6 +210,8 @@ filter { mutate { id => "mutate_add_field_dstASN" add_field => { "[destination][as][full]" => "AS%{[@metadata][resp_asn][asn]} %{[@metadata][resp_asn][as_org]}" } } } + mutate { id => "mutate_add_field_external_destination_segment_public" + add_field => { "[destination][segment][name]" => "public" } } # if reverse DNS for public IP addresses is enabled (via environment variable) do it if (![@metadata][ENV_LOGSTASH_REVERSE_DNS]) { @@ -260,19 +264,12 @@ filter { } else if ("internal_source" in [tags]) and ("external_destination" in [tags]) { mutate { id => "mutate_add_field_metadata_network_direction_outbound" add_field => { "[network][direction]" => "outbound" } } - mutate { id => "mutate_add_field_direction_outbound_destination_segment" - add_field => { "[destination][segment][name]" => "public" } } } else if ("external_source" in [tags]) and ("internal_destination" in [tags]) { mutate { id => "mutate_add_field_metadata_network_direction_inbound" add_field => { "[network][direction]" => "inbound" } } - mutate { id => "mutate_add_field_direction_inbound_source_segment" - add_field => { "[source][segment][name]" => "public" } } } else if ("external_source" in [tags]) and ("external_destination" in [tags]) { mutate { id => "mutate_add_field_metadata_network_direction_external" add_field => { "[network][direction]" => "external" } } - mutate { id => "mutate_add_field_direction_external_segments" - add_field => { "[source][segment][name]" => "public" - "[destination][segment][name]" => "public" } } } else if ("external_source" in [tags]) or ("external_destination" in [tags]) { mutate { id => "mutate_add_field_metadata_network_direction_external_one_sided" add_field => { "[network][direction]" => "external" } } diff --git a/logstash/pipelines/enrichment/21_netbox.conf b/logstash/pipelines/enrichment/21_netbox.conf index f90e6344d..ae2113eac 100644 --- a/logstash/pipelines/enrichment/21_netbox.conf +++ b/logstash/pipelines/enrichment/21_netbox.conf @@ -44,6 +44,7 @@ filter { ruby { id => "ruby_netbox_enrich_dns_ip_to_host" path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb" + add_tag => [ "netbox" ] script_params => { "lookup_type" => "ip_device" "source" => "[dns][resolved_ip]" @@ -70,6 +71,7 @@ filter { ruby { id => "ruby_netbox_enrich_ntlm_host_to_ip" path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb" + add_tag => [ "netbox" ] script_params => { "lookup_type" => "ip_device" "source" => "[source][ip]" @@ -95,6 +97,7 @@ filter { ruby { id => "ruby_netbox_enrich_ntlm_server_nb_computer_to_ip" path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb" + add_tag => [ "netbox" ] script_params => { "lookup_type" => "ip_device" "source" => "[destination][ip]" @@ -117,6 +120,7 @@ filter { ruby { id => "ruby_netbox_enrich_ntlm_server_dns_computer_to_ip" path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb" + add_tag => [ "netbox" ] script_params => { "lookup_type" => "ip_device" "source" => "[destination][ip]" @@ -145,6 +149,7 @@ filter { ruby { id => "ruby_netbox_enrich_dhcp_client_fqdn_to_ip" path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb" + add_tag => [ "netbox" ] script_params => { "lookup_type" => "ip_device" "source" => "[zeek][dhcp][assigned_addr]" @@ -167,6 +172,7 @@ filter { ruby { id => "ruby_netbox_enrich_dhcp_host_name_to_ip" path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb" + add_tag => [ "netbox" ] script_params => { "lookup_type" => "ip_device" "source" => "[zeek][dhcp][assigned_addr]" @@ -196,6 +202,7 @@ filter { ruby { id => "ruby_netbox_enrich_source_ip_segment" path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb" + add_tag => [ "netbox" ] script_params => { "source" => "[source][ip]" "target" => "[source][segment]" @@ -214,6 +221,7 @@ filter { ruby { id => "ruby_netbox_enrich_source_ip_device" path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb" + add_tag => [ "netbox" ] script_params => { "lookup_type" => "ip_device" "source" => "[source][ip]" @@ -243,6 +251,7 @@ filter { ruby { id => "ruby_netbox_enrich_destination_ip_segment" path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb" + add_tag => [ "netbox" ] script_params => { "lookup_type" => "ip_prefix" "source" => "[destination][ip]" @@ -261,6 +270,7 @@ filter { ruby { id => "ruby_netbox_enrich_destination_ip_device" path => "/usr/share/logstash/malcolm-ruby/netbox_enrich.rb" + add_tag => [ "netbox" ] script_params => { "lookup_type" => "ip_device" "source" => "[destination][ip]" From de8d9af026a4a9e2d0b9086dda0f09fcf3f3facc Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Thu, 15 Aug 2024 16:54:43 -0600 Subject: [PATCH 34/47] idaholab/Malcolm#542 add 'public' pseudo-segment for source/destination public IP addresses --- .../677ee170-809e-11ed-8d5b-07069f823b6f.json | 114 +++++++++++------- 1 file changed, 73 insertions(+), 41 deletions(-) diff --git a/dashboards/dashboards/677ee170-809e-11ed-8d5b-07069f823b6f.json b/dashboards/dashboards/677ee170-809e-11ed-8d5b-07069f823b6f.json index 13919e684..15f90535c 100644 --- a/dashboards/dashboards/677ee170-809e-11ed-8d5b-07069f823b6f.json +++ b/dashboards/dashboards/677ee170-809e-11ed-8d5b-07069f823b6f.json @@ -1,5 +1,5 @@ { - "version": "2.14.0", + "version": "2.16.0", "objects": [ { "id": "677ee170-809e-11ed-8d5b-07069f823b6f", @@ -7,13 +7,13 @@ "namespaces": [ "default" ], - "updated_at": "2024-06-12T17:30:00.382Z", - "version": "Wzk5NSwxXQ==", + "updated_at": "2024-08-15T22:10:04.425Z", + "version": "WzEwMTUsMV0=", "attributes": { "title": "Asset Interaction Analysis", "hits": 0, "description": "", - "panelsJSON": "[{\"version\":\"2.14.0\",\"gridData\":{\"x\":0,\"y\":0,\"w\":8,\"h\":38,\"i\":\"1\"},\"panelIndex\":\"1\",\"embeddableConfig\":{},\"panelRefName\":\"panel_0\"},{\"version\":\"2.14.0\",\"gridData\":{\"x\":8,\"y\":0,\"w\":40,\"h\":19,\"i\":\"a3985c0b-155d-4dbd-ae17-b620c23bcffd\"},\"panelIndex\":\"a3985c0b-155d-4dbd-ae17-b620c23bcffd\",\"embeddableConfig\":{\"table\":null,\"vis\":{\"params\":{\"sort\":{\"columnIndex\":6,\"direction\":\"desc\"}},\"sortColumn\":{\"colIndex\":4,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_1\"},{\"version\":\"2.14.0\",\"gridData\":{\"x\":8,\"y\":19,\"w\":15,\"h\":19,\"i\":\"be8823d0-07d0-4769-a3c3-45160ee47074\"},\"panelIndex\":\"be8823d0-07d0-4769-a3c3-45160ee47074\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"2.14.0\",\"gridData\":{\"x\":0,\"y\":38,\"w\":20,\"h\":26,\"i\":\"3b28bc9b-393b-4cf2-b31c-3510b0994281\"},\"panelIndex\":\"3b28bc9b-393b-4cf2-b31c-3510b0994281\",\"embeddableConfig\":{},\"panelRefName\":\"panel_3\"},{\"version\":\"2.14.0\",\"gridData\":{\"x\":23,\"y\":19,\"w\":25,\"h\":19,\"i\":\"6d7baf85-fc04-49be-942e-e61f1fd7f0d0\"},\"panelIndex\":\"6d7baf85-fc04-49be-942e-e61f1fd7f0d0\",\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":4,\"direction\":\"desc\"}},\"sortColumn\":{\"colIndex\":4,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_4\"},{\"version\":\"2.14.0\",\"gridData\":{\"x\":20,\"y\":38,\"w\":28,\"h\":19,\"i\":\"64f0e91b-4263-4400-812d-c378cd6d8565\"},\"panelIndex\":\"64f0e91b-4263-4400-812d-c378cd6d8565\",\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":4,\"direction\":\"desc\"}},\"sortColumn\":{\"colIndex\":3,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_5\"},{\"version\":\"2.14.0\",\"gridData\":{\"x\":0,\"y\":64,\"w\":20,\"h\":28,\"i\":\"2fe74242-3865-4c2e-87b4-ec2580f467eb\"},\"panelIndex\":\"2fe74242-3865-4c2e-87b4-ec2580f467eb\",\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":3,\"direction\":\"desc\"}},\"sortColumn\":{\"colIndex\":0,\"direction\":\"asc\"}}},\"panelRefName\":\"panel_6\"},{\"version\":\"2.14.0\",\"gridData\":{\"x\":20,\"y\":57,\"w\":28,\"h\":26,\"i\":\"d7a5cbcd-1907-4d98-8def-0e6ed61cfe51\"},\"panelIndex\":\"d7a5cbcd-1907-4d98-8def-0e6ed61cfe51\",\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":5,\"direction\":\"desc\"}},\"sortColumn\":{\"colIndex\":5,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_7\"},{\"version\":\"2.14.0\",\"gridData\":{\"x\":0,\"y\":92,\"w\":20,\"h\":28,\"i\":\"c1614a06-d5e7-4bba-93cc-e8fa63205e11\"},\"panelIndex\":\"c1614a06-d5e7-4bba-93cc-e8fa63205e11\",\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":3,\"direction\":\"desc\"}},\"sortColumn\":{\"colIndex\":0,\"direction\":\"asc\"}}},\"panelRefName\":\"panel_8\"},{\"version\":\"2.14.0\",\"gridData\":{\"x\":20,\"y\":83,\"w\":14,\"h\":37,\"i\":\"618fe049-9298-49e7-adf6-31da64a796ad\"},\"panelIndex\":\"618fe049-9298-49e7-adf6-31da64a796ad\",\"embeddableConfig\":{},\"panelRefName\":\"panel_9\"},{\"version\":\"2.14.0\",\"gridData\":{\"x\":34,\"y\":83,\"w\":14,\"h\":37,\"i\":\"707927bd-fd2e-48cd-a045-4db49fb90159\"},\"panelIndex\":\"707927bd-fd2e-48cd-a045-4db49fb90159\",\"embeddableConfig\":{},\"panelRefName\":\"panel_10\"},{\"version\":\"2.14.0\",\"gridData\":{\"x\":0,\"y\":120,\"w\":24,\"h\":22,\"i\":\"3c3e26f4-2485-4f09-a000-2ac61759bbc9\"},\"panelIndex\":\"3c3e26f4-2485-4f09-a000-2ac61759bbc9\",\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":4,\"direction\":\"desc\"}},\"sortColumn\":{\"colIndex\":4,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_11\"},{\"version\":\"2.14.0\",\"gridData\":{\"x\":24,\"y\":120,\"w\":24,\"h\":22,\"i\":\"fe600b29-f6af-4d33-b226-26743ced5290\"},\"panelIndex\":\"fe600b29-f6af-4d33-b226-26743ced5290\",\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":4,\"direction\":\"desc\"}},\"sortColumn\":{\"colIndex\":4,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_12\"},{\"version\":\"2.14.0\",\"gridData\":{\"x\":0,\"y\":142,\"w\":24,\"h\":18,\"i\":\"2abbd1dd-532b-4204-a44d-c2914f47a6fe\"},\"panelIndex\":\"2abbd1dd-532b-4204-a44d-c2914f47a6fe\",\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":3,\"direction\":\"desc\"}},\"sortColumn\":{\"colIndex\":3,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_13\"},{\"version\":\"2.14.0\",\"gridData\":{\"x\":24,\"y\":142,\"w\":24,\"h\":18,\"i\":\"9b1cb1de-e704-4de8-9db5-253492ab5557\"},\"panelIndex\":\"9b1cb1de-e704-4de8-9db5-253492ab5557\",\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":3,\"direction\":\"desc\"}},\"sortColumn\":{\"colIndex\":3,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_14\"},{\"version\":\"2.14.0\",\"gridData\":{\"x\":0,\"y\":160,\"w\":48,\"h\":33,\"i\":\"e85559bf-0ca0-4218-ac74-cad3c4b2d1c6\"},\"panelIndex\":\"e85559bf-0ca0-4218-ac74-cad3c4b2d1c6\",\"embeddableConfig\":{},\"panelRefName\":\"panel_15\"},{\"version\":\"2.14.0\",\"gridData\":{\"x\":0,\"y\":193,\"w\":48,\"h\":31,\"i\":\"aba4ed3d-64e5-435d-8b4d-80614ac32fca\"},\"panelIndex\":\"aba4ed3d-64e5-435d-8b4d-80614ac32fca\",\"embeddableConfig\":{},\"panelRefName\":\"panel_16\"}]", + "panelsJSON": "[{\"version\":\"2.16.0\",\"gridData\":{\"x\":0,\"y\":0,\"w\":8,\"h\":38,\"i\":\"1\"},\"panelIndex\":\"1\",\"embeddableConfig\":{},\"panelRefName\":\"panel_0\"},{\"version\":\"2.16.0\",\"gridData\":{\"x\":8,\"y\":0,\"w\":40,\"h\":19,\"i\":\"a3985c0b-155d-4dbd-ae17-b620c23bcffd\"},\"panelIndex\":\"a3985c0b-155d-4dbd-ae17-b620c23bcffd\",\"embeddableConfig\":{\"table\":null,\"vis\":{\"params\":{\"sort\":{\"columnIndex\":6,\"direction\":\"desc\"}},\"sortColumn\":{\"colIndex\":4,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_1\"},{\"version\":\"2.16.0\",\"gridData\":{\"x\":8,\"y\":19,\"w\":15,\"h\":19,\"i\":\"be8823d0-07d0-4769-a3c3-45160ee47074\"},\"panelIndex\":\"be8823d0-07d0-4769-a3c3-45160ee47074\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"2.16.0\",\"gridData\":{\"x\":0,\"y\":38,\"w\":20,\"h\":26,\"i\":\"3b28bc9b-393b-4cf2-b31c-3510b0994281\"},\"panelIndex\":\"3b28bc9b-393b-4cf2-b31c-3510b0994281\",\"embeddableConfig\":{},\"panelRefName\":\"panel_3\"},{\"version\":\"2.16.0\",\"gridData\":{\"x\":23,\"y\":19,\"w\":25,\"h\":19,\"i\":\"6d7baf85-fc04-49be-942e-e61f1fd7f0d0\"},\"panelIndex\":\"6d7baf85-fc04-49be-942e-e61f1fd7f0d0\",\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":4,\"direction\":\"desc\"}},\"sortColumn\":{\"colIndex\":4,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_4\"},{\"version\":\"2.16.0\",\"gridData\":{\"x\":20,\"y\":38,\"w\":28,\"h\":19,\"i\":\"64f0e91b-4263-4400-812d-c378cd6d8565\"},\"panelIndex\":\"64f0e91b-4263-4400-812d-c378cd6d8565\",\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":4,\"direction\":\"desc\"}},\"sortColumn\":{\"colIndex\":3,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_5\"},{\"version\":\"2.16.0\",\"gridData\":{\"x\":0,\"y\":64,\"w\":20,\"h\":28,\"i\":\"2fe74242-3865-4c2e-87b4-ec2580f467eb\"},\"panelIndex\":\"2fe74242-3865-4c2e-87b4-ec2580f467eb\",\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":3,\"direction\":\"desc\"}},\"sortColumn\":{\"colIndex\":0,\"direction\":\"asc\"}}},\"panelRefName\":\"panel_6\"},{\"version\":\"2.16.0\",\"gridData\":{\"x\":20,\"y\":57,\"w\":28,\"h\":26,\"i\":\"d7a5cbcd-1907-4d98-8def-0e6ed61cfe51\"},\"panelIndex\":\"d7a5cbcd-1907-4d98-8def-0e6ed61cfe51\",\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":5,\"direction\":\"desc\"}},\"sortColumn\":{\"colIndex\":5,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_7\"},{\"version\":\"2.16.0\",\"gridData\":{\"x\":0,\"y\":92,\"w\":20,\"h\":28,\"i\":\"c1614a06-d5e7-4bba-93cc-e8fa63205e11\"},\"panelIndex\":\"c1614a06-d5e7-4bba-93cc-e8fa63205e11\",\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":3,\"direction\":\"desc\"}},\"sortColumn\":{\"colIndex\":0,\"direction\":\"asc\"}}},\"panelRefName\":\"panel_8\"},{\"version\":\"2.16.0\",\"gridData\":{\"x\":20,\"y\":83,\"w\":14,\"h\":37,\"i\":\"618fe049-9298-49e7-adf6-31da64a796ad\"},\"panelIndex\":\"618fe049-9298-49e7-adf6-31da64a796ad\",\"embeddableConfig\":{},\"panelRefName\":\"panel_9\"},{\"version\":\"2.16.0\",\"gridData\":{\"x\":34,\"y\":83,\"w\":14,\"h\":37,\"i\":\"707927bd-fd2e-48cd-a045-4db49fb90159\"},\"panelIndex\":\"707927bd-fd2e-48cd-a045-4db49fb90159\",\"embeddableConfig\":{},\"panelRefName\":\"panel_10\"},{\"version\":\"2.16.0\",\"gridData\":{\"x\":0,\"y\":120,\"w\":24,\"h\":22,\"i\":\"3c3e26f4-2485-4f09-a000-2ac61759bbc9\"},\"panelIndex\":\"3c3e26f4-2485-4f09-a000-2ac61759bbc9\",\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":4,\"direction\":\"desc\"}},\"sortColumn\":{\"colIndex\":4,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_11\"},{\"version\":\"2.16.0\",\"gridData\":{\"x\":24,\"y\":120,\"w\":24,\"h\":22,\"i\":\"fe600b29-f6af-4d33-b226-26743ced5290\"},\"panelIndex\":\"fe600b29-f6af-4d33-b226-26743ced5290\",\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":4,\"direction\":\"desc\"}},\"sortColumn\":{\"colIndex\":4,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_12\"},{\"version\":\"2.16.0\",\"gridData\":{\"x\":0,\"y\":142,\"w\":24,\"h\":18,\"i\":\"2abbd1dd-532b-4204-a44d-c2914f47a6fe\"},\"panelIndex\":\"2abbd1dd-532b-4204-a44d-c2914f47a6fe\",\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":3,\"direction\":\"desc\"}},\"sortColumn\":{\"colIndex\":3,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_13\"},{\"version\":\"2.16.0\",\"gridData\":{\"x\":24,\"y\":142,\"w\":24,\"h\":18,\"i\":\"9b1cb1de-e704-4de8-9db5-253492ab5557\"},\"panelIndex\":\"9b1cb1de-e704-4de8-9db5-253492ab5557\",\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":3,\"direction\":\"desc\"}},\"sortColumn\":{\"colIndex\":3,\"direction\":\"desc\"}}},\"panelRefName\":\"panel_14\"},{\"version\":\"2.16.0\",\"gridData\":{\"x\":0,\"y\":160,\"w\":48,\"h\":33,\"i\":\"e85559bf-0ca0-4218-ac74-cad3c4b2d1c6\"},\"panelIndex\":\"e85559bf-0ca0-4218-ac74-cad3c4b2d1c6\",\"embeddableConfig\":{},\"panelRefName\":\"panel_15\"},{\"version\":\"2.16.0\",\"gridData\":{\"x\":0,\"y\":193,\"w\":48,\"h\":31,\"i\":\"aba4ed3d-64e5-435d-8b4d-80614ac32fca\"},\"panelIndex\":\"aba4ed3d-64e5-435d-8b4d-80614ac32fca\",\"embeddableConfig\":{},\"panelRefName\":\"panel_16\"}]", "optionsJSON": "{\"useMargins\":true}", "version": 1, "timeRestore": false, @@ -118,8 +118,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-06-12T16:51:26.344Z", - "version": "Wzg2MSwxXQ==", + "updated_at": "2024-08-15T21:57:29.068Z", + "version": "Wzg4NywxXQ==", "attributes": { "title": "Navigation", "visState": "{\"title\":\"Navigation\",\"type\":\"markdown\",\"params\":{\"markdown\":\"### General Network Logs\\n[Overview](#/dashboard/0ad3d7c2-3441-485e-9dfe-dbb22e84e576) \\n[Security Overview](#/dashboard/95479950-41f2-11ea-88fa-7151df485405) \\n[ICS/IoT Security Overview](#/dashboard/4a4bde20-4760-11ea-949c-bbb5a9feecbf) \\n[Severity](#/dashboard/d2dd0180-06b1-11ec-8c6b-353266ade330) \\n[Connections](#/dashboard/abdd7550-2c7c-40dc-947e-f6d186a158c4) \\n[Actions and Results](#/dashboard/a33e0a50-afcd-11ea-993f-b7d8522a8bed) \\n[Files](#/dashboard/9ee51f94-3316-4fc5-bd89-93a52af69714) \\n[Executables](#/dashboard/0a490422-0ce9-44bf-9a2d-19329ddde8c3) \\n[Software](#/dashboard/87d990cc-9e0b-41e5-b8fe-b10ae1da0c85) \\n[Zeek Known Summary](#/dashboard/89d1cc50-974c-11ed-bb6b-3fb06c879b11) \\n[Zeek Intelligence](#/dashboard/36ed695f-edcc-47c1-b0ec-50d20c93ce0f) \\n[Zeek Notices](#/dashboard/f1f09567-fc7f-450b-a341-19d2f2bb468b) \\n[Zeek Weird](#/dashboard/1fff49f6-0199-4a0f-820b-721aff9ff1f1) \\n[Signatures](#/dashboard/665d1610-523d-11e9-a30e-e3576242f3ed) \\n[Suricata Alerts](#/dashboard/5694ca60-cbdf-11ec-a50a-5fedd672f5c5) \\n[Asset Interaction Analysis](#/dashboard/677ee170-809e-11ed-8d5b-07069f823b6f) \\n[↪ NetBox](/netbox/) \\n[↪ Arkime](/arkime/) \\n\\n### Common Protocols\\n[DCE/RPC](#/dashboard/432af556-c5c0-4cc3-8166-b274b4e3a406) ● [DHCP](#/dashboard/2d98bb8e-214c-4374-837b-20e1bcd63a5e) ● [DNS](#/dashboard/2cf94cd0-ecab-40a5-95a7-8419f3a39cd9) ● [FTP](#/dashboard/078b9aa5-9bd4-4f02-ae5e-cf80fa6f887b) / [TFTP](#/dashboard/bf5efbb0-60f1-11eb-9d60-dbf0411cfc48) ● [HTTP](#/dashboard/37041ee1-79c0-4684-a436-3173b0e89876) ● [IRC](#/dashboard/76f2f912-80da-44cd-ab66-6a73c8344cc3) ● [Kerberos](#/dashboard/82da3101-2a9c-4ae2-bb61-d447a3fbe673) ● [LDAP](#/dashboard/05e3e000-f118-11e9-acda-83a8e29e1a24) ● [MQTT](#/dashboard/87a32f90-ef58-11e9-974e-9d600036d105) ● [MySQL](#/dashboard/50ced171-1b10-4c3f-8b67-2db9635661a6) ● [NTLM](#/dashboard/543118a9-02d7-43fe-b669-b8652177fc37) ● [NTP](#/dashboard/af5df620-eeb6-11e9-bdef-65a192b7f586) ● [OSPF](#/dashboard/1cc01ff0-5205-11ec-a62c-7bc80e88f3f0) ● [QUIC](#/dashboard/11ddd980-e388-11e9-b568-cf17de8e860c) ● [RADIUS](#/dashboard/ae79b7d1-4281-4095-b2f6-fa7eafda9970) ● [RDP](#/dashboard/7f41913f-cba8-43f5-82a8-241b7ead03e0) ● [RFB](#/dashboard/f77bf097-18a8-465c-b634-eb2acc7a4f26) ● [SIP](#/dashboard/0b2354ae-0fe9-4fd9-b156-1c3870e5c7aa) ● [SMB](#/dashboard/42e831b9-41a9-4f35-8b7d-e1566d368773) ● [SMTP](#/dashboard/bb827f8e-639e-468c-93c8-9f5bc132eb8f) ● [SNMP](#/dashboard/4e5f106e-c60a-4226-8f64-d534abb912ab) ● [SSH](#/dashboard/caef3ade-d289-4d05-a511-149f3e97f238) ● [SSL](#/dashboard/7f77b58a-df3e-4cc2-b782-fd7f8bad8ffb) / [X.509 Certificates](#/dashboard/024062a6-48d6-498f-a91a-3bf2da3a3cd3) ● [STUN](#/dashboard/fa477130-2b8a-11ec-a9f2-3911c8571bfd) ● [Syslog](#/dashboard/92985909-dc29-4533-9e80-d3182a0ecf1d) ● [TDS](#/dashboard/bed185a0-ef82-11e9-b38a-2db3ee640e88) / [TDS RPC](#/dashboard/32587740-ef88-11e9-b38a-2db3ee640e88) / [TDS SQL](#/dashboard/fa141950-ef89-11e9-b38a-2db3ee640e88) ● [Telnet / rlogin / rsh](#/dashboard/c2549e10-7f2e-11ea-9f8a-1fe1327e2cd2) ● [Tunnels](#/dashboard/11be6381-beef-40a7-bdce-88c5398392fc)\\n\\n### ICS/IoT Protocols\\n[BACnet](#/dashboard/2bec1490-eb94-11e9-a384-0fcf32210194) ● [BSAP](#/dashboard/ca5799a0-56b5-11eb-b749-576de068f8ad) ● [DNP3](#/dashboard/870a5862-6c26-4a08-99fd-0c06cda85ba3) ● [EtherCAT](#/dashboard/4a073440-b286-11eb-a4d4-09fa12a6ebd4) ● [EtherNet/IP](#/dashboard/29a1b290-eb98-11e9-a384-0fcf32210194) ● [GENISYS](#/dashboard/03207c00-d07e-11ec-b4a7-d1b4003706b7) ● [GE SRTP](#/dashboard/e233a570-45d9-11ef-96a6-432365601033) ● [Modbus](#/dashboard/152f29dc-51a2-4f53-93e9-6e92765567b8) ● [OPCUA Binary](#/dashboard/dd87edd0-796a-11ec-9ce6-b395c1ff58f4) ● [PROFINET](#/dashboard/a7514350-eba6-11e9-a384-0fcf32210194) ● [S7comm](#/dashboard/e76d05c0-eb9f-11e9-a384-0fcf32210194) ● [Synchrophasor](#/dashboard/2cc56240-e460-11ed-a9d5-9f591c284cb4) ● [Best Guess](#/dashboard/12e3a130-d83b-11eb-a0b0-f328ce09b0b7)\\n\\n### Malcolm and Third-Party Logs\\n\\nResources: [System Overview](#/dashboard/Metricbeat-system-overview-ecs) / [Host Overview](#/dashboard/79ffd6e0-faa0-11e6-947f-177f697178b8-ecs) ● [Hardware Temperature](#/dashboard/0d4955f0-eb25-11ec-a6d4-b3526526c2c7) ● nginx [Overview](#/dashboard/55a9e6e0-a29e-11e7-928f-5dbe6f6f5519-ecs) / [Access and Error Logs](#/dashboard/046212a0-a2a1-11e7-928f-5dbe6f6f5519-ecs) ● Linux [Journald](#/dashboard/f6600310-9943-11ee-a029-e973f4774355) / [Kernel Messages](#/dashboard/3768ef70-d819-11ee-820d-dd9fd73a3921) ● [Windows Events](#/dashboard/79202ee0-d811-11ee-820d-dd9fd73a3921) ● [Malcolm Sensor File Integrity](#/dashboard/903f42c0-f634-11ec-828d-2fb7a4a26e1f) ● [Malcolm Sensor Audit Logs](#/dashboard/7a7e0a60-e8e8-11ec-b9d4-4569bb965430) ● [Packet Capture Statistics](#/dashboard/4ca94c70-d7da-11ee-9ed3-e7afff29e59a)\",\"type\":\"markdown\",\"fontSize\":10,\"openLinksInNewTab\":false},\"aggs\":[]}", @@ -141,8 +141,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-06-12T17:24:41.935Z", - "version": "Wzk5MCwxXQ==", + "updated_at": "2024-08-15T21:56:49.347Z", + "version": "WzQyMywxXQ==", "attributes": { "title": "Traffic by Network Segment", "visState": "{\"title\":\"Traffic by Network Segment\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{\"customLabel\":\"Log Count\"},\"schema\":\"metric\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"related.site\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Site\"},\"schema\":\"bucket\"},{\"id\":\"6\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"network.direction\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"unknown\",\"customLabel\":\"Direction\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"source.segment.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Source Segment\"},\"schema\":\"bucket\"},{\"id\":\"5\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"destination.segment.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Destination Segment\"},\"schema\":\"bucket\"},{\"id\":\"7\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"network.packets\",\"customLabel\":\"Total Packets\"},\"schema\":\"metric\"},{\"id\":\"8\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"network.bytes\",\"customLabel\":\"Total Bytes\"},\"schema\":\"metric\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", @@ -158,7 +158,7 @@ { "name": "search_0", "type": "search", - "id": "b1645e70-8182-11ed-8ccb-af218b39580f" + "id": "cde50940-5b52-11ef-b245-313a0ac785d6" } ], "migrationVersion": { @@ -171,8 +171,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-06-12T17:20:20.975Z", - "version": "Wzk4OCwxXQ==", + "updated_at": "2024-08-15T21:56:49.347Z", + "version": "WzQyNCwxXQ==", "attributes": { "title": "Network Site", "visState": "{\"title\":\"Network Site\",\"type\":\"pie\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"related.site\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Unknown\",\"customLabel\":\"Site\"},\"schema\":\"segment\"}],\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"isDonut\":true,\"labels\":{\"show\":true,\"values\":false,\"last_level\":true,\"truncate\":100}}}", @@ -200,8 +200,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-06-12T16:50:47.700Z", - "version": "WzQzMSwxXQ==", + "updated_at": "2024-08-15T21:56:49.347Z", + "version": "WzQyNSwxXQ==", "attributes": { "title": "Cross Segment Traffic", "visState": "{\"title\":\"Cross Segment Traffic\",\"type\":\"vega\",\"aggs\":[],\"params\":{\"spec\":\"{\\n // thanks to:\\n // - https://www.elastic.co/blog/sankey-visualization-with-vega-in-kibana\\n // - https://blog.davidvassallo.me/2023/09/08/adding-opensearch-dashboards-kibana-filters-to-vega-visuals/\\n $schema: https://vega.github.io/schema/vega/v3.0.json\\n data: [\\n {\\n // query ES based on the currently selected time range and filter string\\n name: rawData\\n url: {\\n %context%: true\\n %timefield%: MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER\\n index: MALCOLM_NETWORK_INDEX_PATTERN_REPLACER\\n body: {\\n size: 0\\n aggs: {\\n table: {\\n composite: {\\n size: 10000\\n sources: [\\n {\\n stk1: {\\n terms: {field: \\\"source.segment.name\\\"}\\n }\\n }\\n {\\n stk2: {\\n terms: {field: \\\"destination.segment.name\\\"}\\n }\\n }\\n ]\\n }\\n }\\n }\\n }\\n }\\n // From the result, take just the data we are interested in\\n format: {property: \\\"aggregations.table.buckets\\\"}\\n // Convert key.stk1 -> stk1 for simpler access below\\n transform: [\\n {type: \\\"formula\\\", expr: \\\"datum.key.stk1\\\", as: \\\"stk1\\\"}\\n {type: \\\"formula\\\", expr: \\\"datum.key.stk2\\\", as: \\\"stk2\\\"}\\n {type: \\\"formula\\\", expr: \\\"datum.doc_count\\\", as: \\\"size\\\"}\\n ]\\n }\\n {\\n name: nodes\\n source: rawData\\n transform: [\\n // when a value is selected, filter out unrelated data\\n {\\n type: filter\\n expr: !groupSelector || groupSelector.stk1 == datum.stk1 || groupSelector.stk2 == datum.stk2\\n }\\n // Set new key for later lookups - identifies each node\\n {type: \\\"formula\\\", expr: \\\"datum.stk1+datum.stk2\\\", as: \\\"key\\\"}\\n // instead of each table row, create two new rows,\\n // one for the source (stack=stk1) and one for destination node (stack=stk2).\\n // The values stored in stk1 and stk2 fields is placed into grpId field.\\n {\\n type: fold\\n fields: [\\\"stk1\\\", \\\"stk2\\\"]\\n as: [\\\"stack\\\", \\\"grpId\\\"]\\n }\\n // Create a sortkey, different for stk1 and stk2 stacks.\\n // Space separator ensures proper sort order in some corner cases.\\n {\\n type: formula\\n expr: datum.stack == 'stk1' ? datum.stk1+' '+datum.stk2 : datum.stk2+' '+datum.stk1\\n as: sortField\\n }\\n // Calculate y0 and y1 positions for stacking nodes one on top of the other,\\n // independently for each stack, and ensuring they are in the proper order,\\n // alphabetical from the top (reversed on the y axis)\\n {\\n type: stack\\n groupby: [\\\"stack\\\"]\\n sort: {field: \\\"sortField\\\", order: \\\"descending\\\"}\\n field: size\\n }\\n // calculate vertical center point for each node, used to draw edges\\n {type: \\\"formula\\\", expr: \\\"(datum.y0+datum.y1)/2\\\", as: \\\"yc\\\"}\\n ]\\n }\\n {\\n name: groups\\n source: nodes\\n transform: [\\n // combine all nodes into groups, summing up the doc counts\\n {\\n type: aggregate\\n groupby: [\\\"stack\\\", \\\"grpId\\\"]\\n fields: [\\\"size\\\"]\\n ops: [\\\"sum\\\"]\\n as: [\\\"total\\\"]\\n }\\n // re-calculate the stacking y0,y1 values\\n {\\n type: stack\\n groupby: [\\\"stack\\\"]\\n sort: {field: \\\"grpId\\\", order: \\\"descending\\\"}\\n field: total\\n }\\n // project y0 and y1 values to screen coordinates\\n // doing it once here instead of doing it several times in marks\\n {type: \\\"formula\\\", expr: \\\"scale('y', datum.y0)\\\", as: \\\"scaledY0\\\"}\\n {type: \\\"formula\\\", expr: \\\"scale('y', datum.y1)\\\", as: \\\"scaledY1\\\"}\\n // boolean flag if the label should be on the right of the stack\\n {type: \\\"formula\\\", expr: \\\"datum.stack == 'stk1'\\\", as: \\\"rightLabel\\\"}\\n // Calculate percentage for this value using \\\"y\\\" scale\\n // domain upper bound, which represents the total\\n {\\n type: formula\\n expr: datum.total/domain('y')[1]\\n as: percentage\\n }\\n ]\\n }\\n {\\n // This is a temp lookup table with all the 'stk2' stack nodes\\n name: destinationNodes\\n source: nodes\\n transform: [\\n {type: \\\"filter\\\", expr: \\\"datum.stack == 'stk2'\\\"}\\n ]\\n }\\n {\\n name: edges\\n source: nodes\\n transform: [\\n // we only want nodes from the left stack\\n {type: \\\"filter\\\", expr: \\\"datum.stack == 'stk1'\\\"}\\n // find corresponding node from the right stack, keep it as \\\"target\\\"\\n {\\n type: lookup\\n from: destinationNodes\\n key: key\\n fields: [\\\"key\\\"]\\n as: [\\\"target\\\"]\\n }\\n // calculate SVG link path between stk1 and stk2 stacks for the node pair\\n {\\n type: linkpath\\n orient: horizontal\\n shape: diagonal\\n sourceY: {expr: \\\"scale('y', datum.yc)\\\"}\\n sourceX: {expr: \\\"scale('x', 'stk1') + bandwidth('x')\\\"}\\n targetY: {expr: \\\"scale('y', datum.target.yc)\\\"}\\n targetX: {expr: \\\"scale('x', 'stk2')\\\"}\\n }\\n // A little trick to calculate the thickness of the line.\\n // The value needs to be the same as the hight of the node, but scaling\\n // size to screen's height gives inversed value because screen's Y\\n // coordinate goes from the top to the bottom, whereas the graph's Y=0\\n // is at the bottom. So subtracting scaled doc count from screen height\\n // (which is the \\\"lower\\\" bound of the \\\"y\\\" scale) gives us the right value\\n {\\n type: formula\\n expr: range('y')[0]-scale('y', datum.size)\\n as: strokeWidth\\n }\\n // Tooltip needs individual link's percentage of all values\\n {\\n type: formula\\n expr: datum.size/domain('y')[1]\\n as: percentage\\n }\\n ]\\n }\\n ]\\n scales: [\\n {\\n // calculates horizontal stack positioning\\n name: x\\n type: band\\n range: width\\n domain: [\\\"stk1\\\", \\\"stk2\\\"]\\n paddingOuter: 0.05\\n paddingInner: 0.95\\n }\\n {\\n // this scale goes up as high as the highest y1 value of all nodes\\n name: y\\n type: linear\\n range: height\\n domain: {data: \\\"nodes\\\", field: \\\"y1\\\"}\\n }\\n {\\n // use rawData to ensure the colors stay the same when clicking.\\n name: color\\n type: ordinal\\n range: category\\n domain: {data: \\\"rawData\\\", fields: [\\\"stk1\\\", \\\"stk2\\\"]}\\n }\\n {\\n // this scale is used to map internal ids (stk1, stk2) to stack names\\n name: stackNames\\n type: ordinal\\n range: [\\\"Source Segment\\\", \\\"Destination Segment\\\"]\\n domain: [\\\"stk1\\\", \\\"stk2\\\"]\\n }\\n ]\\n axes: [\\n {\\n // x axis should use custom label formatting to print proper stack names\\n orient: bottom\\n scale: x\\n encode: {\\n labels: {\\n update: {\\n text: {scale: \\\"stackNames\\\", field: \\\"value\\\"}\\n }\\n }\\n }\\n }\\n {orient: \\\"left\\\", scale: \\\"y\\\"}\\n ]\\n marks: [\\n {\\n // draw the connecting line between stacks\\n type: path\\n name: edgeMark\\n from: {data: \\\"edges\\\"}\\n // this prevents some autosizing issues with large strokeWidth for paths\\n clip: true\\n encode: {\\n update: {\\n // By default use color of the left node, except when showing contributors\\n // from just one value, in which case use destination color.\\n stroke: [\\n {\\n test: groupSelector && groupSelector.stack=='stk1'\\n scale: color\\n field: stk2\\n }\\n {scale: \\\"color\\\", field: \\\"stk1\\\"}\\n ]\\n strokeWidth: {field: \\\"strokeWidth\\\"}\\n path: {field: \\\"path\\\"}\\n // when showing all data, and hovering over a value,\\n // highlight the contributors for that value\\n strokeOpacity: {\\n signal: !groupSelector && (groupHover.stk1 == datum.stk1 || groupHover.stk2 == datum.stk2) ? 0.9 : 0.3\\n }\\n // Ensure that the hover-selected edges show on top\\n zindex: {\\n signal: !groupSelector && (groupHover.stk1 == datum.stk1 || groupHover.stk2 == datum.stk2) ? 1 : 0\\n }\\n // format tooltip string\\n tooltip: {\\n signal: datum.stk1 + ' → ' + datum.stk2 + ' ' + format(datum.size, ',.0f') + ' (' + format(datum.percentage, '.1%') + ')'\\n }\\n }\\n // Simple mouseover highlighting of a single line\\n hover: {\\n strokeOpacity: {value: 1}\\n }\\n }\\n }\\n {\\n // draw stack groups (countries)\\n type: rect\\n name: groupMark\\n from: {data: \\\"groups\\\"}\\n encode: {\\n enter: {\\n fill: {scale: \\\"color\\\", field: \\\"grpId\\\"}\\n width: {scale: \\\"x\\\", band: 1}\\n }\\n update: {\\n x: {scale: \\\"x\\\", field: \\\"stack\\\"}\\n y: {field: \\\"scaledY0\\\"}\\n y2: {field: \\\"scaledY1\\\"}\\n fillOpacity: {value: 0.6}\\n tooltip: {\\n signal: datum.grpId + ' ' + format(datum.total, ',.0f') + ' (' + format(datum.percentage, '.1%') + ')'\\n }\\n }\\n hover: {\\n fillOpacity: {value: 1}\\n }\\n }\\n }\\n {\\n // draw labels on the inner side of the stack\\n type: text\\n from: {data: \\\"groups\\\"}\\n // don't process events for the labels - otherwise line mouseover is unclean\\n interactive: false\\n encode: {\\n update: {\\n // depending on which stack it is, position x with some padding\\n x: {\\n signal: scale('x', datum.stack) + (datum.rightLabel ? bandwidth('x') + 8 : -8)\\n }\\n // middle of the group\\n yc: {signal: \\\"(datum.scaledY0 + datum.scaledY1)/2\\\"}\\n align: {signal: \\\"datum.rightLabel ? 'left' : 'right'\\\"}\\n baseline: {value: \\\"middle\\\"}\\n fontWeight: {value: \\\"bold\\\"}\\n // only show text label if the group's height is large enough\\n text: {signal: \\\"abs(datum.scaledY0-datum.scaledY1) > 13 ? datum.grpId : ''\\\"}\\n }\\n }\\n }\\n {\\n // Create a \\\"show all\\\" button. Shown only when a value is selected.\\n type: group\\n data: [\\n // We need to make the button show only when groupSelector signal is true.\\n // Each mark is drawn as many times as there are elements in the backing data.\\n // Which means that if values list is empty, it will not be drawn.\\n // Here I create a data source with one empty object, and filter that list\\n // based on the signal value. This can only be done in a group.\\n {\\n name: dataForShowAll\\n values: [{}]\\n transform: [{type: \\\"filter\\\", expr: \\\"groupSelector\\\"}]\\n }\\n ]\\n // Set button size and positioning\\n encode: {\\n enter: {\\n xc: {signal: \\\"width/2\\\"}\\n y: {value: 30}\\n width: {value: 80}\\n height: {value: 30}\\n }\\n }\\n marks: [\\n {\\n // This group is shown as a button with rounded corners.\\n type: group\\n // mark name allows signal capturing\\n name: groupReset\\n // Only shows button if dataForShowAll has values.\\n from: {data: \\\"dataForShowAll\\\"}\\n encode: {\\n enter: {\\n cornerRadius: {value: 6}\\n fill: {value: \\\"#f5f5f5\\\"}\\n stroke: {value: \\\"#c1c1c1\\\"}\\n strokeWidth: {value: 2}\\n // use parent group's size\\n height: {\\n field: {group: \\\"height\\\"}\\n }\\n width: {\\n field: {group: \\\"width\\\"}\\n }\\n }\\n update: {\\n // groups are transparent by default\\n opacity: {value: 1}\\n }\\n hover: {\\n opacity: {value: 0.7}\\n }\\n }\\n marks: [\\n {\\n type: text\\n // if true, it will prevent clicking on the button when over text.\\n interactive: false\\n encode: {\\n enter: {\\n // center text in the paren group\\n xc: {\\n field: {group: \\\"width\\\"}\\n mult: 0.5\\n }\\n yc: {\\n field: {group: \\\"height\\\"}\\n mult: 0.5\\n offset: 2\\n }\\n align: {value: \\\"center\\\"}\\n baseline: {value: \\\"middle\\\"}\\n fontWeight: {value: \\\"bold\\\"}\\n text: {value: \\\"Show All\\\"}\\n }\\n }\\n }\\n ]\\n }\\n ]\\n }\\n ]\\n signals: [\\n {\\n // used to highlight data to/from the same value\\n name: groupHover\\n value: {}\\n on: [\\n {\\n events: @groupMark:mouseover\\n update: \\\"{stk1:datum.stack=='stk1' && datum.grpId, stk2:datum.stack=='stk2' && datum.grpId}\\\"\\n }\\n {events: \\\"mouseout\\\", update: \\\"{}\\\"}\\n ]\\n }\\n // used to filter only the data related to the selected value\\n {\\n name: groupSelector\\n value: false\\n on: [\\n {\\n // Clicking groupMark sets this signal to the filter values\\n events: @groupMark:click!\\n update: \\\"datum.stack=='stk1' ? opensearchDashboardsAddFilter({\\\\\\\"match_phrase\\\\\\\": { \\\\\\\"source.segment.name\\\\\\\": datum.grpId } }, 'MALCOLM_NETWORK_INDEX_PATTERN_REPLACER') : opensearchDashboardsAddFilter({\\\\\\\"match_phrase\\\\\\\": { \\\\\\\"destination.segment.name\\\\\\\": datum.grpId } }, 'MALCOLM_NETWORK_INDEX_PATTERN_REPLACER')\\\"\\n }\\n {\\n // Clicking \\\"show all\\\" button, or double-clicking anywhere resets it\\n events: [\\n {type: \\\"click\\\", markname: \\\"groupReset\\\"}\\n {type: \\\"dblclick\\\"}\\n ]\\n update: \\\"false\\\"\\n }\\n ]\\n }\\n ]\\n}\"}}", @@ -223,8 +223,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-06-12T17:05:35.015Z", - "version": "Wzk4MywxXQ==", + "updated_at": "2024-08-15T21:56:49.347Z", + "version": "WzQyNiwxXQ==", "attributes": { "title": "Protocol by Network Segment", "visState": "{\"title\":\"Protocol by Network Segment\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"6\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"related.site\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":50,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Site\"},\"schema\":\"bucket\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"network.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"-\",\"customLabel\":\"Network Segment\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"network.type\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Family\"},\"schema\":\"bucket\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"network.transport\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":150,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Transport\"},\"schema\":\"bucket\"},{\"id\":\"5\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"network.protocol\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Protocol\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", @@ -253,8 +253,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-06-12T16:50:47.700Z", - "version": "WzQyNCwxXQ==", + "updated_at": "2024-08-15T21:56:49.347Z", + "version": "WzQyNywxXQ==", "attributes": { "title": "Event Severity by Network Segment", "visState": "{\"title\":\"Event Severity by Network Segment\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"7\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"related.site\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Site\"},\"schema\":\"bucket\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"network.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Network Segment\"},\"schema\":\"bucket\"},{\"id\":\"5\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"event.severity_tags\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Severity Tag\"},\"schema\":\"bucket\"},{\"id\":\"6\",\"enabled\":true,\"type\":\"max\",\"params\":{\"field\":\"event.risk_score\",\"customLabel\":\"High Raw Severity\"},\"schema\":\"metric\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", @@ -283,8 +283,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-06-12T17:29:01.477Z", - "version": "Wzk5MywxXQ==", + "updated_at": "2024-08-15T21:56:49.347Z", + "version": "WzQyOCwxXQ==", "attributes": { "title": "Source Device Type", "visState": "{\"title\":\"Source Device Type\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"source.device.manufacturer\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Manufacturer\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"source.device.device_type\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Type\"},\"schema\":\"bucket\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"source.device.role\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Role\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":18,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", @@ -313,8 +313,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-06-12T17:23:30.298Z", - "version": "Wzk4OSwxXQ==", + "updated_at": "2024-08-15T21:56:49.347Z", + "version": "WzQyOSwxXQ==", "attributes": { "title": "Notice, Alert and Signature by Network Segment", "visState": "{\"title\":\"Notice, Alert and Signature by Network Segment\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"event.provider\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Provider\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"event.dataset\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Dataset\"},\"schema\":\"bucket\"},{\"id\":\"6\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"related.site\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Site\"},\"schema\":\"bucket\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"network.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Network Segment\"},\"schema\":\"bucket\"},{\"id\":\"5\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"rule.category\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Category\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":15,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", @@ -343,8 +343,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-06-12T17:29:16.965Z", - "version": "Wzk5NCwxXQ==", + "updated_at": "2024-08-15T21:56:49.347Z", + "version": "WzQzMCwxXQ==", "attributes": { "title": "Destination Device Type", "visState": "{\"title\":\"Destination Device Type\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"destination.device.manufacturer\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Manufacturer\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"destination.device.device_type\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Type\"},\"schema\":\"bucket\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"destination.device.role\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Role\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":18,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", @@ -373,8 +373,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-06-12T16:50:47.700Z", - "version": "WzQyMCwxXQ==", + "updated_at": "2024-08-15T21:56:49.347Z", + "version": "WzQzMSwxXQ==", "attributes": { "title": "Source Device Role", "visState": "{\"title\":\"Source Device Role\",\"type\":\"horizontal_bar\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"source.device.role\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":25,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"unknown\",\"customLabel\":\"Role\"},\"schema\":\"segment\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":200},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"square root\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":75,\"filter\":true,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"normal\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"},\"row\":false}}", @@ -403,8 +403,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-06-12T16:50:47.700Z", - "version": "WzQyMSwxXQ==", + "updated_at": "2024-08-15T21:56:49.347Z", + "version": "WzQzMiwxXQ==", "attributes": { "title": "Destination Device Role", "visState": "{\"title\":\"Destination Device Role\",\"type\":\"horizontal_bar\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"destination.device.role\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":25,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"unknown\",\"customLabel\":\"Role\"},\"schema\":\"segment\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":200},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"square root\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":75,\"filter\":true,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"normal\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"},\"row\":false}}", @@ -433,8 +433,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-06-12T17:12:33.760Z", - "version": "Wzk4NiwxXQ==", + "updated_at": "2024-08-15T21:56:49.347Z", + "version": "WzQzMywxXQ==", "attributes": { "title": "Source Device Log Counts", "visState": "{\"title\":\"Source Device Log Counts\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{\"customLabel\":\"Log Count\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"source.device.manufacturer\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Manufacturer\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"source.device.device_type\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Type\"},\"schema\":\"bucket\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"source.device.role\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Role\"},\"schema\":\"bucket\"},{\"id\":\"5\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"source.device.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Name\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":16,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", @@ -463,8 +463,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-06-12T17:12:02.340Z", - "version": "Wzk4NSwxXQ==", + "updated_at": "2024-08-15T21:56:49.347Z", + "version": "WzQzNCwxXQ==", "attributes": { "title": "Destination Device Log Counts", "visState": "{\"title\":\"Destination Device Log Counts\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{\"customLabel\":\"Log Count\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"destination.device.manufacturer\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Manufacturer\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"destination.device.device_type\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Type\"},\"schema\":\"bucket\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"destination.device.role\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Role\"},\"schema\":\"bucket\"},{\"id\":\"5\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"destination.device.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Name\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":18,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", @@ -493,8 +493,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-06-12T16:50:47.700Z", - "version": "WzQyNywxXQ==", + "updated_at": "2024-08-15T21:56:49.347Z", + "version": "WzQzNSwxXQ==", "attributes": { "title": "Uninventoried Internal Source IPs", "visState": "{\"title\":\"Uninventoried Internal Source IPs\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"source.ip\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"IP Address\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"source.segment.site\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Site\"},\"schema\":\"bucket\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"source.segment.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Segment\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", @@ -523,8 +523,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-06-12T16:50:47.700Z", - "version": "WzQyOCwxXQ==", + "updated_at": "2024-08-15T21:56:49.347Z", + "version": "WzQzNiwxXQ==", "attributes": { "title": "Uninventoried Internal Destination IPs", "visState": "{\"title\":\"Uninventoried Internal Destination IPs\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"destination.ip\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"IP Address\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"destination.segment.site\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Site\"},\"schema\":\"bucket\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"destination.segment.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Segment\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", @@ -553,8 +553,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-06-12T16:50:47.700Z", - "version": "WzQyOSwxXQ==", + "updated_at": "2024-08-15T21:56:49.347Z", + "version": "WzQzNywxXQ==", "attributes": { "title": "Uninventoried Internal Assets - Logs", "description": "", @@ -596,8 +596,8 @@ "namespaces": [ "default" ], - "updated_at": "2024-06-12T16:50:47.700Z", - "version": "WzQzMCwxXQ==", + "updated_at": "2024-08-15T21:56:49.347Z", + "version": "WzQzOCwxXQ==", "attributes": { "title": "Uninventoried Observed Services - Logs", "description": "", @@ -632,14 +632,46 @@ "search": "7.9.3" } }, + { + "id": "cde50940-5b52-11ef-b245-313a0ac785d6", + "type": "search", + "namespaces": [ + "default" + ], + "updated_at": "2024-08-15T22:07:47.156Z", + "version": "WzEwMTQsMV0=", + "attributes": { + "title": "NetBox Enrichment or Segment - Logs", + "description": "", + "hits": 0, + "columns": [ + "_source" + ], + "sort": [], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"tags:netbox OR ((source.segment:* OR destination.segment:*) AND (NOT (network.direction:external)))\",\"language\":\"kuery\"},\"highlightAll\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER\",\"fixed_interval\":\"30d\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + } + }, + "references": [ + { + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern", + "id": "MALCOLM_NETWORK_INDEX_PATTERN_REPLACER" + } + ], + "migrationVersion": { + "search": "7.9.3" + } + }, { "id": "b1645e70-8182-11ed-8ccb-af218b39580f", "type": "search", "namespaces": [ "default" ], - "updated_at": "2024-06-12T16:50:47.700Z", - "version": "WzQzMiwxXQ==", + "updated_at": "2024-08-15T21:56:49.347Z", + "version": "WzQzOSwxXQ==", "attributes": { "title": "NetBox Enrichment - Logs", "description": "", From 2b0d534d35a66d4b27024fb8733bc3108b776bbb Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Fri, 16 Aug 2024 11:26:14 -0600 Subject: [PATCH 35/47] minor documentation tweak --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index f8dde1e6a..e29837f74 100644 --- a/README.md +++ b/README.md @@ -4,16 +4,16 @@ Malcolm is a powerful network traffic analysis tool suite designed with the following goals in mind: -* **Easy to use** – Malcolm accepts network traffic data in the form of full packet capture (PCAP) files and Zeek logs. These artifacts can be uploaded via a simple browser-based interface or captured live and forwarded to Malcolm using lightweight forwarders. In either case, the data is automatically normalized, enriched, and correlated for analysis. -* **Powerful traffic analysis** – Visibility into network communications is provided through two intuitive interfaces: OpenSearch Dashboard, a flexible data visualization plugin with dozens of prebuilt dashboards providing an at-a-glance overview of network protocols; and Arkime (formerly Moloch), a powerful tool for finding and identifying the network sessions comprising suspected security incidents. -* **Streamlined deployment** – Malcolm operates as a cluster of Docker containers – isolated sandboxes that each serve a dedicated function of the system. This Docker-based deployment model, combined with a few simple scripts for setup and run-time management, makes Malcolm suitable to be deployed quickly across a variety of platforms and use cases; whether it be for long-term deployment on a Linux server in a security operations center (SOC) or for incident response on a Macbook for an individual engagement. +* **Easy to use** – Malcolm accepts network traffic data in the form of full packet capture (PCAP) files, Zeek logs, and Suricata alerts. These artifacts can be uploaded via a simple browser-based interface or passively captured live and forwarded to Malcolm using lightweight forwarders. In either case, the data is automatically normalized, enriched, and correlated for analysis. +* **Powerful traffic analysis** – Visibility into network communications is provided through two intuitive interfaces: OpenSearch Dashboards, a flexible data visualization plugin with dozens of prebuilt dashboards providing an at-a-glance overview of network protocols; and Arkime, a powerful tool for finding and identifying the network sessions comprising suspected security incidents. +* **Streamlined deployment** – Malcolm operates as a cluster of software containers – isolated sandboxes that each serve a dedicated function of the system. This container-based deployment model, combined with a few simple scripts for setup and run-time management, makes Malcolm suitable to be deployed quickly across a variety of platforms and use cases; whether it be for long-term deployment on a Linux server in a security operations center (SOC) or for incident response on a Macbook for an individual engagement. * **Secure communications** – All communications with Malcolm, both from the user interface and from remote log forwarders, are secured with industry standard encryption protocols. * **Permissive license** – Malcolm is comprised of several widely used open-source tools, making it an attractive alternative to security solutions requiring paid licenses. * **Expanding control systems visibility** – While Malcolm is great for general-purpose network traffic analysis, its creators see a particular need in the community for tools providing insight into protocols used in industrial control systems (ICS) environments. Ongoing Malcolm development will aim to provide additional parsers for common ICS protocols. Although all the open-source tools that make up Malcolm are already available and in general use, Malcolm provides a framework of interconnectivity that makes it greater than the sum of its parts. -In short, Malcolm provides an easily deployable network analysis tool suite for full PCAP files and Zeek logs. While Internet access is required to build Malcolm, internet access is not required at runtime. +In short, Malcolm provides an easily deployable traffic analysis tool suite for network security monitoring. ## Documentation From eca28edb887a31d80f1d0b5c41b4408fbc37f513 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Fri, 16 Aug 2024 11:28:33 -0600 Subject: [PATCH 36/47] added capabilities-and-limitations.md --- docs/capabilities-and-limitations.md | 122 +++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 docs/capabilities-and-limitations.md diff --git a/docs/capabilities-and-limitations.md b/docs/capabilities-and-limitations.md new file mode 100644 index 000000000..4da398fef --- /dev/null +++ b/docs/capabilities-and-limitations.md @@ -0,0 +1,122 @@ +# General Overview + +[Malcolm]({{ site.github.repository_url }}) is a powerful network traffic analysis tool suite designed with the following goals in mind: + +* **Easy to use** – Malcolm accepts network traffic data in the form of full packet capture (PCAP) files, Zeek logs, and Suricata alerts. These artifacts can be uploaded via a simple browser-based interface or passively captured live and forwarded to Malcolm using lightweight forwarders. In either case, the data is automatically normalized, enriched, and correlated for analysis. +* **Powerful traffic analysis** – Visibility into network communications is provided through two intuitive interfaces: OpenSearch Dashboards, a flexible data visualization plugin with dozens of prebuilt dashboards providing an at-a-glance overview of network protocols; and Arkime, a powerful tool for finding and identifying the network sessions comprising suspected security incidents. +* **Streamlined deployment** – Malcolm operates as a cluster of software containers – isolated sandboxes that each serve a dedicated function of the system. This container-based deployment model, combined with a few simple scripts for setup and run-time management, makes Malcolm suitable to be deployed quickly across a variety of platforms and use cases; whether it be for long-term deployment on a Linux server in a security operations center (SOC) or for incident response on a Macbook for an individual engagement. +* **Secure communications** – All communications with Malcolm, both from the user interface and from remote log forwarders, are secured with industry standard encryption protocols. +* **Permissive license** – Malcolm is comprised of several widely used open-source tools, making it an attractive alternative to security solutions requiring paid licenses. +* **Expanding control systems visibility** – While Malcolm is great for general-purpose network traffic analysis, its creators see a particular need in the community for tools providing insight into protocols used in industrial control systems (ICS) environments. Ongoing Malcolm development will aim to provide additional parsers for common ICS protocols. + +Although all the [open-source tools](components.md) that make up Malcolm are already available and in general use, Malcolm provides a framework of interconnectivity that makes it greater than the sum of its parts. + +In short, Malcolm provides an easily deployable traffic analysis tool suite for network security monitoring. + +# Features + +* Network capture + - Malcolm can passively capture network traffic and generate network log data for analysis in a number of scenarios. For less-complicated network configurations, Malcolm runs in a [standalone "all-in-one"](live-analysis.md#LocalPCAP) deployment in which it fulfills the roles of both network sensor and log aggregator. For deployments in which network capture needs to be instrumented at multiple tap points, Malcolm's lightweight network sensor operating system [Hedgehog Linux](hedgehog.md) can be deployed to [passively observe traffic and forward metadata](live-analysis.md#Hedgehog) about the network sessions it observes to the Malcolm aggregator for enrichment, correlation, and indexing. + - Depending on local requirements, network capture is can be performed by a combination of several tools: + + Zeek - an extensible network analysis framework and IDS + + Arkime - a tool for providing traffic capture and managing local storage of raw or compressed packet capture artifacts (PCAP files) + + Suricata - a signature-based IDS and threat detection engine + + netsniff-ng or tcpdump - for simpler capture-only deployments where initial traffic parsing and metadata forwarding are not required + - Previously-captured PCAP files and Zeek logs can also be [uploaded](upload.md#Upload) via a web browser-based form. + - Limitation: Malcolm captures traffic at the Ethernet link layer: its focus is primarily IP network traffic, although there are a few protocol parsers that operate directly at the network internface layer. Malcolm does not have built-in support for capture and dissection of serial-only protocols. + - Limitation: Malcolm does not have the capability to decrypt encrypted traffic. If that is a requirement, other solutions must be deployed to decrypt TLS & SSL communications prior to handing it off to Malcolm for analysis. +* Data Analytics + - Malcolm's data analytics capabilities are provided at several layers of its stack: + + At the point of capture, Zeek provides the capability to correlate session traffic perform some protocol-specific analytics. Good examples of this include the [BZAR](https://github.com/mitre-attack/bzar) and [ACID](https://github.com/cisagov/ACID) packages, developed by MITRE corporation, the former for analysis of SMB/DCE-RPC traffic and its classification to the ATT&CK framework, and the latter for analysis of ICS traffic from a number of OT protocols and its classification to the ATT&CK for ICS framework. ACID was specifically developed by MITRE on top of Zeek [protocol analyzers](https://github.com/cisagov/ICSNPP) created by the Malcolm team at INL. + + File transfers are detected *in situ* in network traffic and [extracted for scanning and analysis](file-scanning.md#ZeekFileExtraction). This automated static file analysis is handled by several signature- and heuristics-based tools including ClamAV (an antivirus engine), capa (a static capabilities analyzer for portable executable files), and YARA (a pattern-matching tool to identify and classify malware). Development is underway to incorporate AssemblyLine, a tool for file triage and dynamic malware analysis. + + For analyst-driven analytics, all network traffic metadata is indexed into an [OpenSearch or Elasticsearch](opensearch-instances.md#OpenSearchInstance) data store where it can be queried by analysts using dozens of pre-built dashboards and visualizations. Custom analytics can be developed using a variety of tools and query languages. Since the data is indexed into these industry-standard data analytics platforms, other tools like [OpenSearch Security Analytics](https://opensearch.org/docs/latest/security-analytics/) or [Elastic Security](https://www.elastic.co/security) can also be overlaid on top of Malcolm's data. Users can write custom queries written with [Lucene](https://lucene.apache.org/core/2_9_4/queryparsersyntax.md), [DQL](https://opensearch.org/docs/latest/dashboards/dql/)/[KQL](https://www.elastic.co/guide/en/kibana/current/kuery-query.md) (Dashboards and Kibana query languages, respectively), [PPL](https://opensearch.org/docs/latest/search-plugins/sql/ppl/index/) (piped processing language), or [SQL](https://opensearch.org/docs/latest/search-plugins/sql/sql/index/) (structured query language). + + Arkime also provides ["Hunt"](arkime.md#ArkimeHunt) capabilities, meaning that users can write and execute full-packet search queries that are performed against the raw PCAP payloads stored at the point of capture. +* Data Visualization + - Data visualization is provided in Malcolm by the following browswer-based user interfaces: + + [OpenSearch Dashboards](dashboards.md#Dashboards) (if using the OpenSearch backend) or Kibana (if using Elasticsearch) provide hundreds of prebuilt visualizations organized in dozens of dashboards. These tools also provide drag-and-drop creation of custom visualizations. These dashboards and visualizations can be accessed via web interface, exported to PDF or PNG files, or automatically sent with triggered [alerts](https://opensearch.org/docs/latest/observing-your-data/alerting/index/) or as scheduled [reports](https://opensearch.org/docs/latest/reporting/report-dashboard-index/). + + [Arkime](arkime.md#Arkime) hosts several unique visualizations: + * The [Sessions](arkime.md#ArkimeSessions) view allows individual events from all of Malcolm's data sources to be viewed in a single pane of glass and full packet capture payloads examined inline in the context of their network sessions. + * The [SPIView](arkime.md#ArkimeSPIView) and [SPIGraph](arkime.md#ArkimeSPIGraph) (SPI - session profile information) views that help visualize network traffic by breaking it down chronologically, geographically, and by whatever data points the user selects. + * The [Connections](arkime.md#ArkimeConnections) view generates a force-directed network graph that can be used to visualize the relationships between network enpoints and the protocols over which they communicate. Baselining is also possible, meaning changes to observed network communications over time can be highlighted. +* [Anomaly Detection](anomaly-detection.md#AnomalyDetection) and Machine Learning/AI opportunities + - OpenSearch's [anomaly detection engine](https://opensearch.org/docs/latest/observing-your-data/ad/index/) automatically detects anomalies in network traffic in near real-time using [Random Cut Forests (RCF)](https://api.semanticscholar.org/CorpusID:927435), an unsupervised (i.e., "black box" or self-training) machine learning algorithm that models a sketch of the data stream to compute an anomaly grade and confidence score value for each incoming data point. These values are used to differentiate an anomaly from normal variations. Malcolm ships with several useful detectors based on this technique, but users can create detectors from any of the nearly 4,000 data points gathered from across Malcolm's supported network protocols. Anomalies and detectors can be managed and viewed in [OpenSearch Dashboards](https://github.com/opensearch-project/anomaly-detection-dashboards-plugin). + - More experienced data scientists can harness the [OpenSearch ML Commons](https://opensearch.org/docs/latest/ml-commons-plugin/) plugin to do machine-learning model-powered search, using pretrained models provided by OpenSearch, uploading their own models to the OpenSearch engine, or connecting to a model hosted on an external platform. This plugin can be managed via a Dashboards interface or through a set of rest APIs. + - Those using the Elasticsearch platform can use the machine learning and anomaly detection [tools provided](https://www.elastic.co/elasticsearch/machine-learning) by that platform depending on their Elasticsearch license subscription. + - Limitation: Anomaly detection and machine learning algorithms rely on enough data (for network data, this generally means at least several weeks' worth or more) to be able to build a baseline of what is normal before they can accurately flag anomalies, and each network is different. Anomaly detection and ML are typically not useful for limited deployments without the available traffic to build that baseline. + - Limitation: While Malcolm provides some powerful tools in the anomaly detection and ML realm, as of yet they have not been built out to provide the value that they will probably one day realize. +* Threat ingestion + - Malcolm can ingest threat indicators in the form of static MISP- or STIX-formatted files. It can also subscribe to and periodically update threat indicators from [MISP](zeek-intel.md#ZeekIntelMISP) and [TAXII](zeek-intel.md#ZeekIntelSTIX) feeds. These indicators are converted into a format that is read by Zeek, and matches in network traffic are [surfaced through the Zeek intelligence framework](zeek-intel.md#ZeekIntel) for logging. + - Limitation: Some formats for threat indicators allow for complex definitions and logic. For STIX/TAXII, only indicators of cyber-observable objects matched with the equals (=) comparison operator against a single value can be expressed as Zeek intelligence items. Similarly, only a subset of MISP attribute types can be expressed with the Zeek intelligence indicator types. While this is generally sufficient to cover most indicators interest, more complex indicators are silently ignored. +* Network Modeling + - Malcolm provides an instance of [NetBox](https://netboxlabs.com/oss/netbox/), an open-source "solution for modeling and documenting modern networks" which is used to model instrumented networks and enrich passively-observed network traffic from that model, a technique Malcolm calls ["Asset Interaction Analysis"](asset-interaction-analysis.md#AssetInteractionAnalysis). Users can pivot between the network visualization tools (the Asset Interaction Analysis and Zeek Known Summary dashboards in OpenSearch Dashboards, and the Arkime Sessions interface) and the NetBox UI to investigate and examine network assets. + - The NetBox network inventory can be built manually (through manual object entry or by importing CSV, XML, JSON, etc. files) or through [passively-gathered](asset-interaction-analysis.md#NetBoxPopPassive) network traffic metadata. + - Limitation: Malcolm does not actively probe a network for asset discovery. As its only methods for automatic inventory population are passive, layer 2 network devices are not usually visible nor discoverable through observed network traffic and must be entered manually. There is, however, some research being conducted to overcome this limitation. Active discovery is also being discovered as a possibility for future development. +* Payload Analysis + - The [Arkime sessions interface](arkime.md#ArkimeSessions) provides access to full PCAP payload. Full payloads are viewable inline in the context of their sessions, or can be [exported](arkime.md#ArkimePCAPExport) for analysis in other tools like WireShark, Network Miner, GRASSMARLIN, etc. +* Throughput limits + - Malcolm does not specify any particular hardware profile and can be run on a variety of hardware. By providing Malcolm with sufficient computing and storage resources, it should be capable to scale to handle a wide range of throughput requirements. One of Malcolm's partners in the US military is deploying Malcolm in environments with tens of gigabits per second of network throughput. + - Limitation: Not much time has been spent by Malcolm developers probing the limits of network throughput, so we don't have a lot of solid numbers on this. +* Storage + - The initial artifacts associated with network traffic capture (PCAP files, raw Zeek and Suricata logs, extracted files, etc.) are stored at the point of capture, whether that be on a standalone Malcolm instance or on sensors running Hedgehog Linux. Indices for the OpenSearch or Elasticsearch data store are managed locally by the Malcolm aggregator, or [remote data store clusters](opensearch-instances.md#OpenSearchInstance) can be used. Solid state storage or RAID configurations are usually required to keep up with streaming full PCAP to disk. For long-running deployments, Malcolm can be [configured to](malcolm-config.md#DiskUsage) prune old indices and artifacts to make room for new data. + - Limitation: Malcolm does not currently provide a mechanism for configuring automatic backups or rollover of network traffic artifacts to offline or network-attached storage. +* Extensiblity + - Malcolm provides its own set of [REST APIs](api.md#API) to allow other platforms to interface directly with its data or to access the APIs of its underlying components (e.g., OpenSearch or Elasticsearch, Logstash, NetBox). + - [Custom rules, scripts, and plugins](custom-rules.md#CustomRulesAndScripts) can easily be dropped-in for several of Malcolm's components: + + Zeek scripts and plugins + + Suricata rules + + YARA rules + + [Threat intelligence feeds](zeek-intel.md#ZeekIntel) + + NetBox plugins + + Sigma rules* + * Sigma rules are supported through OpenSearch Security Analytics, although development on the upcoming Malcolm roadmap is needed to make this integration fully capable + - A [contributor's guide](contributing-guide.md) in the documentation provides guidelines for outside developers who wish to enhance or extend Malcolm. + - Limitation: A full-fledged plugin system for adding features to Malcolm does not yet exist, though is in early design stages. +* Automation + - For the most part, Malcolm can run with a "set it and forget it" mindset should keep itself running without much hands-on maintenance. + - Automatic updates of threat intelligence feeds and rules for file scanning engines can be scheduled to run automatically in the background. + - Automatic [alerting](alerting.md#Alerting) and [reporting](reporting.md#Reporting) capabilities are available. + - Limitation: Malcolm does not yet have a well-defined plan for [automated upgrades](malcolm-upgrade.md#UpgradePlan): it's a pretty manual process, especially for the [Hedgehog Linux sensors](hedgehog-upgrade.md#HedgehogUpgradePlan). +* Cloud deployment capabilities + - Malcolm can be cloud-deployed with [Kubernetes](kubernetes.md#Kubernetes) either on-prem, on [Amazon AWS with Elastic Kubernetes Service](kubernetes-eks.md#KubernetesEKS), or ostensibly on any other Kubernetes-based platform. A US military partner is deploying Malcolm in this manner. + - As Hedgehog Linux is a dedicated OS for traffic capture, it is not used directly capture in cloud deployments. Instead, for cloud deployments, traffic can be collected in a few ways: + + Capture in physical network environments can be done normally with Hedgehog Linux and forwarded to a Malcolm instance in the cloud. + + Malcolm can run in "standalone" mode to do capture of a virtual or physical [interface available](live-analysis.md#LocalPCAP) to nodes the Kubernetes cluster, or run in the capture-only [Hedgehog run profile](live-analysis.md#Profiles). + + On AWS, [VPC traffic mirroring](https://docs.aws.amazon.com/vpc/latest/mirroring/what-is-traffic-mirroring.md) can be [configured](https://www.awsnetworkshops.com/030_module1/mirrortraffic/) and expose an interface for Malcolm to listen on. + - Limitation: As the variety of cloud platforms is varied, some of the configuration for traffic capture on cloud networks is left up to the user somewhat; Malcolm can listen on anything provided to it as a virtual interface, but assumes that the traffic-mirroring interface has been configured correctly by the user. + - Limitation: The documened instructions for cloud deployment don't yet walk the user through many of the scale-out capabilities Kubernetes provides. However, a US military partner is working on a [Helm chart](https://github.com/idaholab/Malcolm-Helm) that expands its cloud capabilities and has contributed that back upstream to the main Malcolm project. + - Limitation: Malcolm [does not yet](https://github.com/idaholab/Malcolm/issues/232) have instrumentation for receiving and capturing cloud infrastructure logs, such as VPC flow logs, CloudFront access logs, ELB logs, S3 bucket logs, Route 53 query logs, Amazon RDS logs, Azure infrastructure logs, etc. +* Resource requirements + - Hardware + + Malcolm + * Minimum system requirements + - 16 gigabytes RAM + - 8 CPU cores (amd64 and arm64 supported) + - As much storage (preferably solid-state or RAID) as necessary to meet data retention requirements + * Recommended system requirements + - 32+ gigabytes RAM + - 16+ CPU cores (amd64 and arm64 supported) + - As much storage (preferably solid-state or RAID) as necessary to meet data retention requirements + + Hedgehog Linux + * Minimum system requirements + - 8 gigabytes RAM + - 4 CPU cores (amd64 supported, arm64 supported for Raspberry Pi image) + - As much storage (preferably solid-state or RAID) as necessary to meet full PCAP retention requirements + * Recommended system requirements + - 16+ gigabytes RAM + - 8+ CPU cores (amd64 supported, arm64 supported for Raspberry Pi image) + - As much storage (preferably solid-state or RAID) as necessary to meet data retention requirements + - License + + Malcolm is [distributed](https://github.com/idaholab/Malcolm/blob/main/LICENSE.txt) under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) on an as-is basis, without warranties or conditions of any kind, either express or implied. + + Malcolm's [components](components.md) are for the most part permissive open-source licenses as well. In cases where underlying system components are released under more restrictive copyleft license terms (e.g., GPL), Malcolm interfaces with those tools or libraries in a loosely-coupled way that is, for example, satisfied by the GPLv2's "mere aggregation" clause and the GPLv3 "aggregates" clause, in addition to the fact that in Malcolm's software containers those tools run as "separate and independent" processes. [This article from opensource.com](https://opensource.com/article/18/1/containers-gpl-and-copyleft) explains the ramifications of GPL and copyleft as it pertains to Linux containers. + - Support + + The organizations supporting Malcolm's development do not offer paid professional services or any contract-based support. Any support provided is done the same way most other open-source projects do so, which is through GitHub, for example: + * [Community discussions board](https://github.com/cisagov/Malcolm/discussions) + * [Issue and feature tracking](https://github.com/idaholab/malcolm/issues?q=is%3Aissue+is%3Aopen+-label%3Atraining) + * [Project roadmap tracking](https://github.com/orgs/idaholab/projects/1) + + Beyond engagement via the forums listed here, no expectation of any particular level of support should be assumed. +* User experience/UI/intuitive + - Malcolm's user-friendly analyst-facing interfaces are accessible via an easy-to-nagivate [landing page](images/screenshots/malcolm_landing_page.png). + - Malcolm's [documentation](README.md) is robust and usable. + - A series of [YouTube videos](https://www.youtube.com/@malcolmnetworktrafficanalysis/playlists) are being developed for a [variety of topics](https://github.com/idaholab/Malcolm/wiki/Malcolm-Learning-Tree). + - A technical training course is being developed which will be publicly released to help organizations train Malcolm users. + - Limitation: Because Malcolm is a framework hosting discrete tools, several of which have their own user interfaces, some users may feel that the Malcolm user experience isn't as "unified" as it would be if these interfaces had all been created in tandem by one entity. From 2de2a1f9746c2e1d485218b7d252cb84e40b15c2 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Fri, 16 Aug 2024 11:38:17 -0600 Subject: [PATCH 37/47] idaholab/Malcolm#530, work in progress on netbox plugin installation for k8s --- kubernetes/18-netbox.yml | 6 +++++- scripts/malcolm_kubernetes.py | 6 ++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/kubernetes/18-netbox.yml b/kubernetes/18-netbox.yml index af9b5de48..f8ad3dfdf 100644 --- a/kubernetes/18-netbox.yml +++ b/kubernetes/18-netbox.yml @@ -79,7 +79,8 @@ spec: name: netbox-var-local-catrust-volume - mountPath: /etc/netbox/config/configmap name: netbox-config-volume - # TODO: custom-plugins + - mountPath: /opt/netbox-custom-plugins + name: netbox-custom-plugins-volume - mountPath: /opt/netbox-preload/configmap name: netbox-preload-volume - mountPath: /opt/netbox/netbox/media @@ -107,6 +108,9 @@ spec: - name: netbox-config-volume configMap: name: netbox-config + - name: netbox-custom-plugins-volume + configMap: + name: netbox-custom-plugins - name: netbox-preload-volume configMap: name: netbox-preload diff --git a/scripts/malcolm_kubernetes.py b/scripts/malcolm_kubernetes.py index eb1121047..7f8223a7a 100644 --- a/scripts/malcolm_kubernetes.py +++ b/scripts/malcolm_kubernetes.py @@ -149,6 +149,12 @@ 'path': os.path.join(MalcolmPath, os.path.join('netbox', 'config')), }, ], + 'netbox-custom-plugins': [ + { + 'secret': False, + 'path': os.path.join(MalcolmPath, os.path.join('netbox', 'custom-plugins')), + }, + ], 'netbox-preload': [ { 'secret': False, From ea53ae79573a07704629aa59c113604b24c91bcb Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Fri, 16 Aug 2024 12:04:33 -0600 Subject: [PATCH 38/47] idaholab/Malcolm#536, during auth_setup 'all' operation, do required operations without prompting if the files don't already exist --- scripts/control.py | 39 +++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/scripts/control.py b/scripts/control.py index 9e22a2e12..f6127f0b8 100755 --- a/scripts/control.py +++ b/scripts/control.py @@ -1267,6 +1267,7 @@ def authSetup(): "Configure all authentication-related settings", True, True, + [], ), ( 'admin', @@ -1274,6 +1275,7 @@ def authSetup(): False, (not args.cmdAuthSetupNonInteractive) or (bool(args.authUserName) and bool(args.authPasswordOpenssl) and bool(args.authPasswordHtpasswd)), + [], ), ( 'webcerts', @@ -1286,6 +1288,7 @@ def authSetup(): os.path.join(MalcolmPath, os.path.join('nginx', os.path.join('certs', 'key.pem'))) ) ), + [os.path.join(MalcolmPath, os.path.join('nginx', os.path.join('certs', 'key.pem')))], ), ( 'fwcerts', @@ -1294,63 +1297,67 @@ def authSetup(): not args.cmdAuthSetupNonInteractive or ( args.authGenFwCerts - or not os.path.isfile( - os.path.join(MalcolmPath, os.path.join('logstash', os.path.join('certs', 'server.key'))) - ) + or not os.path.isfile(os.path.join(logstashPath, 'server.key')) + or not os.path.isfile(os.path.join(filebeatPath, 'client.key')) ), + [ + os.path.join(logstashPath, 'server.key'), + os.path.join(filebeatPath, 'client.key'), + ], ), ( 'remoteos', "Configure remote primary or secondary OpenSearch/Elasticsearch instance", False, False, + [], ), ( 'email', "Store username/password for OpenSearch Alerting email sender account", False, False, + [], ), ( 'netbox', "(Re)generate internal passwords for NetBox", False, - ( - not os.path.isfile( - os.path.join(MalcolmPath, os.path.join('netbox', os.path.join('env', 'netbox-secret.env'))) - ) - ) - or (args.cmdAuthSetupNonInteractive and args.authGenNetBoxPasswords), + args.cmdAuthSetupNonInteractive and args.authGenNetBoxPasswords, + [], ), ( 'arkime', "Store password hash secret for Arkime viewer cluster", False, False, + [], ), ( 'txfwcerts', "Transfer self-signed client certificates to a remote log forwarder", False, False, + [], ), )[: 9 if txRxScript else -1] authMode = ( ChooseOne( 'Configure Authentication', - choices=[x[:-1] for x in authModeChoices], + choices=[x[:-2] for x in authModeChoices], ) if not args.cmdAuthSetupNonInteractive else 'all' ) - defaultBehavior = ( + noninteractiveBehavior = ( UserInputDefaultsBehavior.DefaultsPrompt - if not args.cmdAuthSetupNonInteractive - else UserInputDefaultsBehavior.DefaultsPrompt | UserInputDefaultsBehavior.DefaultsAccept | UserInputDefaultsBehavior.DefaultsNonInteractive ) + defaultBehavior = ( + UserInputDefaultsBehavior.DefaultsPrompt if not args.cmdAuthSetupNonInteractive else noninteractiveBehavior + ) for authItem in authModeChoices[1:]: if ( @@ -1358,7 +1365,11 @@ def authSetup(): and YesOrNo( f'{authItem[1]}?', default=authItem[3], - defaultBehavior=defaultBehavior, + defaultBehavior=( + noninteractiveBehavior + if (authItem[4] and (not all([os.path.isfile(x) for x in authItem[4]]))) + else defaultBehavior + ), ) ) or ((authMode != 'all') and (authMode == authItem[0])): if authItem[0] == 'admin': From 076f8b3f1333ec14aa7e92a40dec0bba6bafc75b Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Fri, 16 Aug 2024 13:30:32 -0600 Subject: [PATCH 39/47] fix idaholab/Malcolm#522, netbox service templates not populating; by lowercasing TCP and UDP values in the preload .yml file --- netbox/preload/service_templates.yml | 110 +++++++++++++-------------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/netbox/preload/service_templates.yml b/netbox/preload/service_templates.yml index 51c35a821..c00e4929b 100644 --- a/netbox/preload/service_templates.yml +++ b/netbox/preload/service_templates.yml @@ -1,64 +1,64 @@ - name: FTP - protocol: TCP + protocol: tcp ports: - 20 - 21 - name: SSH - protocol: TCP + protocol: tcp ports: - 22 - name: Telnet - protocol: TCP + protocol: tcp ports: - 23 - name: SMTP - protocol: TCP + protocol: tcp ports: - 25 - 2525 - name: DNS (UDP) - protocol: UDP + protocol: udp ports: - 53 - 853 - 5353 - name: DNS (TCP) - protocol: TCP + protocol: tcp ports: - 53 - 853 - 5353 - name: DHCP - protocol: UDP + protocol: udp ports: - 67 - 68 - 546 - 547 - name: HTTP - protocol: TCP + protocol: tcp ports: - 80 - 8000 - 8080 - name: COTP and S7comm - protocol: TCP + protocol: tcp ports: - 102 - name: POP3 - protocol: TCP + protocol: tcp ports: - 110 - name: SFTP - protocol: TCP + protocol: tcp ports: - 115 - name: NTP - protocol: TCP + protocol: tcp ports: - 123 - name: Microsoft-DS (UDP) - protocol: UDP + protocol: udp ports: - 135 - 137 @@ -66,7 +66,7 @@ - 445 - 3702 - name: Microsoft-DS (TCP) - protocol: TCP + protocol: tcp ports: - 135 - 139 @@ -74,168 +74,168 @@ - 5357 - 5358 - name: NetBIOS (TCP) - protocol: TCP + protocol: tcp ports: - 137 - 138 - 139 - name: NetBIOS (UDP) - protocol: UDP + protocol: udp ports: - 137 - 138 - 139 - name: IMAP - protocol: TCP + protocol: tcp ports: - 143 - name: SNMP - protocol: UDP + protocol: udp ports: - 161 - 162 - name: LDAP (UDP) - protocol: UDP + protocol: udp ports: - 389 - 3268 - name: LDAP (TCP) - protocol: TCP + protocol: tcp ports: - 389 - 3268 - name: HTTPS - protocol: TCP + protocol: tcp ports: - 443 - 8443 - name: SMTP SSL/TLS - protocol: TCP + protocol: tcp ports: - 465 - 587 - name: IPSec (UDP) - protocol: UDP + protocol: udp ports: - 500 - 4500 - name: IPSec (TCP) - protocol: TCP + protocol: tcp ports: - 500 - 4500 - name: Modbus (UDP) - protocol: UDP + protocol: udp ports: - 502 - name: Modbus (TCP) - protocol: TCP + protocol: tcp ports: - 502 - 802 - name: Syslog (UDP) - protocol: UDP + protocol: udp ports: - 514 - 601 - name: Syslog (TCP) - protocol: TCP + protocol: tcp ports: - 514 - 601 - name: LPD - protocol: TCP + protocol: tcp ports: - 515 - name: IPP and CUPS (UDP) - protocol: UDP + protocol: udp ports: - 631 - name: IPP and CUPS (TCP) - protocol: TCP + protocol: tcp ports: - 631 - name: LDAP SSL/TLS (UDP) - protocol: UDP + protocol: udp ports: - 636 - 3269 - name: LDAP SSL/TLS (TCP) - protocol: TCP + protocol: tcp ports: - 636 - 3269 - name: rsync - protocol: TCP + protocol: tcp ports: - 873 - name: FTP SSL/TLS - protocol: TCP + protocol: tcp ports: - 989 - 990 - name: Telnet SSL/TLS - protocol: TCP + protocol: tcp ports: - 992 - name: IMAP SSL/TLS - protocol: TCP + protocol: tcp ports: - 993 - name: POP3 SSL/TLS - protocol: TCP + protocol: tcp ports: - 995 - name: OpenVPN (UDP) - protocol: UDP + protocol: udp ports: - 1194 - name: OpenVPN (TCP) - protocol: TCP + protocol: tcp ports: - 1194 - name: MSSQL - protocol: TCP + protocol: tcp ports: - 1433 - 1434 - 5022 - name: Tabular Data Stream - protocol: TCP + protocol: tcp ports: - 1443 - name: CIP and EtherNet/IP (UDP) - protocol: UDP + protocol: udp ports: - 2222 - 44818 - name: CIP and EtherNet/IP (TCP) - protocol: TCP + protocol: tcp ports: - 2222 - 44818 - name: MySQL - protocol: TCP + protocol: tcp ports: - 3306 - name: RDP - protocol: TCP + protocol: tcp ports: - 3389 - name: LLMNR (UDP) - protocol: UDP + protocol: udp ports: - 5355 - name: LLMNR (TCP) - protocol: TCP + protocol: tcp ports: - 5355 - name: PostgreSQL - protocol: TCP + protocol: tcp ports: - 5432 - name: VNC - protocol: TCP + protocol: tcp ports: - 5800 - 5900 @@ -251,24 +251,24 @@ - 5910 - 6000 - name: AppSocket and JetDirect - protocol: TCP + protocol: tcp ports: - 9100 - 9101 - 9102 - name: Git - protocol: TCP + protocol: tcp ports: - 9418 - name: BACnet (UDP) - protocol: UDP + protocol: udp ports: - 47808 - name: BACnet (TCP) - protocol: TCP + protocol: tcp ports: - 47808 - name: WireGuard - protocol: UDP + protocol: udp ports: - 51820 From 7ad3f38b8f17b4a974e9f8ced389e4660acc1910 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Fri, 16 Aug 2024 13:30:51 -0600 Subject: [PATCH 40/47] added gitignore --- netbox/custom-plugins/.gitignore | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 netbox/custom-plugins/.gitignore diff --git a/netbox/custom-plugins/.gitignore b/netbox/custom-plugins/.gitignore new file mode 100644 index 000000000..a5baada18 --- /dev/null +++ b/netbox/custom-plugins/.gitignore @@ -0,0 +1,3 @@ +* +!.gitignore + From 7e6fe6e76ec3b0b5407d1fc758632b77ba706f52 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Fri, 16 Aug 2024 14:22:33 -0600 Subject: [PATCH 41/47] remove old configuration option --- .../virter/malcolm-setup-02-clone-install.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/third-party-environments/virter/malcolm-setup-02-clone-install.toml b/scripts/third-party-environments/virter/malcolm-setup-02-clone-install.toml index a634a96bb..6e26178df 100644 --- a/scripts/third-party-environments/virter/malcolm-setup-02-clone-install.toml +++ b/scripts/third-party-environments/virter/malcolm-setup-02-clone-install.toml @@ -67,8 +67,6 @@ if [[ $- == *i* ]] && [[ -d ~/Malcolm ]] && [[ ! -f ~/Malcolm/.configured ]]; t --extracted-file-yara \ --netbox \ --netbox-enrich \ - --netbox-autopopulate false \ - --netbox-preload-prefixes \ --netbox-site-name malcolm # username: analyst From 6abaa1beb1bb0b5a4a547dc10352d8426e9fe1dd Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Sun, 18 Aug 2024 23:38:01 -0600 Subject: [PATCH 42/47] for idaholab/Malcolm#499: change base for dashboards-helper from alpine to debian:12-slim to avoid weirdness with alpine's busybox curl --- Dockerfiles/dashboards-helper.Dockerfile | 33 +++++++++++++++--------- dashboards/supervisord.conf | 7 ++++- docs/development.md | 4 +-- 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/Dockerfiles/dashboards-helper.Dockerfile b/Dockerfiles/dashboards-helper.Dockerfile index d7f2ecaf5..0a5eb6aa8 100644 --- a/Dockerfiles/dashboards-helper.Dockerfile +++ b/Dockerfiles/dashboards-helper.Dockerfile @@ -1,6 +1,6 @@ ARG TARGETPLATFORM=linux/amd64 -FROM --platform=${TARGETPLATFORM} alpine:3.20 +FROM --platform=${TARGETPLATFORM} debian:12-slim # Copyright (c) 2020 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" @@ -60,17 +60,26 @@ COPY --chmod=755 shared/bin/opensearch_read_only.py /data/ ADD scripts/malcolm_utils.py /data/ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') && \ - apk update --no-cache && \ - apk upgrade --no-cache && \ - apk --no-cache add bash python3 py3-pip curl openssl procps psmisc moreutils npm rsync shadow jq tini && \ - npm install -g http-server && \ + apt-get -q update && \ + apt-get -y -q --no-install-recommends upgrade && \ + apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages install --no-install-recommends \ + bash \ + curl \ + jq \ + moreutils \ + openssl \ + procps \ + psmisc \ + python3 \ + python3-pip \ + rsync \ + tini && \ pip3 install --break-system-packages supervisor humanfriendly requests && \ curl -fsSL -o /usr/local/bin/supercronic "${SUPERCRONIC_URL}${BINARCH}" && \ chmod +x /usr/local/bin/supercronic && \ - addgroup -g ${DEFAULT_GID} ${PGROUP} ; \ - adduser -D -H -u ${DEFAULT_UID} -h /nonexistant -s /sbin/nologin -G ${PGROUP} -g ${PUSER} ${PUSER} ; \ - addgroup ${PUSER} tty ; \ - addgroup ${PUSER} shadow ; \ + groupadd --gid ${DEFAULT_GID} ${PUSER} && \ + useradd -M --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} -d /nonexistant -s /sbin/nologin ${PUSER} && \ + usermod -a -G tty ${PUSER} && \ mkdir -p /data/init /opt/ecs && \ cd /opt && \ curl -sSL "$(curl -sSL "$ECS_RELEASES_URL" | jq '.tarball_url' | tr -d '"')" | tar xzf - -C ./ecs --strip-components 1 && \ @@ -93,17 +102,17 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') /opt/templates && \ chmod 755 /data/*.sh /data/*.py /data/init && \ chmod 400 /opt/maps/* && \ - (echo -e "*/2 * * * * /data/shared-object-creation.sh\n0 10 * * * /data/index-refresh.py --index MALCOLM_NETWORK_INDEX_PATTERN --template malcolm_template --unassigned\n30 */2 * * * /data/index-refresh.py --index MALCOLM_OTHER_INDEX_PATTERN --template malcolm_beats_template --unassigned\n*/20 * * * * /data/opensearch_index_size_prune.py" > ${SUPERCRONIC_CRONTAB}) + (echo "*/2 * * * * /data/shared-object-creation.sh\n0 10 * * * /data/index-refresh.py --index MALCOLM_NETWORK_INDEX_PATTERN --template malcolm_template --unassigned\n30 */2 * * * /data/index-refresh.py --index MALCOLM_OTHER_INDEX_PATTERN --template malcolm_beats_template --unassigned\n*/20 * * * * /data/opensearch_index_size_prune.py" > ${SUPERCRONIC_CRONTAB}) EXPOSE $OFFLINE_REGION_MAPS_PORT -ENTRYPOINT ["/sbin/tini", \ +ENTRYPOINT ["/usr/bin/tini", \ "--", \ "/usr/local/bin/docker-uid-gid-setup.sh", \ "/usr/local/bin/service_check_passthrough.sh", \ "-s", "dashboards-helper"] -CMD ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf", "-n"] +CMD ["/usr/local/bin/supervisord", "-c", "/etc/supervisord.conf", "-n"] VOLUME ["/data/init"] diff --git a/dashboards/supervisord.conf b/dashboards/supervisord.conf index b6cee6932..0fcb1fe2d 100644 --- a/dashboards/supervisord.conf +++ b/dashboards/supervisord.conf @@ -30,7 +30,12 @@ stdout_logfile_maxbytes=0 redirect_stderr=true [program:maps] -command=/usr/local/bin/http-server /opt/maps --cors='*' -d false -i false --no-dotfiles -p %(ENV_OFFLINE_REGION_MAPS_PORT)s +command=/usr/bin/goStatic + -path /opt/maps + -port %(ENV_OFFLINE_REGION_MAPS_PORT)s + -append-header 'Access-Control-Allow-Origin:*' + -enable-health + -vhost '' autostart=true autorestart=true startsecs=0 diff --git a/docs/development.md b/docs/development.md index 1d2691abc..8e3b6cdee 100644 --- a/docs/development.md +++ b/docs/development.md @@ -53,14 +53,14 @@ Then, go take a walk or something since it will be a while. When you are done, y * `ghcr.io/idaholab/malcolm/api` (based on `python:3-slim`) * `ghcr.io/idaholab/malcolm/arkime` (based on `debian:12-slim`) -* `ghcr.io/idaholab/malcolm/dashboards-helper` (based on `alpine:3.20`) +* `ghcr.io/idaholab/malcolm/dashboards-helper` (based on `debian:12-slim`) * `ghcr.io/idaholab/malcolm/dashboards` (based on `opensearchproject/opensearch-dashboards`) * `ghcr.io/idaholab/malcolm/file-monitor` (based on `debian:12-slim`) * `ghcr.io/idaholab/malcolm/file-upload` (based on `debian:12-slim`) * `ghcr.io/idaholab/malcolm/filebeat-oss` (based on `docker.elastic.co/beats/filebeat-oss`) * `ghcr.io/idaholab/malcolm/freq` (based on `debian:12-slim`) * `ghcr.io/idaholab/malcolm/htadmin` (based on `debian:11-slim`) -* `ghcr.io/idaholab/malcolm/logstash-oss` (based on `opensearchproject/logstash-oss-with-opensearch-output-plugin`) +* `ghcr.io/idaholab/malcolm/logstash-oss` (based on `docker.elastic.co/logstash/logstash-oss`) * `ghcr.io/idaholab/malcolm/netbox` (based on `netboxcommunity/netbox:latest`) * `ghcr.io/idaholab/malcolm/nginx-proxy` (based on `alpine:3.20`) * `ghcr.io/idaholab/malcolm/opensearch` (based on `opensearchproject/opensearch`) From e28cf99b612a355390e8a9ceb668588ffdf079f4 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 19 Aug 2024 09:39:58 -0600 Subject: [PATCH 43/47] only try to expose opensearch port if we're actually doing local opensearch --- scripts/install.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/install.py b/scripts/install.py index 068970c0e..dd012b1f9 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -2282,9 +2282,10 @@ def tweak_malcolm_runtime(self, malcolm_install_path): data['services']['nginx-proxy']['ports'] = [ f"{'0.0.0.0:443' if nginxSSL else '127.0.0.1:80'}:443", ] - data['services']['nginx-proxy']['ports'].append( - f"{'0.0.0.0' if opensearchOpen else '127.0.0.1'}:{'9200' if nginxSSL else '9201'}:9200" - ) + if opensearchPrimaryMode == DatabaseMode.OpenSearchLocal: + data['services']['nginx-proxy']['ports'].append( + f"{'0.0.0.0' if opensearchOpen else '127.0.0.1'}:{'9200' if nginxSSL else '9201'}:9200" + ) # enable/disable/configure traefik labels if applicable for label in ( From 10d9e0515427ed5c6360c463a71fbde88f3131f4 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 26 Aug 2024 07:46:40 -0600 Subject: [PATCH 44/47] Don't install spicy-versions of built-in network analyzers --- Dockerfiles/zeek.Dockerfile | 6 ------ config/zeek.env.example | 3 --- .../includes.chroot/usr/local/etc/zeek/local.zeek | 12 ------------ hedgehog-iso/interface/sensor_ctl/control_vars.conf | 3 --- shared/bin/zeek_install_plugins.sh | 4 ---- zeek/config/local.zeek | 12 ------------ 6 files changed, 40 deletions(-) diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile index 0eb6e3341..bf858e300 100644 --- a/Dockerfiles/zeek.Dockerfile +++ b/Dockerfiles/zeek.Dockerfile @@ -249,9 +249,6 @@ ARG ZEEK_DISABLE_TRACK_ALL_ASSETS= ARG ZEEK_DISABLE_BEST_GUESS_ICS=true # TODO: assess spicy-analyzer that replace built-in Zeek parsers # for now, disable them by default when a Zeek parser exists -ARG ZEEK_DISABLE_SPICY_DHCP=true -ARG ZEEK_DISABLE_SPICY_DNS=true -ARG ZEEK_DISABLE_SPICY_HTTP=true ARG ZEEK_DISABLE_SPICY_IPSEC= ARG ZEEK_DISABLE_SPICY_LDAP= ARG ZEEK_DISABLE_SPICY_OPENVPN= @@ -269,9 +266,6 @@ ENV ZEEK_DISABLE_SSL_VALIDATE_CERTS $ZEEK_DISABLE_SSL_VALIDATE_CERTS ENV ZEEK_DISABLE_TRACK_ALL_ASSETS $ZEEK_DISABLE_TRACK_ALL_ASSETS ENV ZEEK_DISABLE_BEST_GUESS_ICS $ZEEK_DISABLE_BEST_GUESS_ICS -ENV ZEEK_DISABLE_SPICY_DHCP $ZEEK_DISABLE_SPICY_DHCP -ENV ZEEK_DISABLE_SPICY_DNS $ZEEK_DISABLE_SPICY_DNS -ENV ZEEK_DISABLE_SPICY_HTTP $ZEEK_DISABLE_SPICY_HTTP ENV ZEEK_DISABLE_SPICY_IPSEC $ZEEK_DISABLE_SPICY_IPSEC ENV ZEEK_DISABLE_SPICY_LDAP $ZEEK_DISABLE_SPICY_LDAP ENV ZEEK_DISABLE_SPICY_OPENVPN $ZEEK_DISABLE_SPICY_OPENVPN diff --git a/config/zeek.env.example b/config/zeek.env.example index 51106d3ec..512cb9d20 100644 --- a/config/zeek.env.example +++ b/config/zeek.env.example @@ -68,9 +68,6 @@ ZEEK_DISABLE_HASH_ALL_FILES= ZEEK_DISABLE_LOG_PASSWORDS= ZEEK_DISABLE_SSL_VALIDATE_CERTS= ZEEK_DISABLE_TRACK_ALL_ASSETS= -ZEEK_DISABLE_SPICY_DHCP=true -ZEEK_DISABLE_SPICY_DNS=true -ZEEK_DISABLE_SPICY_HTTP=true ZEEK_DISABLE_SPICY_IPSEC= ZEEK_DISABLE_SPICY_LDAP= ZEEK_DISABLE_SPICY_OPENVPN= diff --git a/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek b/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek index 85da475a9..a2794d2fe 100644 --- a/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek +++ b/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek @@ -18,9 +18,6 @@ global genisys_ports_str = getenv("ZEEK_GENISYS_PORTS"); global enip_ports_str = getenv("ZEEK_ENIP_PORTS"); global zeek_local_nets_str = getenv("ZEEK_LOCAL_NETS"); -global disable_spicy_dhcp = (getenv("ZEEK_DISABLE_SPICY_DHCP") == true_regex) ? T : F; -global disable_spicy_dns = (getenv("ZEEK_DISABLE_SPICY_DNS") == true_regex) ? T : F; -global disable_spicy_http = (getenv("ZEEK_DISABLE_SPICY_HTTP") == true_regex) ? T : F; global disable_spicy_ipsec = (getenv("ZEEK_DISABLE_SPICY_IPSEC") == true_regex) ? T : F; global disable_spicy_ldap = (getenv("ZEEK_DISABLE_SPICY_LDAP") == true_regex) ? T : F; global disable_spicy_openvpn = (getenv("ZEEK_DISABLE_SPICY_OPENVPN") == true_regex) ? T : F; @@ -170,15 +167,6 @@ event zeek_init() &priority=-5 { Spicy::disable_protocol_analyzer(Analyzer::ANALYZER_SYNCHROPHASOR_TCP); Spicy::disable_protocol_analyzer(Analyzer::ANALYZER_SYNCHROPHASOR_UDP); } - if (disable_spicy_dhcp) { - Spicy::disable_protocol_analyzer(Analyzer::ANALYZER_SPICY_DHCP); - } - if (disable_spicy_dns) { - Spicy::disable_protocol_analyzer(Analyzer::ANALYZER_SPICY_DNS); - } - if (disable_spicy_http) { - Spicy::disable_protocol_analyzer(Analyzer::ANALYZER_SPICY_HTTP); - } if (disable_spicy_ipsec) { Spicy::disable_protocol_analyzer(Analyzer::ANALYZER_SPICY_IPSEC_TCP); Spicy::disable_protocol_analyzer(Analyzer::ANALYZER_SPICY_IPSEC_UDP); diff --git a/hedgehog-iso/interface/sensor_ctl/control_vars.conf b/hedgehog-iso/interface/sensor_ctl/control_vars.conf index 532d27f40..dc7e5bef6 100644 --- a/hedgehog-iso/interface/sensor_ctl/control_vars.conf +++ b/hedgehog-iso/interface/sensor_ctl/control_vars.conf @@ -121,9 +121,6 @@ export ZEEK_DISABLE_HASH_ALL_FILES= export ZEEK_DISABLE_LOG_PASSWORDS= export ZEEK_DISABLE_SSL_VALIDATE_CERTS= export ZEEK_DISABLE_TRACK_ALL_ASSETS= -export ZEEK_DISABLE_SPICY_DHCP=true -export ZEEK_DISABLE_SPICY_DNS=true -export ZEEK_DISABLE_SPICY_HTTP=true export ZEEK_DISABLE_SPICY_IPSEC= export ZEEK_DISABLE_SPICY_LDAP= export ZEEK_DISABLE_SPICY_OPENVPN= diff --git a/shared/bin/zeek_install_plugins.sh b/shared/bin/zeek_install_plugins.sh index c14549d99..ab535b333 100755 --- a/shared/bin/zeek_install_plugins.sh +++ b/shared/bin/zeek_install_plugins.sh @@ -109,10 +109,6 @@ ZKG_GITHUB_URLS=( "https://github.com/ncsa/bro-simple-scan" "https://github.com/precurse/zeek-httpattacks" "https://github.com/mmguero-dev/GQUIC_Protocol_Analyzer" - "https://github.com/zeek/spicy-dhcp" - "https://github.com/zeek/spicy-dns" - "https://github.com/zeek/spicy-http" - "https://github.com/zeek/spicy-pe" "https://github.com/zeek/spicy-tftp" "https://github.com/zeek/spicy-zip" ) diff --git a/zeek/config/local.zeek b/zeek/config/local.zeek index ba4b7b1aa..7de26a3c7 100644 --- a/zeek/config/local.zeek +++ b/zeek/config/local.zeek @@ -18,9 +18,6 @@ global genisys_ports_str = getenv("ZEEK_GENISYS_PORTS"); global enip_ports_str = getenv("ZEEK_ENIP_PORTS"); global zeek_local_nets_str = getenv("ZEEK_LOCAL_NETS"); -global disable_spicy_dhcp = (getenv("ZEEK_DISABLE_SPICY_DHCP") == true_regex) ? T : F; -global disable_spicy_dns = (getenv("ZEEK_DISABLE_SPICY_DNS") == true_regex) ? T : F; -global disable_spicy_http = (getenv("ZEEK_DISABLE_SPICY_HTTP") == true_regex) ? T : F; global disable_spicy_ipsec = (getenv("ZEEK_DISABLE_SPICY_IPSEC") == true_regex) ? T : F; global disable_spicy_ldap = (getenv("ZEEK_DISABLE_SPICY_LDAP") == true_regex) ? T : F; global disable_spicy_openvpn = (getenv("ZEEK_DISABLE_SPICY_OPENVPN") == true_regex) ? T : F; @@ -170,15 +167,6 @@ event zeek_init() &priority=-5 { Spicy::disable_protocol_analyzer(Analyzer::ANALYZER_SYNCHROPHASOR_TCP); Spicy::disable_protocol_analyzer(Analyzer::ANALYZER_SYNCHROPHASOR_UDP); } - if (disable_spicy_dhcp) { - Spicy::disable_protocol_analyzer(Analyzer::ANALYZER_SPICY_DHCP); - } - if (disable_spicy_dns) { - Spicy::disable_protocol_analyzer(Analyzer::ANALYZER_SPICY_DNS); - } - if (disable_spicy_http) { - Spicy::disable_protocol_analyzer(Analyzer::ANALYZER_SPICY_HTTP); - } if (disable_spicy_ipsec) { Spicy::disable_protocol_analyzer(Analyzer::ANALYZER_SPICY_IPSEC_TCP); Spicy::disable_protocol_analyzer(Analyzer::ANALYZER_SPICY_IPSEC_UDP); From 536948516dbcd624d4cf48aeed32e2679e9bac87 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 26 Aug 2024 08:03:25 -0600 Subject: [PATCH 45/47] Don't install spicy-versions of built-in network analyzers --- Dockerfiles/zeek.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile index bf858e300..72f91a1a5 100644 --- a/Dockerfiles/zeek.Dockerfile +++ b/Dockerfiles/zeek.Dockerfile @@ -173,7 +173,7 @@ RUN groupadd --gid ${DEFAULT_GID} ${PUSER} && \ # sanity checks to make sure the plugins installed and copied over correctly # these ENVs should match the third party scripts/plugins installed by zeek_install_plugins.sh -ENV ZEEK_THIRD_PARTY_PLUGINS_GREP "(Zeek::Spicy|ANALYZER_SPICY_DHCP|ANALYZER_SPICY_DNS|ANALYZER_SPICY_HTTP|ANALYZER_SPICY_OSPF|ANALYZER_SPICY_OPENVPN_UDP\b|ANALYZER_SPICY_IPSEC_UDP\b|ANALYZER_SPICY_TFTP|ANALYZER_SPICY_WIREGUARD|ANALYZER_SYNCHROPHASOR_TCP|ANALYZER_GENISYS_TCP|ANALYZER_SPICY_GE_SRTP|ANALYZER_SPICY_PROFINET_IO_CM|ANALYZER_S7COMM_TCP|Corelight::PE_XOR|ICSNPP::BACnet|ICSNPP::BSAP|ICSNPP::ENIP|ICSNPP::ETHERCAT|ICSNPP::OPCUA_Binary|Salesforce::GQUIC|Zeek::PROFINET|Zeek::TDS)" +ENV ZEEK_THIRD_PARTY_PLUGINS_GREP "(Zeek::Spicy|ANALYZER_SPICY_OSPF|ANALYZER_SPICY_OPENVPN_UDP\b|ANALYZER_SPICY_IPSEC_UDP\b|ANALYZER_SPICY_TFTP|ANALYZER_SPICY_WIREGUARD|ANALYZER_SYNCHROPHASOR_TCP|ANALYZER_GENISYS_TCP|ANALYZER_SPICY_GE_SRTP|ANALYZER_SPICY_PROFINET_IO_CM|ANALYZER_S7COMM_TCP|Corelight::PE_XOR|ICSNPP::BACnet|ICSNPP::BSAP|ICSNPP::ENIP|ICSNPP::ETHERCAT|ICSNPP::OPCUA_Binary|Salesforce::GQUIC|Zeek::PROFINET|Zeek::TDS)" ENV ZEEK_THIRD_PARTY_SCRIPTS_GREP "(bro-is-darknet/main|bro-simple-scan/scan|bzar/main|callstranger-detector/callstranger|cve-2020-0601/cve-2020-0601|cve-2020-13777/cve-2020-13777|CVE-2020-16898/CVE-2020-16898|CVE-2021-38647/omigod|CVE-2021-31166/detect|CVE-2021-41773/CVE_2021_41773|CVE-2021-42292/main|cve-2021-44228/CVE_2021_44228|cve-2022-22954/main|cve-2022-26809/main|CVE-2022-3602/__load__|hassh/hassh|http-more-files-names/main|ja4/main|pingback/detect|ripple20/ripple20|SIGRed/CVE-2020-1350|zeek-EternalSafety/main|zeek-httpattacks/main|zeek-sniffpass/__load__|zerologon/main)\.(zeek|bro)" RUN mkdir -p /tmp/logs && \ From 7326f1d602495dba50e760703a4c7de731f37db3 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 26 Aug 2024 08:12:40 -0600 Subject: [PATCH 46/47] capa to v7.2.0 --- shared/bin/capa-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/bin/capa-build.sh b/shared/bin/capa-build.sh index 4c71a8fdc..c77970a3f 100644 --- a/shared/bin/capa-build.sh +++ b/shared/bin/capa-build.sh @@ -1,6 +1,6 @@ #!/bin/bash -export CAPA_VERSION="7.1.0" +export CAPA_VERSION="7.2.0" export CAPA_SRC_URL="https://github.com/mandiant/capa/archive/refs/tags/v${CAPA_VERSION}.zip" export CAPA_RULES_URL="https://github.com/mandiant/capa-rules/archive/refs/tags/v${CAPA_VERSION}.zip" From ee3495ffc2ca0c3b26e8c3f4af75c8529bc54377 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 26 Aug 2024 17:05:27 -0600 Subject: [PATCH 47/47] fix bug idaholab/Malcolm#548, missing cracklib-runtime --- .../config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot | 1 - .../config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot | 1 - 2 files changed, 2 deletions(-) diff --git a/hedgehog-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot b/hedgehog-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot index 528cb1b81..553c3da62 100755 --- a/hedgehog-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot +++ b/hedgehog-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot @@ -15,7 +15,6 @@ rm -rf /var/spool/ccache # remove unwanted packages apt-get -y --purge remove \ bluez-firmware \ - cracklib-runtime \ firmware-libertas \ firmware-netronome \ firmware-nvidia* \ diff --git a/malcolm-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot b/malcolm-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot index 196d7639a..7fba6edc2 100755 --- a/malcolm-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot +++ b/malcolm-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot @@ -5,7 +5,6 @@ apt-get -y --purge remove build-essential sparse $(dpkg --get-selections | grep # remove unwanted packages apt-get -y --purge remove bluez-firmware \ - cracklib-runtime \ docker-scan-plugin \ docker-buildx-plugin \ docker-ce-rootless-extras \