diff --git a/.github/workflows/api-build-and-push-ghcr.yml b/.github/workflows/api-build-and-push-ghcr.yml index f1f5a5d5f..5a3ab5eee 100644 --- a/.github/workflows/api-build-and-push-ghcr.yml +++ b/.github/workflows/api-build-and-push-ghcr.yml @@ -31,14 +31,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -46,23 +46,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/api.Dockerfile diff --git a/.github/workflows/arkime-build-and-push-ghcr.yml b/.github/workflows/arkime-build-and-push-ghcr.yml index ca9d674a5..51e258b03 100644 --- a/.github/workflows/arkime-build-and-push-ghcr.yml +++ b/.github/workflows/arkime-build-and-push-ghcr.yml @@ -31,14 +31,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -46,23 +46,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/arkime.Dockerfile diff --git a/.github/workflows/dashboards-build-and-push-ghcr.yml b/.github/workflows/dashboards-build-and-push-ghcr.yml index 20fb0371d..5f4e729ee 100644 --- a/.github/workflows/dashboards-build-and-push-ghcr.yml +++ b/.github/workflows/dashboards-build-and-push-ghcr.yml @@ -31,14 +31,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -46,23 +46,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/dashboards.Dockerfile diff --git a/.github/workflows/dashboards-helper-build-and-push-ghcr.yml b/.github/workflows/dashboards-helper-build-and-push-ghcr.yml index 624d2ac30..2a4fb6a47 100644 --- a/.github/workflows/dashboards-helper-build-and-push-ghcr.yml +++ b/.github/workflows/dashboards-helper-build-and-push-ghcr.yml @@ -31,14 +31,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -46,23 +46,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/dashboards-helper.Dockerfile diff --git a/.github/workflows/dirinit-build-and-push-ghcr.yml b/.github/workflows/dirinit-build-and-push-ghcr.yml index af49b0e66..c57ed918a 100644 --- a/.github/workflows/dirinit-build-and-push-ghcr.yml +++ b/.github/workflows/dirinit-build-and-push-ghcr.yml @@ -23,14 +23,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -38,23 +38,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/dirinit.Dockerfile diff --git a/.github/workflows/file-monitor-build-and-push-ghcr.yml b/.github/workflows/file-monitor-build-and-push-ghcr.yml index e056c0393..c74860015 100644 --- a/.github/workflows/file-monitor-build-and-push-ghcr.yml +++ b/.github/workflows/file-monitor-build-and-push-ghcr.yml @@ -31,14 +31,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -46,23 +46,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/file-monitor.Dockerfile diff --git a/.github/workflows/file-upload-build-and-push-ghcr.yml b/.github/workflows/file-upload-build-and-push-ghcr.yml index b04551dd6..eb42c5ee1 100644 --- a/.github/workflows/file-upload-build-and-push-ghcr.yml +++ b/.github/workflows/file-upload-build-and-push-ghcr.yml @@ -31,14 +31,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -46,23 +46,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/file-upload.Dockerfile diff --git a/.github/workflows/filebeat-build-and-push-ghcr.yml b/.github/workflows/filebeat-build-and-push-ghcr.yml index b7cf3a385..c56c1c6a8 100644 --- a/.github/workflows/filebeat-build-and-push-ghcr.yml +++ b/.github/workflows/filebeat-build-and-push-ghcr.yml @@ -31,14 +31,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -46,23 +46,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/filebeat.Dockerfile diff --git a/.github/workflows/freq-build-and-push-ghcr.yml b/.github/workflows/freq-build-and-push-ghcr.yml index a04a6d758..b8e589f19 100644 --- a/.github/workflows/freq-build-and-push-ghcr.yml +++ b/.github/workflows/freq-build-and-push-ghcr.yml @@ -31,14 +31,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -46,23 +46,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/freq.Dockerfile diff --git a/.github/workflows/htadmin-build-and-push-ghcr.yml b/.github/workflows/htadmin-build-and-push-ghcr.yml index b6173f98b..04ac783ce 100644 --- a/.github/workflows/htadmin-build-and-push-ghcr.yml +++ b/.github/workflows/htadmin-build-and-push-ghcr.yml @@ -31,14 +31,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -46,23 +46,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/htadmin.Dockerfile diff --git a/.github/workflows/logstash-build-and-push-ghcr.yml b/.github/workflows/logstash-build-and-push-ghcr.yml index 3151903e4..f358e2226 100644 --- a/.github/workflows/logstash-build-and-push-ghcr.yml +++ b/.github/workflows/logstash-build-and-push-ghcr.yml @@ -31,14 +31,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -46,23 +46,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/logstash.Dockerfile diff --git a/.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml b/.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml index 9f68aaaa8..488575fe8 100644 --- a/.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml +++ b/.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml @@ -29,17 +29,17 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master @@ -78,7 +78,7 @@ jobs: sudo rm -rf /tmp/live-build /tmp/live-build*.deb - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -139,14 +139,14 @@ jobs: sarif_file: 'trivy-results.sarif' - name: ghcr.io login - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push ISO image - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: ./malcolm-iso push: true diff --git a/.github/workflows/netbox-build-and-push-ghcr.yml b/.github/workflows/netbox-build-and-push-ghcr.yml index d67a45ec6..65cc11a71 100644 --- a/.github/workflows/netbox-build-and-push-ghcr.yml +++ b/.github/workflows/netbox-build-and-push-ghcr.yml @@ -31,14 +31,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -46,23 +46,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/netbox.Dockerfile diff --git a/.github/workflows/nginx-build-and-push-ghcr.yml b/.github/workflows/nginx-build-and-push-ghcr.yml index 4c63f216a..7ef3e9a7f 100644 --- a/.github/workflows/nginx-build-and-push-ghcr.yml +++ b/.github/workflows/nginx-build-and-push-ghcr.yml @@ -38,14 +38,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -58,23 +58,23 @@ jobs: id: extract_commit_sha - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/nginx.Dockerfile diff --git a/.github/workflows/opensearch-build-and-push-ghcr.yml b/.github/workflows/opensearch-build-and-push-ghcr.yml index 40b743459..225b89a2d 100644 --- a/.github/workflows/opensearch-build-and-push-ghcr.yml +++ b/.github/workflows/opensearch-build-and-push-ghcr.yml @@ -30,14 +30,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -45,23 +45,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/opensearch.Dockerfile diff --git a/.github/workflows/pcap-capture-build-and-push-ghcr.yml b/.github/workflows/pcap-capture-build-and-push-ghcr.yml index 7ded814d0..e8de170c4 100644 --- a/.github/workflows/pcap-capture-build-and-push-ghcr.yml +++ b/.github/workflows/pcap-capture-build-and-push-ghcr.yml @@ -31,14 +31,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -46,23 +46,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/pcap-capture.Dockerfile diff --git a/.github/workflows/pcap-monitor-build-and-push-ghcr.yml b/.github/workflows/pcap-monitor-build-and-push-ghcr.yml index aea7ac085..ca5138e18 100644 --- a/.github/workflows/pcap-monitor-build-and-push-ghcr.yml +++ b/.github/workflows/pcap-monitor-build-and-push-ghcr.yml @@ -31,14 +31,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -46,23 +46,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/pcap-monitor.Dockerfile diff --git a/.github/workflows/postgresql-build-and-push-ghcr.yml b/.github/workflows/postgresql-build-and-push-ghcr.yml index 36db42be1..29aefda44 100644 --- a/.github/workflows/postgresql-build-and-push-ghcr.yml +++ b/.github/workflows/postgresql-build-and-push-ghcr.yml @@ -30,14 +30,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -45,23 +45,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/postgresql.Dockerfile diff --git a/.github/workflows/redis-build-and-push-ghcr.yml b/.github/workflows/redis-build-and-push-ghcr.yml index b89103496..a04d0369b 100644 --- a/.github/workflows/redis-build-and-push-ghcr.yml +++ b/.github/workflows/redis-build-and-push-ghcr.yml @@ -30,14 +30,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -45,23 +45,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/redis.Dockerfile diff --git a/.github/workflows/sensor-iso-build-docker-wrap-push-ghcr.yml b/.github/workflows/sensor-iso-build-docker-wrap-push-ghcr.yml index ce522185f..d5ce7f2b2 100644 --- a/.github/workflows/sensor-iso-build-docker-wrap-push-ghcr.yml +++ b/.github/workflows/sensor-iso-build-docker-wrap-push-ghcr.yml @@ -27,17 +27,17 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master @@ -74,7 +74,7 @@ jobs: sudo rm -rf /tmp/live-build /tmp/live-build*.deb - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -97,6 +97,8 @@ jobs: cp ./scripts/malcolm_utils.py ./sensor-iso/shared/bin/ cp ./scripts/documentation_build.sh ./sensor-iso/docs/ cp -r ./arkime/patch ./sensor-iso/shared/arkime_patch + mkdir -p ./sensor-iso/suricata + cp -r ./suricata/rules-default ./sensor-iso/suricata/ pushd ./sensor-iso echo "${{ steps.extract_malcolm_version.outputs.mversion }}" > ./shared/version.txt echo "${{ secrets.MAXMIND_GEOIP_DB_LICENSE_KEY }}" > ./shared/maxmind_license.txt @@ -104,7 +106,7 @@ jobs: echo "VCS_REVSION=${{ steps.extract_commit_sha.outputs.sha }}" > ./shared/environment.chroot echo "BUILD_JOBS=2" > ./shared/environment.chroot sudo /usr/bin/env bash ./build.sh - rm -rf ./shared/ ./docs/ ./_config.yml ./_includes ./_layouts /Gemfile ./README.md + rm -rf ./shared/ ./docs/ ./_config.yml ./_includes ./_layouts /Gemfile ./README.md ./suricata sudo chmod 644 ./hedgehog-*.* popd - @@ -129,14 +131,14 @@ jobs: sarif_file: 'trivy-results.sarif' - name: ghcr.io login - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push ISO image - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: ./sensor-iso push: true diff --git a/.github/workflows/suricata-build-and-push-ghcr.yml b/.github/workflows/suricata-build-and-push-ghcr.yml index e76abb523..bd07b7006 100644 --- a/.github/workflows/suricata-build-and-push-ghcr.yml +++ b/.github/workflows/suricata-build-and-push-ghcr.yml @@ -31,14 +31,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -46,23 +46,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/suricata.Dockerfile diff --git a/.github/workflows/zeek-build-and-push-ghcr.yml b/.github/workflows/zeek-build-and-push-ghcr.yml index 2cd12c3ab..4cb14636c 100644 --- a/.github/workflows/zeek-build-and-push-ghcr.yml +++ b/.github/workflows/zeek-build-and-push-ghcr.yml @@ -30,14 +30,14 @@ jobs: steps: - name: Cancel previous run in progress - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.0 with: ignore_sha: true all_but_latest: true access_token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract branch name shell: bash @@ -45,23 +45,23 @@ jobs: id: extract_branch - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: driver-opts: | image=moby/buildkit:master - name: Log in to registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfiles/zeek.Dockerfile diff --git a/.gitignore b/.gitignore index 11b451b19..c47bd8a97 100644 --- a/.gitignore +++ b/.gitignore @@ -30,7 +30,7 @@ config.*/ .vagrant malcolm_*images.tar.gz malcolm_*images.tar.xz -malcolm_netbox_backup_*.psql.gz +malcolm_netbox_backup_*.gz *.iso *-build.log Gemfile.lock diff --git a/.trigger_iso_workflow_build b/.trigger_iso_workflow_build index 4ade3f725..bf8dee067 100644 --- a/.trigger_iso_workflow_build +++ b/.trigger_iso_workflow_build @@ -1,2 +1,2 @@ # this file exists solely for the purpose of being updated and seen by github to trigger a commit build action -1 \ No newline at end of file +2 \ No newline at end of file diff --git a/.trigger_workflow_build b/.trigger_workflow_build index 4ade3f725..bf8dee067 100644 --- a/.trigger_workflow_build +++ b/.trigger_workflow_build @@ -1,2 +1,2 @@ # this file exists solely for the purpose of being updated and seen by github to trigger a commit build action -1 \ No newline at end of file +2 \ No newline at end of file diff --git a/Dockerfiles/api.Dockerfile b/Dockerfiles/api.Dockerfile index 08809510e..7023d633a 100644 --- a/Dockerfiles/api.Dockerfile +++ b/Dockerfiles/api.Dockerfile @@ -20,7 +20,7 @@ RUN python3 -m pip wheel --no-cache-dir --no-deps --wheel-dir /usr/src/app/wheel FROM python:3-slim-bookworm -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' LABEL org.opencontainers.image.url='https://github.com/idaholab/Malcolm' diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index 96701db93..e1fbcc82e 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -1,6 +1,6 @@ FROM debian:12-slim AS build -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ENV DEBIAN_FRONTEND noninteractive ENV TERM xterm @@ -92,7 +92,13 @@ ENV DEFAULT_UID $DEFAULT_UID ENV DEFAULT_GID $DEFAULT_GID ENV PUSER "arkime" ENV PGROUP "arkime" -ENV PUSER_PRIV_DROP true +# not dropping privileges globally: supervisord will take care of it +# for all processes, but first we need root to sure capabilities for +# traffic capturing tools are in-place before they are started. +# despite doing setcap here in the Dockerfile, the chown in +# docker-uid-gid-setup.sh will cause them to be lost, so we need +# a final check in docker_entrypoint.sh before startup +ENV PUSER_PRIV_DROP false ENV PUSER_RLIMIT_UNLOCK true ENV DEBIAN_FRONTEND noninteractive @@ -106,18 +112,26 @@ ARG MALCOLM_USERNAME=admin ARG ARKIME_ECS_PROVIDER=arkime ARG ARKIME_ECS_DATASET=session ARG ARKIME_INTERFACE=eth0 -ARG ARKIME_ANALYZE_PCAP_THREADS=1 +ARG ARKIME_AUTO_ANALYZE_PCAP_FILES=false +ARG ARKIME_AUTO_ANALYZE_PCAP_THREADS=1 +ARG ARKIME_PACKET_THREADS=1 ARG OPENSEARCH_MAX_SHARDS_PER_NODE=2500 ARG WISE=on ARG VIEWER=on +ARG ARKIME_VIEWER_PORT=8005 #Whether or not Arkime is in charge of deleting old PCAP files to reclaim space ARG MANAGE_PCAP_FILES=false +ARG ARKIME_PCAP_PROCESSOR=true +ARG ARKIME_LIVE_CAPTURE=false +ARG ARKIME_ROTATED_PCAP=true +ARG ARKIME_COMPRESSION_TYPE=none +ARG ARKIME_COMPRESSION_LEVEL=0 + #Whether or not to auto-tag logs based on filename ARG AUTO_TAG=true ARG PCAP_PIPELINE_VERBOSITY="" ARG PCAP_MONITOR_HOST=pcap-monitor ARG PCAP_NODE_NAME=malcolm -ARG PCAP_NODE_HOST= ARG MAXMIND_GEOIP_DB_LICENSE_KEY="" # Declare envs vars for each arg @@ -130,16 +144,23 @@ ENV ARKIME_PASSWORD "ignored" ENV ARKIME_ECS_PROVIDER $ARKIME_ECS_PROVIDER ENV ARKIME_ECS_DATASET $ARKIME_ECS_DATASET ENV ARKIME_DIR "/opt/arkime" -ENV ARKIME_ANALYZE_PCAP_THREADS $ARKIME_ANALYZE_PCAP_THREADS +ENV ARKIME_AUTO_ANALYZE_PCAP_FILES $ARKIME_AUTO_ANALYZE_PCAP_FILES +ENV ARKIME_AUTO_ANALYZE_PCAP_THREADS $ARKIME_AUTO_ANALYZE_PCAP_THREADS +ENV ARKIME_PACKET_THREADS $ARKIME_PACKET_THREADS +ENV ARKIME_PCAP_PROCESSOR $ARKIME_PCAP_PROCESSOR +ENV ARKIME_LIVE_CAPTURE $ARKIME_LIVE_CAPTURE +ENV ARKIME_COMPRESSION_TYPE $ARKIME_COMPRESSION_TYPE +ENV ARKIME_COMPRESSION_LEVEL $ARKIME_COMPRESSION_LEVEL +ENV ARKIME_ROTATED_PCAP $ARKIME_ROTATED_PCAP ENV OPENSEARCH_MAX_SHARDS_PER_NODE $OPENSEARCH_MAX_SHARDS_PER_NODE ENV WISE $WISE ENV VIEWER $VIEWER +ENV ARKIME_VIEWER_PORT $ARKIME_VIEWER_PORT ENV MANAGE_PCAP_FILES $MANAGE_PCAP_FILES ENV AUTO_TAG $AUTO_TAG ENV PCAP_PIPELINE_VERBOSITY $PCAP_PIPELINE_VERBOSITY ENV PCAP_MONITOR_HOST $PCAP_MONITOR_HOST ENV PCAP_NODE_NAME $PCAP_NODE_NAME -ENV PCAP_NODE_HOST $PCAP_NODE_HOST COPY --from=build $ARKIME_DIR $ARKIME_DIR @@ -147,7 +168,9 @@ RUN sed -i "s/main$/main contrib non-free/g" /etc/apt/sources.list.d/debian.sour apt-get -q update && \ apt-get -y -q --no-install-recommends upgrade && \ apt-get install -q -y --no-install-recommends \ + bc \ curl \ + ethtool \ file \ geoip-bin \ gettext \ @@ -191,6 +214,7 @@ RUN sed -i "s/main$/main contrib non-free/g" /etc/apt/sources.list.d/debian.sour COPY --chmod=755 shared/bin/docker-uid-gid-setup.sh /usr/local/bin/ COPY --chmod=755 shared/bin/service_check_passthrough.sh /usr/local/bin/ COPY --chmod=755 shared/bin/self_signed_key_gen.sh /usr/local/bin/ +COPY --chmod=755 shared/bin/nic-capture-setup.sh /usr/local/bin/ COPY --chmod=755 shared/bin/opensearch_status.sh /opt COPY --chmod=755 shared/bin/pcap_processor.py /opt/ COPY --chmod=644 shared/bin/pcap_utils.py /opt/ @@ -199,6 +223,7 @@ COPY --chmod=644 shared/bin/watch_common.py /opt/ COPY --chmod=644 arkime/supervisord.conf /etc/supervisord.conf ADD arkime/scripts /opt/ ADD arkime/etc $ARKIME_DIR/etc/ +ADD arkime/rules/*.yml $ARKIME_DIR/rules/ ADD arkime/wise/source.*.js $ARKIME_DIR/wiseService/ COPY --from=ghcr.io/mmguero-dev/gostatic --chmod=755 /goStatic /usr/bin/goStatic @@ -211,12 +236,12 @@ RUN [ ${#MAXMIND_GEOIP_DB_LICENSE_KEY} -gt 1 ] && for DB in ASN Country City; do cd /tmp && \ curl -s -S -L -o "GeoLite2-$DB.mmdb.tar.gz" "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-$DB&license_key=$MAXMIND_GEOIP_DB_LICENSE_KEY&suffix=tar.gz" && \ tar xf "GeoLite2-$DB.mmdb.tar.gz" --wildcards --no-anchored '*.mmdb' --strip=1 && \ - mkdir -p $ARKIME_DIR/etc/ $ARKIME_DIR/logs/ && \ + mkdir -p $ARKIME_DIR/etc/ $ARKIME_DIR/rules/ $ARKIME_DIR/logs/ && \ mv -v "GeoLite2-$DB.mmdb" $ARKIME_DIR/etc/; \ rm -f "GeoLite2-$DB*"; \ done; \ curl -s -S -L -o $ARKIME_DIR/etc/ipv4-address-space.csv "https://www.iana.org/assignments/ipv4-address-space/ipv4-address-space.csv" && \ - curl -s -S -L -o $ARKIME_DIR/etc/oui.txt "https://gitlab.com/wireshark/wireshark/raw/release-4.0/manuf" + curl -s -S -L -o $ARKIME_DIR/etc/oui.txt "https://www.wireshark.org/download/automated/data/manuf" RUN groupadd --gid $DEFAULT_GID $PGROUP && \ useradd -M --uid $DEFAULT_UID --gid $DEFAULT_GID --home $ARKIME_DIR $PUSER && \ @@ -225,9 +250,13 @@ RUN groupadd --gid $DEFAULT_GID $PGROUP && \ ln -sfr /opt/pcap_processor.py /opt/pcap_arkime_processor.py && \ cp -f /opt/arkime_update_geo.sh $ARKIME_DIR/bin/arkime_update_geo.sh && \ mv $ARKIME_DIR/etc/config.ini $ARKIME_DIR/etc/config.orig.ini && \ - chmod u+s $ARKIME_DIR/bin/capture && \ + cp $ARKIME_DIR/bin/capture $ARKIME_DIR/bin/capture-offline && \ + chown root:${PGROUP} $ARKIME_DIR/bin/capture && \ + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' $ARKIME_DIR/bin/capture && \ + chown root:${PGROUP} /sbin/ethtool && \ + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool && \ mkdir -p /var/run/arkime && \ - chown -R $PUSER:$PGROUP $ARKIME_DIR/etc $ARKIME_DIR/logs /var/run/arkime + chown -R $PUSER:$PGROUP $ARKIME_DIR/etc $ARKIME_DIR/rules $ARKIME_DIR/logs /var/run/arkime #Update Path ENV PATH="/opt:$ARKIME_DIR/bin:${PATH}" diff --git a/Dockerfiles/dashboards-helper.Dockerfile b/Dockerfiles/dashboards-helper.Dockerfile index 72d86bf73..217c139fe 100644 --- a/Dockerfiles/dashboards-helper.Dockerfile +++ b/Dockerfiles/dashboards-helper.Dockerfile @@ -47,10 +47,10 @@ ENV DASHBOARDS_URL $DASHBOARDS_URL ENV DASHBOARDS_DARKMODE $DASHBOARDS_DARKMODE ENV PATH="/data:${PATH}" -ENV SUPERCRONIC_VERSION "0.2.28" +ENV SUPERCRONIC_VERSION "0.2.29" ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64" ENV SUPERCRONIC "supercronic-linux-amd64" -ENV SUPERCRONIC_SHA1SUM "fe1a81a8a5809deebebbd7a209a3b97e542e2bcd" +ENV SUPERCRONIC_SHA1SUM "cd48d45c4b10f3f0bfdd3a57d054cd05ac96812b" ENV SUPERCRONIC_CRONTAB "/etc/crontab" ENV ECS_RELEASES_URL "https://api.github.com/repos/elastic/ecs/releases/latest" diff --git a/Dockerfiles/dirinit.Dockerfile b/Dockerfiles/dirinit.Dockerfile index 806252251..e13689ee5 100644 --- a/Dockerfiles/dirinit.Dockerfile +++ b/Dockerfiles/dirinit.Dockerfile @@ -1,6 +1,6 @@ FROM alpine:3.18 -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' LABEL org.opencontainers.image.url='https://github.com/idaholab/Malcolm' @@ -24,7 +24,7 @@ COPY --chmod=755 shared/bin/docker-uid-gid-setup.sh /usr/local/bin/ RUN apk update --no-cache && \ apk upgrade --no-cache && \ - apk --no-cache add bash psmisc shadow tini && \ + apk --no-cache add bash psmisc rsync shadow tini && \ addgroup -g ${DEFAULT_GID} ${PGROUP} ; \ adduser -D -H -u ${DEFAULT_UID} -h /nonexistant -s /sbin/nologin -G ${PGROUP} -g ${PUSER} ${PUSER} ; \ addgroup ${PUSER} tty ; \ diff --git a/Dockerfiles/file-monitor.Dockerfile b/Dockerfiles/file-monitor.Dockerfile index 6bfd6b86e..6ed4d539e 100644 --- a/Dockerfiles/file-monitor.Dockerfile +++ b/Dockerfiles/file-monitor.Dockerfile @@ -1,6 +1,6 @@ FROM debian:12-slim -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' LABEL org.opencontainers.image.url='https://github.com/idaholab/Malcolm' @@ -93,10 +93,10 @@ ENV EXTRACTED_FILE_HTTP_SERVER_ENCRYPT $EXTRACTED_FILE_HTTP_SERVER_ENCRYPT ENV EXTRACTED_FILE_HTTP_SERVER_KEY $EXTRACTED_FILE_HTTP_SERVER_KEY ENV EXTRACTED_FILE_HTTP_SERVER_PORT $EXTRACTED_FILE_HTTP_SERVER_PORT -ENV SUPERCRONIC_VERSION "0.2.28" +ENV SUPERCRONIC_VERSION "0.2.29" ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64" ENV SUPERCRONIC "supercronic-linux-amd64" -ENV SUPERCRONIC_SHA1SUM "fe1a81a8a5809deebebbd7a209a3b97e542e2bcd" +ENV SUPERCRONIC_SHA1SUM "cd48d45c4b10f3f0bfdd3a57d054cd05ac96812b" ENV SUPERCRONIC_CRONTAB "/etc/crontab" COPY --chmod=755 shared/bin/yara_rules_setup.sh /usr/local/bin/ diff --git a/Dockerfiles/file-upload.Dockerfile b/Dockerfiles/file-upload.Dockerfile index 35175bc9a..fd582306a 100644 --- a/Dockerfiles/file-upload.Dockerfile +++ b/Dockerfiles/file-upload.Dockerfile @@ -1,6 +1,6 @@ FROM debian:12-slim AS npmget -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ENV DEBIAN_FRONTEND noninteractive @@ -49,10 +49,10 @@ ENV FILEPOND_SERVER_BRANCH $FILEPOND_SERVER_BRANCH ARG STALE_UPLOAD_DELETE_MIN=360 ENV STALE_UPLOAD_DELETE_MIN $STALE_UPLOAD_DELETE_MIN -ENV SUPERCRONIC_VERSION "0.2.28" +ENV SUPERCRONIC_VERSION "0.2.29" ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64" ENV SUPERCRONIC "supercronic-linux-amd64" -ENV SUPERCRONIC_SHA1SUM "fe1a81a8a5809deebebbd7a209a3b97e542e2bcd" +ENV SUPERCRONIC_SHA1SUM "cd48d45c4b10f3f0bfdd3a57d054cd05ac96812b" ENV SUPERCRONIC_CRONTAB "/etc/crontab" COPY --from=npmget /usr/local/lib/node_modules/filepond /var/www/upload/filepond diff --git a/Dockerfiles/filebeat.Dockerfile b/Dockerfiles/filebeat.Dockerfile index b8c4fcdbc..f3ee8e66c 100644 --- a/Dockerfiles/filebeat.Dockerfile +++ b/Dockerfiles/filebeat.Dockerfile @@ -1,6 +1,6 @@ -FROM docker.elastic.co/beats/filebeat-oss:8.11.1 +FROM docker.elastic.co/beats/filebeat-oss:8.11.3 -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' LABEL org.opencontainers.image.url='https://github.com/idaholab/Malcolm' @@ -60,11 +60,12 @@ ARG FILEBEAT_TCP_PARSE_SOURCE_FIELD="message" ARG FILEBEAT_TCP_PARSE_TARGET_FIELD="" ARG FILEBEAT_TCP_PARSE_DROP_FIELD="" ARG FILEBEAT_TCP_TAG="_malcolm_beats" +ARG PCAP_NODE_NAME=malcolm -ENV SUPERCRONIC_VERSION "0.2.28" +ENV SUPERCRONIC_VERSION "0.2.29" ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64" ENV SUPERCRONIC "supercronic-linux-amd64" -ENV SUPERCRONIC_SHA1SUM "fe1a81a8a5809deebebbd7a209a3b97e542e2bcd" +ENV SUPERCRONIC_SHA1SUM "cd48d45c4b10f3f0bfdd3a57d054cd05ac96812b" ENV SUPERCRONIC_CRONTAB "/etc/crontab" ENV TINI_VERSION v0.19.0 @@ -163,6 +164,7 @@ ENV FILEBEAT_TCP_PARSE_DROP_FIELD $FILEBEAT_TCP_PARSE_DROP_FIELD ENV FILEBEAT_TCP_TAG $FILEBEAT_TCP_TAG ENV FILEBEAT_REGISTRY_FILE "/usr/share/filebeat/data/registry/filebeat/data.json" ENV FILEBEAT_ZEEK_DIR "/zeek/" +ENV PCAP_NODE_NAME $PCAP_NODE_NAME VOLUME ["/usr/share/filebeat/data", "/usr/share/filebeat-nginx/data", "/usr/share/filebeat-tcp/data"] diff --git a/Dockerfiles/freq.Dockerfile b/Dockerfiles/freq.Dockerfile index a07318cef..82f9067b4 100644 --- a/Dockerfiles/freq.Dockerfile +++ b/Dockerfiles/freq.Dockerfile @@ -1,6 +1,6 @@ FROM debian:12-slim -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' LABEL org.opencontainers.image.url='https://github.com/idaholab/Malcolm' diff --git a/Dockerfiles/htadmin.Dockerfile b/Dockerfiles/htadmin.Dockerfile index 35c3728e9..ffcc27bd9 100644 --- a/Dockerfiles/htadmin.Dockerfile +++ b/Dockerfiles/htadmin.Dockerfile @@ -1,6 +1,6 @@ FROM debian:11-slim -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' LABEL org.opencontainers.image.url='https://github.com/idaholab/Malcolm' diff --git a/Dockerfiles/logstash.Dockerfile b/Dockerfiles/logstash.Dockerfile index ef0777b55..e5e792f2e 100644 --- a/Dockerfiles/logstash.Dockerfile +++ b/Dockerfiles/logstash.Dockerfile @@ -1,4 +1,4 @@ -FROM docker.elastic.co/logstash/logstash-oss:8.11.1 +FROM docker.elastic.co/logstash/logstash-oss:8.11.3 LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' diff --git a/Dockerfiles/netbox.Dockerfile b/Dockerfiles/netbox.Dockerfile index c075c03a2..aa8724f36 100644 --- a/Dockerfiles/netbox.Dockerfile +++ b/Dockerfiles/netbox.Dockerfile @@ -1,6 +1,6 @@ -FROM netboxcommunity/netbox:v3.6.6 +FROM netboxcommunity/netbox:v3.6.7 -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' LABEL org.opencontainers.image.url='https://github.com/idaholab/Malcolm' @@ -24,10 +24,10 @@ ENV PUSER "ubuntu" ENV PGROUP "ubuntu" ENV PUSER_PRIV_DROP true -ENV SUPERCRONIC_VERSION "0.2.28" +ENV SUPERCRONIC_VERSION "0.2.29" ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64" ENV SUPERCRONIC "supercronic-linux-amd64" -ENV SUPERCRONIC_SHA1SUM "fe1a81a8a5809deebebbd7a209a3b97e542e2bcd" +ENV SUPERCRONIC_SHA1SUM "cd48d45c4b10f3f0bfdd3a57d054cd05ac96812b" ENV SUPERCRONIC_CRONTAB "/etc/crontab" ENV YQ_VERSION "4.33.3" diff --git a/Dockerfiles/nginx.Dockerfile b/Dockerfiles/nginx.Dockerfile index 5765943bd..32428d198 100644 --- a/Dockerfiles/nginx.Dockerfile +++ b/Dockerfiles/nginx.Dockerfile @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. #################################################################################### # thanks to: nginx - https://github.com/nginxinc/docker-nginx/blob/master/mainline/alpine/Dockerfile diff --git a/Dockerfiles/opensearch.Dockerfile b/Dockerfiles/opensearch.Dockerfile index 95ea31bff..bdfccaf61 100644 --- a/Dockerfiles/opensearch.Dockerfile +++ b/Dockerfiles/opensearch.Dockerfile @@ -1,6 +1,6 @@ FROM opensearchproject/opensearch:2.11.1 -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' LABEL org.opencontainers.image.url='https://github.com/idaholab/Malcolm' diff --git a/Dockerfiles/pcap-capture.Dockerfile b/Dockerfiles/pcap-capture.Dockerfile index 6244eaad9..c582e9e26 100644 --- a/Dockerfiles/pcap-capture.Dockerfile +++ b/Dockerfiles/pcap-capture.Dockerfile @@ -1,6 +1,6 @@ FROM debian:12-slim -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' LABEL org.opencontainers.image.url='https://github.com/idaholab/Malcolm' diff --git a/Dockerfiles/pcap-monitor.Dockerfile b/Dockerfiles/pcap-monitor.Dockerfile index e8a3cd894..b1baabbee 100644 --- a/Dockerfiles/pcap-monitor.Dockerfile +++ b/Dockerfiles/pcap-monitor.Dockerfile @@ -1,6 +1,6 @@ FROM debian:12-slim -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' LABEL org.opencontainers.image.url='https://github.com/idaholab/Malcolm' diff --git a/Dockerfiles/postgresql.Dockerfile b/Dockerfiles/postgresql.Dockerfile index 757d13684..dde2e3b4d 100644 --- a/Dockerfiles/postgresql.Dockerfile +++ b/Dockerfiles/postgresql.Dockerfile @@ -1,6 +1,6 @@ FROM postgres:15-alpine -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' LABEL org.opencontainers.image.url='https://github.com/idaholab/Malcolm' diff --git a/Dockerfiles/redis.Dockerfile b/Dockerfiles/redis.Dockerfile index 6585f2ea2..a7ce82e61 100644 --- a/Dockerfiles/redis.Dockerfile +++ b/Dockerfiles/redis.Dockerfile @@ -1,6 +1,6 @@ FROM redis:7-alpine -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' LABEL org.opencontainers.image.url='https://github.com/idaholab/Malcolm' diff --git a/Dockerfiles/suricata.Dockerfile b/Dockerfiles/suricata.Dockerfile index ae4539970..62f026d4b 100644 --- a/Dockerfiles/suricata.Dockerfile +++ b/Dockerfiles/suricata.Dockerfile @@ -30,10 +30,10 @@ ENV PGROUP "suricata" ENV PUSER_PRIV_DROP false ENV PUSER_RLIMIT_UNLOCK true -ENV SUPERCRONIC_VERSION "0.2.28" +ENV SUPERCRONIC_VERSION "0.2.29" ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64" ENV SUPERCRONIC "supercronic-linux-amd64" -ENV SUPERCRONIC_SHA1SUM "fe1a81a8a5809deebebbd7a209a3b97e542e2bcd" +ENV SUPERCRONIC_SHA1SUM "cd48d45c4b10f3f0bfdd3a57d054cd05ac96812b" ENV SUPERCRONIC_CRONTAB "/etc/crontab" ENV YQ_VERSION "4.33.3" @@ -42,6 +42,7 @@ ENV YQ_URL "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_ ENV SURICATA_CONFIG_DIR /etc/suricata ENV SURICATA_CONFIG_FILE "$SURICATA_CONFIG_DIR"/suricata.yaml ENV SURICATA_CUSTOM_RULES_DIR /opt/suricata/rules +ENV SURICATA_DEFAULT_RULES_DIR /opt/suricata/rules-default ENV SURICATA_CUSTOM_CONFIG_DIR /opt/suricata/include-configs ENV SURICATA_LOG_DIR /var/log/suricata ENV SURICATA_MANAGED_DIR /var/lib/suricata @@ -52,15 +53,12 @@ ENV SURICATA_UPDATE_DIR "$SURICATA_MANAGED_DIR/update" ENV SURICATA_UPDATE_SOURCES_DIR "$SURICATA_UPDATE_DIR/sources" ENV SURICATA_UPDATE_CACHE_DIR "$SURICATA_UPDATE_DIR/cache" -COPY --chmod=644 suricata/default-rules/ /tmp/default-rules/ - RUN sed -i "s/main$/main contrib non-free/g" /etc/apt/sources.list.d/debian.sources && \ apt-get -q update && \ apt-get -y -q --no-install-recommends upgrade && \ apt-get install -q -y --no-install-recommends \ bc \ curl \ - ethtool \ file \ inotify-tools \ iproute2 \ @@ -117,15 +115,14 @@ RUN sed -i "s/main$/main contrib non-free/g" /etc/apt/sources.list.d/debian.sour usermod -a -G tty ${PUSER} && \ ln -sfr /usr/local/bin/pcap_processor.py /usr/local/bin/pcap_suricata_processor.py && \ (echo "*/5 * * * * /usr/local/bin/eve-clean-logs.sh\n0 */6 * * * /bin/bash /usr/local/bin/suricata-update-rules.sh\n" > ${SUPERCRONIC_CRONTAB}) && \ - mkdir -p "$SURICATA_CUSTOM_RULES_DIR" "$SURICATA_CUSTOM_CONFIG_DIR" && \ - chown -R ${PUSER}:${PGROUP} "$SURICATA_CUSTOM_RULES_DIR" "$SURICATA_CUSTOM_CONFIG_DIR" && \ + mkdir -p "$SURICATA_CUSTOM_RULES_DIR" "$SURICATA_DEFAULT_RULES_DIR" "$SURICATA_CUSTOM_CONFIG_DIR" && \ + chown -R ${PUSER}:${PGROUP} "$SURICATA_CUSTOM_RULES_DIR" "$SURICATA_DEFAULT_RULES_DIR" "$SURICATA_CUSTOM_CONFIG_DIR" && \ cp "$(dpkg -L suricata-update | grep 'update\.yaml$' | head -n 1)" \ "$SURICATA_UPDATE_CONFIG_FILE" && \ - find /tmp/default-rules/ -not -path '*/.gitignore' -type f -exec cp "{}" "$SURICATA_CONFIG_DIR"/rules/ \; && \ suricata-update update-sources --verbose --data-dir "$SURICATA_MANAGED_DIR" --config "$SURICATA_UPDATE_CONFIG_FILE" --suricata-conf "$SURICATA_CONFIG_FILE" && \ suricata-update update --fail --verbose --etopen --data-dir "$SURICATA_MANAGED_DIR" --config "$SURICATA_UPDATE_CONFIG_FILE" --suricata-conf "$SURICATA_CONFIG_FILE" && \ - chown root:${PGROUP} /sbin/ethtool /usr/bin/suricata && \ - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool && \ + cp /usr/bin/suricata /usr/bin/suricata-offline && \ + chown root:${PGROUP} /usr/bin/suricata && \ setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /usr/bin/suricata && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* @@ -135,13 +132,13 @@ COPY --chmod=644 suricata/supervisord.conf /etc/supervisord.conf COPY --chmod=755 shared/bin/docker-uid-gid-setup.sh /usr/local/bin/ COPY --chmod=755 shared/bin/service_check_passthrough.sh /usr/local/bin/ COPY --from=ghcr.io/mmguero-dev/gostatic --chmod=755 /goStatic /usr/bin/goStatic -COPY --chmod=755 shared/bin/nic-capture-setup.sh /usr/local/bin/ COPY --chmod=755 shared/bin/pcap_processor.py /usr/local/bin/ COPY --chmod=644 scripts/malcolm_utils.py /usr/local/bin/ COPY --chmod=755 shared/bin/suricata_config_populate.py /usr/local/bin/ COPY --chmod=755 suricata/scripts/docker_entrypoint.sh /usr/local/bin/ COPY --chmod=755 suricata/scripts/eve-clean-logs.sh /usr/local/bin/ COPY --chmod=755 suricata/scripts/suricata-update-rules.sh /usr/local/bin/ +COPY --chmod=u=rwX,go=rX suricata/rules-default/ "$SURICATA_DEFAULT_RULES_DIR"/ ARG PCAP_PIPELINE_VERBOSITY="" ARG PCAP_MONITOR_HOST=pcap-monitor @@ -161,6 +158,7 @@ ARG SURICATA_ROTATED_PCAP=false ARG PCAP_IFACE=lo ARG PCAP_IFACE_TWEAK=false ARG PCAP_FILTER= +ARG PCAP_NODE_NAME=malcolm ENV PCAP_PIPELINE_VERBOSITY $PCAP_PIPELINE_VERBOSITY ENV PCAP_MONITOR_HOST $PCAP_MONITOR_HOST @@ -179,6 +177,8 @@ ENV SURICATA_ROTATED_PCAP $SURICATA_ROTATED_PCAP ENV PCAP_IFACE $PCAP_IFACE ENV PCAP_IFACE_TWEAK $PCAP_IFACE_TWEAK ENV PCAP_FILTER $PCAP_FILTER +ENV PCAP_NODE_NAME $PCAP_NODE_NAME + ENV PUSER_CHOWN "$SURICATA_CONFIG_DIR;$SURICATA_MANAGED_DIR;$SURICATA_LOG_DIR;$SURICATA_RUN_DIR" diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile index 6c5c0cfc4..b71ae6801 100644 --- a/Dockerfiles/zeek.Dockerfile +++ b/Dockerfiles/zeek.Dockerfile @@ -62,7 +62,7 @@ RUN apt-get -q update && \ FROM debian:12-slim -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' @@ -92,12 +92,11 @@ ENV PGROUP "zeeker" # docker-uid-gid-setup.sh will cause them to be lost, so we need # a final check in docker_entrypoint.sh before startup ENV PUSER_PRIV_DROP false -ENV PUSER_RLIMIT_UNLOCK true -ENV SUPERCRONIC_VERSION "0.2.28" +ENV SUPERCRONIC_VERSION "0.2.29" ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64" ENV SUPERCRONIC "supercronic-linux-amd64" -ENV SUPERCRONIC_SHA1SUM "fe1a81a8a5809deebebbd7a209a3b97e542e2bcd" +ENV SUPERCRONIC_SHA1SUM "cd48d45c4b10f3f0bfdd3a57d054cd05ac96812b" ENV SUPERCRONIC_CRONTAB "/etc/crontab" # for download and install @@ -133,7 +132,6 @@ RUN export DEBARCH=$(dpkg --print-architecture) && \ clang \ cmake \ curl \ - ethtool \ file \ flex \ git \ @@ -224,7 +222,6 @@ ADD zeek/config/*.txt ${ZEEK_DIR}/share/zeek/site/ ADD zeek/scripts/docker_entrypoint.sh /usr/local/bin/ ADD shared/bin/zeek_intel_setup.sh ${ZEEK_DIR}/bin/ ADD shared/bin/zeekdeploy.sh ${ZEEK_DIR}/bin/ -ADD shared/bin/nic-capture-setup.sh /usr/local/bin/ # sanity checks to make sure the plugins installed and copied over correctly # these ENVs should match the number of third party scripts/plugins installed by zeek_install_plugins.sh @@ -245,10 +242,10 @@ RUN mkdir -p /tmp/logs && \ RUN groupadd --gid ${DEFAULT_GID} ${PUSER} && \ useradd -M --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} --home /nonexistant ${PUSER} && \ usermod -a -G tty ${PUSER} && \ - chown root:${PGROUP} /sbin/ethtool "${ZEEK_DIR}"/bin/zeek "${ZEEK_DIR}"/bin/capstats && \ - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool && \ - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' "${ZEEK_DIR}"/bin/zeek && \ - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' "${ZEEK_DIR}"/bin/capstats && \ + cp "${ZEEK_DIR}"/bin/zeek "${ZEEK_DIR}"/bin/zeek-offline && \ + chown root:${PGROUP} "${ZEEK_DIR}"/bin/zeek "${ZEEK_DIR}"/bin/capstats && \ + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' "${ZEEK_DIR}"/bin/zeek && \ + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' "${ZEEK_DIR}"/bin/capstats && \ touch "${SUPERCRONIC_CRONTAB}" && \ chown -R ${DEFAULT_UID}:${DEFAULT_GID} "${ZEEK_DIR}"/share/zeek/site/intel "${SUPERCRONIC_CRONTAB}" && \ ln -sfr /usr/local/bin/pcap_processor.py /usr/local/bin/pcap_zeek_processor.py && \ @@ -269,6 +266,8 @@ ARG ZEEK_INTEL_REFRESH_THREADS=2 ARG ZEEK_INTEL_FEED_SINCE= ARG ZEEK_EXTRACTOR_MODE=none ARG ZEEK_EXTRACTOR_PATH=/zeek/extract_files +ARG ZEEK_INTEL_PATH=/opt/zeek/share/zeek/site/intel +ARG ZEEK_CUSTOM_PATH=/opt/zeek/share/zeek/site/custom ARG PCAP_PIPELINE_VERBOSITY="" ARG PCAP_MONITOR_HOST=pcap-monitor ARG ZEEK_LIVE_CAPTURE=false @@ -277,6 +276,7 @@ ARG ZEEK_ROTATED_PCAP=false ARG PCAP_IFACE=lo ARG PCAP_IFACE_TWEAK=false ARG PCAP_FILTER= +ARG PCAP_NODE_NAME=malcolm ENV AUTO_TAG $AUTO_TAG ENV ZEEK_PCAP_PROCESSOR $ZEEK_PCAP_PROCESSOR @@ -289,6 +289,8 @@ ENV ZEEK_INTEL_REFRESH_THREADS $ZEEK_INTEL_REFRESH_THREADS ENV ZEEK_INTEL_FEED_SINCE $ZEEK_INTEL_FEED_SINCE ENV ZEEK_EXTRACTOR_MODE $ZEEK_EXTRACTOR_MODE ENV ZEEK_EXTRACTOR_PATH $ZEEK_EXTRACTOR_PATH +ENV ZEEK_INTEL_PATH $ZEEK_INTEL_PATH +ENV ZEEK_CUSTOM_PATH $ZEEK_CUSTOM_PATH ENV PCAP_PIPELINE_VERBOSITY $PCAP_PIPELINE_VERBOSITY ENV PCAP_MONITOR_HOST $PCAP_MONITOR_HOST ENV ZEEK_LIVE_CAPTURE $ZEEK_LIVE_CAPTURE @@ -296,6 +298,7 @@ ENV ZEEK_ROTATED_PCAP $ZEEK_ROTATED_PCAP ENV PCAP_IFACE $PCAP_IFACE ENV PCAP_IFACE_TWEAK $PCAP_IFACE_TWEAK ENV PCAP_FILTER $PCAP_FILTER +ENV PCAP_NODE_NAME $PCAP_NODE_NAME # environment variables for zeek runtime tweaks (used in local.zeek) ARG ZEEK_DISABLE_HASH_ALL_FILES= diff --git a/LICENSE.txt b/LICENSE.txt index db2310bed..e4db15ef7 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright 2023, Battelle Energy Alliance, LLC +Copyright 2024 Battelle Energy Alliance, LLC Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/NOTICE.txt b/NOTICE.txt index 55d121c11..4365ece5a 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -4,7 +4,7 @@ https://github.com/idaholab/Malcolm See LICENSE.txt for license terms. -Malcolm is Copyright (c) 2023 Battelle Energy Alliance, LLC, and is developed +Malcolm is Copyright (c) 2024 Battelle Energy Alliance, LLC, and is developed and released through the cooperation of the Cybersecurity and Infrastructure Security Agency of the U.S. Department of Homeland Security. All rights reserved. diff --git a/README.md b/README.md index 31b2adab6..fb4a574d1 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ You can help steer Malcolm's development by sharing your ideas and feedback. Ple ## Copyright and License -Malcolm is Copyright 2023 Battelle Energy Alliance, LLC, and is developed and released through the cooperation of the [Cybersecurity and Infrastructure Security Agency](https://www.cisa.gov/) of the [U.S. Department of Homeland Security](https://www.dhs.gov/). +Malcolm is Copyright 2024 Battelle Energy Alliance, LLC, and is developed and released through the cooperation of the [Cybersecurity and Infrastructure Security Agency](https://www.cisa.gov/) of the [U.S. Department of Homeland Security](https://www.dhs.gov/). Malcolm is licensed under the Apache License, version 2.0. See `LICENSE.txt` for the terms of its release. diff --git a/api/requirements.txt b/api/requirements.txt index 19243e4ca..f76b2a87f 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -5,5 +5,5 @@ opensearch-py==2.4.2 requests==2.31.0 regex==2022.3.2 dateparser==1.1.1 -elasticsearch==8.11.0 +elasticsearch==8.11.1 elasticsearch-dsl==8.11.0 \ No newline at end of file diff --git a/arkime/arkime_regression_test_harness/docker-compose.yml b/arkime/arkime_regression_test_harness/docker-compose.yml index c205d0a7a..1e112a7a6 100644 --- a/arkime/arkime_regression_test_harness/docker-compose.yml +++ b/arkime/arkime_regression_test_harness/docker-compose.yml @@ -13,12 +13,6 @@ services: cluster.routing.allocation.node_initial_primaries_recoveries : 8 expose: - 9200 - ulimits: - memlock: - soft: -1 - hard: -1 - cap_add: - - IPC_LOCK healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9200"] interval: 30s diff --git a/arkime/etc/config.ini b/arkime/etc/config.ini index 3caf5a270..34aed4c57 100644 --- a/arkime/etc/config.ini +++ b/arkime/etc/config.ini @@ -4,69 +4,83 @@ # [default] -elasticsearch=http://opensearch:9200 +antiSynDrop=false +certFile=/opt/arkime/etc/viewer.crt +compressES=false cronQueries=true -rotateIndex=daily -passwordSecret=Malcolm +dropGroup=arkime +dropUser=arkime +elasticsearch=http://opensearch:9200 +freeSpaceG=10% +geoLite2ASN=/opt/arkime/etc/GeoLite2-ASN.mmdb +geoLite2Country=/opt/arkime/etc/GeoLite2-Country.mmdb httpRealm=Arkime -userAuthIps=::,0.0.0.0/0 +icmpTimeout=10 interface=eth0 -wiseHost=127.0.0.1 -wisePort=8081 +bpf= +keyFile=/opt/arkime/etc/viewer.key +logESRequests=false +logEveryXPackets=500000 +logFileCreation=true +logHTTPConnections=false +logUnknownProtocols=false +maxESConns=30 +maxESRequests=500 +maxFileSizeG=4 +maxFileTimeM=180 +maxPackets=10000 +maxReqBody=64 +maxStreams=1000000 +ouiFile=/opt/arkime/etc/oui.txt +packetsPerPoll=50000 +parseQSValue=false +parsersDir=/opt/arkime/parsers +parseSMB=true +parseSMTP=true +passwordSecret=Malcolm pcapDir=/data/pcap/processed +plugins=wise.so +pluginsDir=/opt/arkime/plugins readTruncatedPackets=true -maxFileSizeG=4 -tcpTimeout=600 +reqBodyOnlyUtf8=true +rirFile=/opt/arkime/etc/ipv4-address-space.csv +rotateIndex=daily +rulesFiles= +smtpIpHeaders=X-Originating-IP:;X-Barracuda-Apparent-Source-IP: +spiDataMaxIndices=7 +supportSha256=false tcpSaveTimeout=720 +tcpTimeout=600 udpTimeout=30 -icmpTimeout=10 -maxStreams=1000000 -maxPackets=10000 -freeSpaceG=10% +uploadCommand= +userAuthIps=::,0.0.0.0/0 +viewerPlugins=wise.js viewPort=8005 -certFile=/opt/arkime/etc/viewer.crt -keyFile=/opt/arkime/etc/viewer.key -geoLite2Country=/opt/arkime/etc/GeoLite2-Country.mmdb -geoLite2ASN=/opt/arkime/etc/GeoLite2-ASN.mmdb -rirFile=/opt/arkime/etc/ipv4-address-space.csv -ouiFile=/opt/arkime/etc/oui.txt -dropUser=arkime -dropGroup=arkime +wiseHost=127.0.0.1 +wisePort=8081 # implicit auto-creation of users for Arkime (see https://github.com/arkime/arkime/pull/1120) # The userAutoCreateTmpl should more or less match what's in /etc/user_settings.json # which is what's used when creating the default admin user. userNameHeader=http_auth_http_user userAutoCreateTmpl={"userId": "${this.http_auth_http_user}", "userName": "${this.http_auth_http_user}", "enabled": true, "createEnabled": false, "webEnabled": true, "headerAuthEnabled": true, "emailSearch": true, "removeEnabled": false, "packetSearch": true, "hideStats": false, "hideFiles": false, "hidePcap": false, "disablePcapDownload": false, "settings": { "timezone": "local", "detailFormat": "last", "showTimestamps": "last", "sortColumn": "firstPacket", "sortDirection": "desc", "spiGraph": "protocol", "connSrcField": "source.ip", "connDstField": "destination.ip", "numPackets": "last", "theme" : "custom1: #222222,#E2E2E2,#FFFFFF,#00789E,#004A79,#017D73,#092B40,#42b7c5,#2A7580,#ecb30a,#333333,#89ADCC,#6D6D6D,#FFE7E7,#ECFEFF", "manualQuery": false }, "tableStates": { "sessionsNew": { "order": [ [ "firstPacket", "desc" ] ], "visibleHeaders": [ "protocol", "event.provider", "event.dataset", "firstPacket", "lastPacket", "src", "source.port", "dst", "destination.port", "network.packets", "dbby", "tags", "info" ] } } } -parseSMTP=true -parseSMB=true -parseQSValue=false -supportSha256=false -maxReqBody=64 -reqBodyOnlyUtf8=true -smtpIpHeaders=X-Originating-IP:;X-Barracuda-Apparent-Source-IP: -parsersDir=/opt/arkime/parsers -pluginsDir=/opt/arkime/plugins -plugins=wise.so -viewerPlugins=wise.js -spiDataMaxIndices=7 -uploadCommand= -packetThreads=2 -pcapWriteMethod=simple -pcapWriteSize=262143 -simpleCompression=zstd -simpleZstdLevel=3 -compressES=false -maxESConns=30 -maxESRequests=500 -packetsPerPoll=50000 -antiSynDrop=true -logEveryXPackets=100000 -logUnknownProtocols=false -logESRequests=true -logFileCreation=true # temporarily disabling viewer autocomplete to see if it helps slugishness valueAutoComplete=false +### High Performance settings +# https://github.com/arkime/arkime/wiki/Settings#High_Performance_Settings +magicMode=basic +pcapReadMethod=tpacketv3 +tpacketv3NumThreads=2 +tpacketv3BlockSize=8388608 +pcapWriteMethod=simple +pcapWriteSize=2560000 +simpleCompression=none +simpleZstdLevel=3 +simpleGzipLevel=3 +packetThreads=2 +maxPacketsInQueue=300000 +dbBulkSize=4000000 + [custom-fields] # see https://docs.zeek.org/en/stable/script-reference/log-files.html for Zeek logfile documentation diff --git a/arkime-logs/.gitignore b/arkime/rules/.gitignore similarity index 100% rename from arkime-logs/.gitignore rename to arkime/rules/.gitignore diff --git a/arkime/rules/single_session_no_spi.yml b/arkime/rules/single_session_no_spi.yml new file mode 100644 index 000000000..fdfb73069 --- /dev/null +++ b/arkime/rules/single_session_no_spi.yml @@ -0,0 +1,11 @@ +--- +version: 1 +rules: + - name: "Dont save SPI sessions with only 1 source packet" + when: "beforeFinalSave" + fields: + packets.src: 1 + packets.dst: 0 + tcpflags.syn: 1 + ops: + _dontSaveSPI: 1 diff --git a/arkime/rules/ssh_trunate.yml b/arkime/rules/ssh_trunate.yml new file mode 100644 index 000000000..4d5b099d9 --- /dev/null +++ b/arkime/rules/ssh_trunate.yml @@ -0,0 +1,10 @@ +--- +version: 1 +rules: + - name: "Only save first n packets of SSH" + when: "fieldSet" + fields: + protocols: + - ssh + ops: + _maxPacketsToSave: 20 diff --git a/arkime/rules/tls_trunate.yml b/arkime/rules/tls_trunate.yml new file mode 100644 index 000000000..0e51a8e3d --- /dev/null +++ b/arkime/rules/tls_trunate.yml @@ -0,0 +1,10 @@ +--- +version: 1 +rules: + - name: "Only save first n packets of TLS" + when: "fieldSet" + fields: + protocols: + - tls + ops: + _maxPacketsToSave: 15 diff --git a/arkime/scripts/arkime_update_geo.sh b/arkime/scripts/arkime_update_geo.sh index 64be5973e..aed3c1ccf 100755 --- a/arkime/scripts/arkime_update_geo.sh +++ b/arkime/scripts/arkime_update_geo.sh @@ -1,6 +1,6 @@ #!/bin/sh -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. cd "${ARKIME_DIR:-/opt/arkime}"/etc @@ -8,7 +8,7 @@ wget -nv --no-check-certificate -O ipv4-address-space.csv_new https://www.iana.o mv -f ipv4-address-space.csv_new ipv4-address-space.csv || \ rm -f ipv4-address-space.csv_new -wget -nv -O oui.txt_new https://gitlab.com/wireshark/wireshark/raw/release-4.0/manuf && \ +wget -nv -O oui.txt_new https://www.wireshark.org/download/automated/data/manuf && \ mv -f oui.txt_new oui.txt || \ rm -f oui.txt_new diff --git a/arkime/scripts/docker_entrypoint.sh b/arkime/scripts/docker_entrypoint.sh index a7b2fe542..1a6b4b031 100755 --- a/arkime/scripts/docker_entrypoint.sh +++ b/arkime/scripts/docker_entrypoint.sh @@ -9,8 +9,13 @@ function urlencodeall() { } ARKIME_DIR=${ARKIME_DIR:-"/opt/arkime"} +ARKIME_RULES_DIR=${ARKIME_RULES_DIR:-"/opt/arkime/rules"} +ARKIME_CONFIG_FILE="${ARKIME_DIR}"/etc/config.ini ARKIME_PASSWORD_SECRET=${ARKIME_PASSWORD_SECRET:-"Malcolm"} ARKIME_FREESPACEG=${ARKIME_FREESPACEG:-"10%"} +CAPTURE_INTERFACE=${PCAP_IFACE:-} +LIVE_CAPTURE=${ARKIME_LIVE_CAPTURE:-false} +VIEWER_PORT=${ARKIME_VIEWER_PORT:-8005} MALCOLM_PROFILE=${MALCOLM_PROFILE:-"malcolm"} OPENSEARCH_URL_FINAL=${OPENSEARCH_URL:-"http://opensearch:9200"} @@ -45,22 +50,106 @@ if ( [[ "$OPENSEARCH_PRIMARY" == "opensearch-remote" ]] || [[ "$OPENSEARCH_PRIMA OPENSEARCH_URL_FINAL="${PROTOCOL}${NEW_USER}:${NEW_PASSWORD}@${HOSTPORT}" fi -if [[ -r "${ARKIME_DIR}"/etc/config.orig.ini ]]; then - cp "${ARKIME_DIR}"/etc/config.orig.ini "${ARKIME_DIR}"/etc/config.ini - sed -i "s|^\(elasticsearch=\).*|\1"${OPENSEARCH_URL_FINAL}"|" "${ARKIME_DIR}"/etc/config.ini - sed -i "s/^\(passwordSecret=\).*/\1"${ARKIME_PASSWORD_SECRET}"/" "${ARKIME_DIR}"/etc/config.ini - sed -i "s/^\(freeSpaceG=\).*/\1"${ARKIME_FREESPACEG}"/" "${ARKIME_DIR}"/etc/config.ini - if [[ "$MALCOLM_PROFILE" == "hedgehog" ]]; then - sed -i "s/^\(userNameHeader=\)/# \1/" "${ARKIME_DIR}"/etc/config.ini - sed -i "s/^\(userAuthIps=\)/# \1/" "${ARKIME_DIR}"/etc/config.ini - sed -i "s/^\(userAutoCreateTmpl=\)/# \1/" "${ARKIME_DIR}"/etc/config.ini - sed -i "s/^\(wiseHost=\)/# \1/" "${ARKIME_DIR}"/etc/config.ini - sed -i "s/^\(wisePort=\)/# \1/" "${ARKIME_DIR}"/etc/config.ini - sed -i "s/^\(plugins=\)/# \1/" "${ARKIME_DIR}"/etc/config.ini - sed -i "s/^\(viewerPlugins=\)/# \1/" "${ARKIME_DIR}"/etc/config.ini - sed -i '/^\[custom-fields\]/,$d' "${ARKIME_DIR}"/etc/config.ini +# iff config.ini does not exist but config.orig.ini does, use it as a basis and modify based on env. vars +if [[ ! -f "${ARKIME_CONFIG_FILE}" ]] && [[ -r "${ARKIME_DIR}"/etc/config.orig.ini ]]; then + cp "${ARKIME_DIR}"/etc/config.orig.ini "${ARKIME_CONFIG_FILE}" + + sed -i "s|^\(elasticsearch=\).*|\1"${OPENSEARCH_URL_FINAL}"|" "${ARKIME_CONFIG_FILE}" + sed -i "s/^\(passwordSecret=\).*/\1"${ARKIME_PASSWORD_SECRET}"/" "${ARKIME_CONFIG_FILE}" + sed -i "s/^\(freeSpaceG=\).*/\1"${ARKIME_FREESPACEG}"/" "${ARKIME_CONFIG_FILE}" + sed -i "s/^\(viewPort=\).*/\1"${VIEWER_PORT}"/" "${ARKIME_CONFIG_FILE}" + sed -i "s/^\(pcapDir=\).*/\1\/data\/pcap\/arkime-live/" "${ARKIME_CONFIG_FILE}" + + # performance tuning parameters + [[ -n "$ARKIME_DB_BULK_SIZE" ]] && \ + sed -r -i "s/(dbBulkSize)\s*=\s*.*/\1=$ARKIME_DB_BULK_SIZE/" "${ARKIME_CONFIG_FILE}" + [[ -n "$ARKIME_MAGIC_MODE" ]] && \ + sed -r -i "s/(magicMode)\s*=\s*.*/\1=$ARKIME_MAGIC_MODE/" "${ARKIME_CONFIG_FILE}" + [[ -n "$ARKIME_MAX_PACKETS_IN_QUEUE" ]] && \ + sed -r -i "s/(maxPacketsInQueue)\s*=\s*.*/\1=$ARKIME_MAX_PACKETS_IN_QUEUE/" "${ARKIME_CONFIG_FILE}" + [[ -n "$ARKIME_PACKET_THREADS" ]] && \ + sed -r -i "s/(packetThreads)\s*=\s*.*/\1=$ARKIME_PACKET_THREADS/" "${ARKIME_CONFIG_FILE}" + [[ -n "$ARKIME_PCAP_WRITE_METHOD" ]] && \ + sed -r -i "s/(pcapWriteMethod)\s*=\s*.*/\1=$ARKIME_PCAP_WRITE_METHOD/" "${ARKIME_CONFIG_FILE}" + [[ -n "$ARKIME_PCAP_WRITE_SIZE" ]] && \ + sed -r -i "s/(pcapWriteSize)\s*=\s*.*/\1=$ARKIME_PCAP_WRITE_SIZE/" "${ARKIME_CONFIG_FILE}" + [[ -n "$ARKIME_PCAP_READ_METHOD" ]] && \ + sed -r -i "s/(pcapReadMethod)\s*=\s*.*/\1=$ARKIME_PCAP_READ_METHOD/" "${ARKIME_CONFIG_FILE}" + [[ -n "$ARKIME_TPACKETV3_NUM_THREADS" ]] && \ + sed -r -i "s/(tpacketv3NumThreads)\s*=\s*.*/\1=$ARKIME_TPACKETV3_NUM_THREADS/" "${ARKIME_CONFIG_FILE}" + [[ -n "$ARKIME_TPACKETV3_BLOCK_SIZE" ]] && \ + sed -r -i "s/(tpacketv3BlockSize)\s*=\s*.*/\1=$ARKIME_TPACKETV3_BLOCK_SIZE/" "${ARKIME_CONFIG_FILE}" + + # capture interface(s) + if [[ -n "$CAPTURE_INTERFACE" ]] && [[ "$LIVE_CAPTURE" == "true" ]] ; then + + # in config.ini multiple interfaces are separated by ; + ARKIME_CAPTURE_INTERFACE="$(echo "$CAPTURE_INTERFACE" | sed "s/,/;/g")" + + # place capture interfaces in the config file + sed -r -i "s|(interface)\s*=\s*.*|\1=$ARKIME_CAPTURE_INTERFACE|" "${ARKIME_CONFIG_FILE}" + sed -i "s/^\(readTruncatedPackets=\).*/\1"false"/" "${ARKIME_CONFIG_FILE}" + sed -r -i "s/(bpf)\s*=\s*.*/\1=${PCAP_FILTER:-}/" "${ARKIME_CONFIG_FILE}" + + # convert pcap rotation size units (MB to GB) and stick in config file + if [[ -n $PCAP_ROTATE_MEGABYTES ]]; then + PCAP_ROTATE_GIGABYTES=$(echo "($PCAP_ROTATE_MEGABYTES + 1024 - 1)/1024" | bc) + sed -r -i "s/(maxFileSizeG)\s*=\s*.*/\1=$PCAP_ROTATE_GIGABYTES/" "${ARKIME_CONFIG_FILE}" + fi + + # convert pcap rotation time units (sec to min) and stick in config file + if [[ -n $PCAP_ROTATE_SECONDS ]]; then + PCAP_ROTATE_MINUTES=$(echo "($PCAP_ROTATE_SECONDS + 60 - 1)/60" | bc) + sed -r -i "s/(maxFileTimeM)\s*=\s*.*/\1=$PCAP_ROTATE_MINUTES/" "${ARKIME_CONFIG_FILE}" + fi + + # pcap compression + COMPRESSION_TYPE="${ARKIME_COMPRESSION_TYPE:-none}" + COMPRESSION_LEVEL="${ARKIME_COMPRESSION_LEVEL:-0}" + sed -r -i "s/(simpleCompression)\s*=\s*.*/\1=$COMPRESSION_TYPE/" "${ARKIME_CONFIG_FILE}" + if [[ "$COMPRESSION_TYPE" == "zstd" ]]; then + sed -r -i "s/(simpleZstdLevel)\s*=\s*.*/\1=$COMPRESSION_LEVEL/" "${ARKIME_CONFIG_FILE}" + elif [[ "$COMPRESSION_TYPE" == "gzip" ]]; then + sed -r -i "s/(simpleGzipLevel)\s*=\s*.*/\1=$COMPRESSION_LEVEL/" "${ARKIME_CONFIG_FILE}" + fi + + # ensure capabilities for capture + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' "${ARKIME_DIR}"/bin/capture || true + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool || true + + # disable NIC hardware offloading features and adjust ring buffer sizes for each interface + for IFACE in ${CAPTURE_INTERFACE//,/ }; do + + [[ "${PCAP_IFACE_TWEAK:-false}" == "true" ]] && \ + [[ "$IFACE" != "lo" ]] && \ + [[ -x /usr/local/bin/nic-capture-setup.sh ]] && \ + /usr/local/bin/nic-capture-setup.sh "$IFACE" >/dev/null 2>&1 || true + + done # loop over capture interfaces + + fi # capture interface(s) defined and live capture enabled + + # rules files + if [[ -d "${ARKIME_RULES_DIR}" ]]; then + RULES_FILES="$(find "${ARKIME_RULES_DIR}" -mindepth 1 -maxdepth 1 -type f -size +0c \( -name '*.yml' -o -name '*.yaml' \) | tr '\n' ';' | sed 's/;$//' )" + sed -r -i "s|(rulesFiles)\s*=\s*.*|\1=$RULES_FILES|" "${ARKIME_CONFIG_FILE}" + fi + + # comment-out features that are unused in hedgehog run profile mode and in live-capture mode + if [[ "$MALCOLM_PROFILE" == "hedgehog" ]] || [[ "$LIVE_CAPTURE" == "true" ]]; then + sed -i "s/^\(userNameHeader=\)/# \1/" "${ARKIME_CONFIG_FILE}" + sed -i "s/^\(userAuthIps=\)/# \1/" "${ARKIME_CONFIG_FILE}" + sed -i "s/^\(userAutoCreateTmpl=\)/# \1/" "${ARKIME_CONFIG_FILE}" + sed -i "s/^\(wiseHost=\)/# \1/" "${ARKIME_CONFIG_FILE}" + sed -i "s/^\(wisePort=\)/# \1/" "${ARKIME_CONFIG_FILE}" + sed -i "s/^\(plugins=\)/# \1/" "${ARKIME_CONFIG_FILE}" + sed -i "s/^\(viewerPlugins=\)/# \1/" "${ARKIME_CONFIG_FILE}" + sed -i '/^\[custom-fields\]/,$d' "${ARKIME_CONFIG_FILE}" fi - chmod 600 "${ARKIME_DIR}"/etc/config.ini + + chmod 600 "${ARKIME_CONFIG_FILE}" || true + [[ -n ${PUID} ]] && chown -f ${PUID} "${ARKIME_CONFIG_FILE}" || true + [[ -n ${PGID} ]] && chown -f :${PGID} "${ARKIME_CONFIG_FILE}" || true fi unset OPENSEARCH_URL_FINAL diff --git a/arkime/scripts/initarkime.sh b/arkime/scripts/initarkime.sh index 06f59eb28..a47caab03 100755 --- a/arkime/scripts/initarkime.sh +++ b/arkime/scripts/initarkime.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. MALCOLM_PROFILE=${MALCOLM_PROFILE:-"malcolm"} OPENSEARCH_URL=${OPENSEARCH_URL:-"http://opensearch:9200"} @@ -35,12 +35,12 @@ fi if [[ "$MALCOLM_PROFILE" == "malcolm" ]]; then - echo "Giving $OPENSEARCH_PRIMARY time to start..." - /opt/opensearch_status.sh 2>&1 && echo "$OPENSEARCH_PRIMARY is running!" - # download and/or update geo updates $ARKIME_DIR/bin/arkime_update_geo.sh + echo "Giving $OPENSEARCH_PRIMARY time to start..." + /opt/opensearch_status.sh 2>&1 && echo "$OPENSEARCH_PRIMARY is running!" + # start and wait patiently for WISE if [[ "$WISE" = "on" ]] ; then touch /var/run/arkime/runwise @@ -56,7 +56,7 @@ if [[ "$MALCOLM_PROFILE" == "malcolm" ]]; then fi # initialize the contents of the OpenSearch database if it has never been initialized (ie., the users_v# table hasn't been created) - if [[ $(curl "${CURL_CONFIG_PARAMS[@]}" -fs -XGET -H'Content-Type: application/json' "${OPENSEARCH_URL}/_cat/indices/arkime_users_v*" | wc -l) < 1 ]]; then + if (( $(curl "${CURL_CONFIG_PARAMS[@]}" -fs -XGET -H'Content-Type: application/json' "${OPENSEARCH_URL}/_cat/indices/arkime_users_v*" | wc -l) < 1 )); then echo "Initializing $OPENSEARCH_PRIMARY database..." @@ -71,7 +71,7 @@ if [[ "$MALCOLM_PROFILE" == "malcolm" ]]; then # this is a hacky way to get all of the Arkime-parseable field definitions put into E.S. touch /tmp/not_a_packet.pcap - $ARKIME_DIR/bin/capture $DB_SSL_FLAG --packetcnt 0 -r /tmp/not_a_packet.pcap >/dev/null 2>&1 + $ARKIME_DIR/bin/capture-offline $DB_SSL_FLAG --packetcnt 0 -r /tmp/not_a_packet.pcap >/dev/null 2>&1 rm -f /tmp/not_a_packet.pcap echo "Initializing views..." diff --git a/arkime/scripts/live_capture.sh b/arkime/scripts/live_capture.sh new file mode 100755 index 000000000..19072706a --- /dev/null +++ b/arkime/scripts/live_capture.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. + +ARKIME_DIR=${ARKIME_DIR:-"/opt/arkime"} +CERT_FILE="${ARKIME_DIR}"/etc/viewer.crt +KEY_FILE="${ARKIME_DIR}"/etc/viewer.key +ARKIME_PACKET_THREADS=${ARKIME_PACKET_THREADS:-1} +PUSER=${PUSER:-"arkime"} +PGROUP=${PGROUP:-"arkime"} +NODE_NAME=${PCAP_NODE_NAME:-"malcolm"}-live +NODE_HOST=${ARKIME_LIVE_NODE_HOST:-""} + +OPENSEARCH_PRIMARY=${OPENSEARCH_PRIMARY:-"opensearch-local"} +OPENSEARCH_URL=${OPENSEARCH_URL:-"http://opensearch:9200"} +OPENSEARCH_PRIMARY=${OPENSEARCH_PRIMARY:-"opensearch-local"} +OPENSEARCH_SSL_CERTIFICATE_VERIFICATION=${OPENSEARCH_SSL_CERTIFICATE_VERIFICATION:-"false"} +OPENSEARCH_CREDS_CONFIG_FILE=${OPENSEARCH_CREDS_CONFIG_FILE:-"/var/local/curlrc/.opensearch.primary.curlrc"} +if ( [[ "$OPENSEARCH_PRIMARY" == "opensearch-remote" ]] || [[ "$OPENSEARCH_PRIMARY" == "elasticsearch-remote" ]] ) && [[ -r "$OPENSEARCH_CREDS_CONFIG_FILE" ]]; then + CURL_CONFIG_PARAMS=( + --config + "$OPENSEARCH_CREDS_CONFIG_FILE" + ) +else + CURL_CONFIG_PARAMS=() +fi + +rm -f /var/run/arkime/initialized /var/run/arkime/runwise + +# make sure TLS certificates exist prior to starting up +CERT_FILE=$ARKIME_DIR/etc/viewer.crt +KEY_FILE=$ARKIME_DIR/etc/viewer.key +if ( [[ ! -f "$CERT_FILE" ]] || [[ ! -f "$KEY_FILE" ]] ) && [[ -x /usr/local/bin/self_signed_key_gen.sh ]]; then + rm -f "$CERT_FILE" "$KEY_FILE" ./newcerts + pushd $ARKIME_DIR/etc/ >/dev/null 2>&1 + /usr/local/bin/self_signed_key_gen.sh -n -o ./newcerts >/dev/null 2>&1 + mv ./newcerts/server.crt "$CERT_FILE" + mv ./newcerts/server.key "$KEY_FILE" + rm -rf ./newcerts + popd >/dev/null 2>&1 +fi + +# download and/or update geo updates +$ARKIME_DIR/bin/arkime_update_geo.sh + +# we haven't dropUser/dropGroup'ed yet, so make sure the regular user owns the files we just touched +[[ -n ${PUID} ]] && [[ -n ${PGID} ]] && chown -f -R ${PUID}:${PGID} "${ARKIME_DIR}"/etc/ || true + +# wait patiently for the non-live Arkime to initialize the database +echo "Giving $OPENSEARCH_PRIMARY time to start..." +/opt/opensearch_status.sh -t malcolm_template 2>&1 && echo "$OPENSEARCH_PRIMARY is running!" +echo "Giving Arkime time to initialize..." +sleep 5 +until (( $(curl "${CURL_CONFIG_PARAMS[@]}" -fs -XGET -H'Content-Type: application/json' "${OPENSEARCH_URL}/_cat/indices/arkime_users_v*" | wc -l) >= 1 )); do + sleep 1 +done + +# this will also allow viewer to kick off +touch /var/run/arkime/initialized + +echo "Arkime is initialized!" +echo + +"${ARKIME_DIR}"/bin/capture --insecure \ + -c "${ARKIME_DIR}"/etc/config.ini \ + -o pcapDir=/data/pcap/arkime-live \ + -o dropUser=${PUSER} \ + -o dropGroup=${PGROUP} \ + -o ecsEventProvider=arkime \ + -o ecsEventDataset=session \ + --node "${NODE_NAME}" \ + --host "${NODE_HOST}" diff --git a/arkime/scripts/viewer_service.sh b/arkime/scripts/viewer_service.sh index d1785b49b..00314c460 100755 --- a/arkime/scripts/viewer_service.sh +++ b/arkime/scripts/viewer_service.sh @@ -1,13 +1,15 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. + +[[ "${ARKIME_LIVE_CAPTURE:-false}" == "true" ]] && LIVE_NODE_FLAG=-live || LIVE_NODE_FLAG= while true; do if [[ -f /var/run/arkime/initialized && "$VIEWER" == "on" ]]; then echo "Launch viewer..." rm -f $ARKIME_DIR/logs/viewer* pushd $ARKIME_DIR/viewer >/dev/null 2>&1 - $ARKIME_DIR/bin/node viewer.js --insecure -n "${PCAP_NODE_NAME:-malcolm}" -c $ARKIME_DIR/etc/config.ini | tee -a $ARKIME_DIR/logs/viewer.log 2>&1 + $ARKIME_DIR/bin/node viewer.js --insecure -n "${PCAP_NODE_NAME:-malcolm}${LIVE_NODE_FLAG}" -c $ARKIME_DIR/etc/config.ini popd >/dev/null 2>&1 fi sleep 5 diff --git a/arkime/scripts/wipearkime.sh b/arkime/scripts/wipearkime.sh index 99b010118..6779dd6d1 100755 --- a/arkime/scripts/wipearkime.sh +++ b/arkime/scripts/wipearkime.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. [[ ${OPENSEARCH_SSL_CERTIFICATE_VERIFICATION:-"false"} != "true" ]] && DB_SSL_FLAG="--insecure" || DB_SSL_FLAG="" OPENSEARCH_URL_FULL="$(grep -Pi '^elasticsearch\s*=' $ARKIME_DIR/etc/config.ini | cut -d'=' -f2-)" diff --git a/arkime/scripts/wise_service.sh b/arkime/scripts/wise_service.sh index ff9e26b34..199d39977 100755 --- a/arkime/scripts/wise_service.sh +++ b/arkime/scripts/wise_service.sh @@ -1,13 +1,13 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. while true; do if [[ ("$WISE" == "on") && (-f /var/run/arkime/runwise) && (-f $ARKIME_DIR/etc/wise.ini) ]]; then echo "Launch wise..." rm -f $ARKIME_DIR/logs/wise* pushd $ARKIME_DIR/wiseService >/dev/null 2>&1 - $ARKIME_DIR/bin/node wiseService.js --insecure -c $ARKIME_DIR/etc/wise.ini | tee -a $ARKIME_DIR/logs/wise.log 2>&1 + $ARKIME_DIR/bin/node wiseService.js --insecure -c $ARKIME_DIR/etc/wise.ini popd >/dev/null 2>&1 fi sleep 5 diff --git a/arkime/supervisord.conf b/arkime/supervisord.conf index 44d0d10ed..57ba0656d 100644 --- a/arkime/supervisord.conf +++ b/arkime/supervisord.conf @@ -1,4 +1,4 @@ -; Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +; Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. [unix_http_server] file=/tmp/supervisor.sock ; (the path to the socket file) @@ -6,6 +6,7 @@ chmod=0700 [supervisord] nodaemon=true +user=root logfile=/dev/null logfile_maxbytes=0 pidfile=/tmp/supervisord.pid @@ -20,7 +21,7 @@ serverurl=unix:///tmp/supervisor.sock command=/opt/initarkime.sh startsecs=0 startretries=0 -autostart=true +autostart=%(ENV_ARKIME_PCAP_PROCESSOR)s autorestart=false stopasgroup=true killasgroup=true @@ -28,6 +29,7 @@ directory=%(ENV_ARKIME_DIR)s stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 redirect_stderr=true +user=%(ENV_PUSER)s [program:wise] command=/opt/wise_service.sh @@ -38,8 +40,10 @@ autorestart=true stopasgroup=true killasgroup=true directory=%(ENV_ARKIME_DIR)s/wiseService -stdout_logfile=%(ENV_ARKIME_DIR)s/logs/wise.log +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 redirect_stderr=true +user=%(ENV_PUSER)s [program:viewer] command=/opt/viewer_service.sh @@ -53,25 +57,44 @@ directory=%(ENV_ARKIME_DIR)s stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 redirect_stderr=true +user=%(ENV_PUSER)s [program:pcap-arkime] command=python3 /opt/pcap_arkime_processor.py %(ENV_PCAP_PIPELINE_VERBOSITY)s --start-sleep 10 - --threads %(ENV_ARKIME_ANALYZE_PCAP_THREADS)s + --threads %(ENV_ARKIME_AUTO_ANALYZE_PCAP_THREADS)s --publisher "%(ENV_PCAP_MONITOR_HOST)s" --pcap-directory /data/pcap/processed --node "%(ENV_PCAP_NODE_NAME)s" - --host "%(ENV_PCAP_NODE_HOST)s" - --arkime /opt/arkime/bin/capture + --arkime "%(ENV_ARKIME_DIR)s"/bin/capture-offline + --autoarkime "%(ENV_ARKIME_AUTO_ANALYZE_PCAP_FILES)s" + --forcearkime "%(ENV_ARKIME_ROTATED_PCAP)s" --autotag "%(ENV_AUTO_TAG)s" --managed "%(ENV_MANAGE_PCAP_FILES)s" startsecs=15 startretries=1 -autostart=true -autorestart=true +autostart=%(ENV_ARKIME_PCAP_PROCESSOR)s +autorestart=%(ENV_ARKIME_PCAP_PROCESSOR)s stopasgroup=true killasgroup=true directory=%(ENV_ARKIME_DIR)s stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 redirect_stderr=true +user=%(ENV_PUSER)s + +[program:live-arkime] +command=/opt/live_capture.sh +autostart=%(ENV_ARKIME_LIVE_CAPTURE)s +autorestart=%(ENV_ARKIME_LIVE_CAPTURE)s +startsecs=30 +startretries=2000000000 +stopwaitsecs=15 +stopasgroup=true +killasgroup=true +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 +redirect_stderr=true +user=root ; fear not, capture will dropUser/dropGroup to PUSER +directory=/data/pcap/arkime-live + diff --git a/arkime/wise/source.zeeklogs.js b/arkime/wise/source.zeeklogs.js index ddd1b0e1c..40a9c4035 100644 --- a/arkime/wise/source.zeeklogs.js +++ b/arkime/wise/source.zeeklogs.js @@ -10,7 +10,7 @@ const WISESource = require('./wiseSource.js'); // Data may be populated with Malcolm's Logstash filters: // (https://github.com/idaholab/Malcolm/tree/main/logstash/pipelines) // -// Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +// Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. // see https://raw.githubusercontent.com/idaholab/Malcolm/main/LICENSE.txt ////////////////////////////////////////////////////////////////////////////////// diff --git a/config/arkime-live.env.example b/config/arkime-live.env.example new file mode 100644 index 000000000..da1d54b02 --- /dev/null +++ b/config/arkime-live.env.example @@ -0,0 +1,25 @@ +# Whether or Arkime should monitor live traffic on a local +# interface (PCAP_IFACE in pcap-capture.env specifies interface) +ARKIME_LIVE_CAPTURE=false + +# Live capture tuning parameters +ARKIME_COMPRESSION_TYPE=none +ARKIME_COMPRESSION_LEVEL=0 +ARKIME_DB_BULK_SIZE=4000000 +ARKIME_MAGIC_MODE=basic +ARKIME_MAX_PACKETS_IN_QUEUE=300000 +ARKIME_PACKET_THREADS=2 +ARKIME_PCAP_WRITE_METHOD=simple +ARKIME_PCAP_WRITE_SIZE=2560000 +ARKIME_PCAP_READ_METHOD=tpacketv3 +ARKIME_TPACKETV3_NUM_THREADS=2 +ARKIME_TPACKETV3_BLOCK_SIZE=8388608 + +# The node host (e.g., the IP address of the machine running Malcolm) to associate with +# network traffic metadata when ARKIME_LIVE_CAPTURE is true +# (optional, defaults to PCAP_NODE_NAME if unspecified) +ARKIME_LIVE_NODE_HOST= + +ARKIME_PCAP_PROCESSOR=false +VIEWER=on +WISE=off \ No newline at end of file diff --git a/config/arkime-offline.env.example b/config/arkime-offline.env.example new file mode 100644 index 000000000..aa53cdc87 --- /dev/null +++ b/config/arkime-offline.env.example @@ -0,0 +1,14 @@ +# Whether or not Arkime should analyze uploaded PCAP files +ARKIME_AUTO_ANALYZE_PCAP_FILES=true +# The number of Arkime processes for analyzing uploaded PCAP files allowed +# to run concurrently +ARKIME_AUTO_ANALYZE_PCAP_THREADS=1 +# Whether or not Arkime should analyze captured PCAP files captured +# by netsniff-ng/tcpdump (see PCAP_ENABLE_NETSNIFF and PCAP_ENABLE_TCPDUMP +# below). If ARKIME_LIVE_CAPTURE is true, this should be false: otherwise +# Arkime will see duplicate traffic. +ARKIME_ROTATED_PCAP=true + +ARKIME_PCAP_PROCESSOR=true +VIEWER=on +WISE=on \ No newline at end of file diff --git a/config/arkime.env.example b/config/arkime.env.example index 8248a636d..3df6ecfc1 100644 --- a/config/arkime.env.example +++ b/config/arkime.env.example @@ -1,8 +1,7 @@ +ARKIME_VIEWER_PORT=8005 # Whether or not Arkime is allowed to delete uploaded/captured PCAP (see # https://arkime.com/faq#pcap-deletion) MANAGE_PCAP_FILES=false ARKIME_FREESPACEG=10% -# The number of Arkime capture processes allowed to run concurrently -ARKIME_ANALYZE_PCAP_THREADS=1 OPENSEARCH_MAX_SHARDS_PER_NODE=2500 \ No newline at end of file diff --git a/config/suricata.env.example b/config/suricata.env.example index aada5d39a..d652854ae 100644 --- a/config/suricata.env.example +++ b/config/suricata.env.example @@ -3,6 +3,7 @@ SURICATA_CUSTOM_RULES_ONLY=false SURICATA_UPDATE_RULES=false SURICATA_UPDATE_DEBUG=false SURICATA_UPDATE_ETOPEN=true +SURICATA_DISABLE_ICS_ALL=false # suricata_config_populate.py can use MANY more environment variables to tweak # suricata.yaml (see https://github.com/OISF/suricata/blob/master/suricata.yaml.in and # https://suricata.readthedocs.io/en/latest/configuration/suricata-yaml.html). diff --git a/config/upload-common.env.example b/config/upload-common.env.example index 17a89fd95..ad55df213 100644 --- a/config/upload-common.env.example +++ b/config/upload-common.env.example @@ -4,9 +4,6 @@ AUTO_TAG=true # The node name (e.g., the hostname of this machine running Malcolm) to associate with # network traffic metadata PCAP_NODE_NAME=malcolm -# The node host (e.g., the IP address of the machine running Malcolm) to associate with -# network traffic metadata (optional, defaults to PCAP_NODE_NAME if unspecified) -PCAP_NODE_HOST= # Verbosity flag for pcap pipeline debugging (e.g., -v, -vv, -vvv, etc.) PCAP_PIPELINE_VERBOSITY= # Whether or not PCAP files extant in ./pcap/ will be ignored on startup diff --git a/config/zeek-live.env.example b/config/zeek-live.env.example index 3659dacac..4cf6fc0cb 100644 --- a/config/zeek-live.env.example +++ b/config/zeek-live.env.example @@ -5,6 +5,4 @@ ZEEK_LIVE_CAPTURE=false ZEEK_PCAP_PROCESSOR=false ZEEK_CRON=true ZEEK_LOG_PATH=/zeek/live -ZEEK_INTEL_PATH=/opt/zeek/share/zeek/site/intel -ZEEK_CUSTOM_PATH=/opt/zeek/share/zeek/site/custom EXTRACT_FILES_PATH=/zeek/extract_files \ No newline at end of file diff --git a/config/zeek.env.example b/config/zeek.env.example index ca0c9d6c0..7e0e64249 100644 --- a/config/zeek.env.example +++ b/config/zeek.env.example @@ -12,6 +12,8 @@ ZEEK_INTEL_FEED_SINCE= # Specifies a cron expression indicating the refresh interval for generating the # Zeek Intelligence Framework files ('' disables automatic refresh) ZEEK_INTEL_REFRESH_CRON_EXPRESSION= +# Number of threads to use for querying feeds for generating Zeek Intelligence Framework files +ZEEK_INTEL_REFRESH_THREADS=2 # Determines the file extraction behavior for file transfers detected by Zeek ZEEK_EXTRACTOR_MODE=none # Whether or not to use polling vs. native inotify API to watch for files diff --git a/dashboards/dashboards/beats/db0312a0-e842-11ec-88e0-67bf497b7120.json b/dashboards/dashboards/beats/db0312a0-e842-11ec-88e0-67bf497b7120.json deleted file mode 100644 index 06ff9ba5b..000000000 --- a/dashboards/dashboards/beats/db0312a0-e842-11ec-88e0-67bf497b7120.json +++ /dev/null @@ -1,212 +0,0 @@ -{ - "version": "2.0.0", - "objects": [ - { - "id": "db0312a0-e842-11ec-88e0-67bf497b7120", - "type": "dashboard", - "namespaces": [ - "default" - ], - "updated_at": "2022-06-09T23:00:39.461Z", - "version": "WzkyMywxXQ==", - "attributes": { - "title": "Malcolm Sensor Syslog", - "hits": 0, - "description": "Syslog logs from Malcolm sensors", - "panelsJSON": "[{\"version\":\"2.0.0\",\"gridData\":{\"x\":0,\"y\":0,\"w\":9,\"h\":19,\"i\":\"e1d6d345-c417-4f95-8907-17806a08cc34\"},\"panelIndex\":\"e1d6d345-c417-4f95-8907-17806a08cc34\",\"embeddableConfig\":{},\"panelRefName\":\"panel_0\"},{\"version\":\"2.0.0\",\"gridData\":{\"x\":9,\"y\":0,\"w\":9,\"h\":19,\"i\":\"ec19724f-f84a-4dde-acff-f539df5afef4\"},\"panelIndex\":\"ec19724f-f84a-4dde-acff-f539df5afef4\",\"embeddableConfig\":{},\"panelRefName\":\"panel_1\"},{\"version\":\"2.0.0\",\"gridData\":{\"x\":18,\"y\":0,\"w\":15,\"h\":19,\"i\":\"ff1d5bec-24fd-43b1-8a7b-e1b6942aa306\"},\"panelIndex\":\"ff1d5bec-24fd-43b1-8a7b-e1b6942aa306\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"2.0.0\",\"gridData\":{\"x\":33,\"y\":0,\"w\":15,\"h\":19,\"i\":\"6874104c-2a3e-432f-9cf1-3324dce134c9\"},\"panelIndex\":\"6874104c-2a3e-432f-9cf1-3324dce134c9\",\"embeddableConfig\":{},\"panelRefName\":\"panel_3\"},{\"version\":\"2.0.0\",\"gridData\":{\"x\":0,\"y\":19,\"w\":48,\"h\":19,\"i\":\"f8d075cc-f2fc-4da7-8c0b-c1e691386eaf\"},\"panelIndex\":\"f8d075cc-f2fc-4da7-8c0b-c1e691386eaf\",\"embeddableConfig\":{},\"panelRefName\":\"panel_4\"}]", - "optionsJSON": "{\"hidePanelTitles\":false,\"useMargins\":true}", - "version": 1, - "timeRestore": false, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}" - } - }, - "references": [ - { - "name": "panel_0", - "type": "visualization", - "id": "6ccdb970-e842-11ec-88e0-67bf497b7120" - }, - { - "name": "panel_1", - "type": "visualization", - "id": "a6549790-e842-11ec-88e0-67bf497b7120" - }, - { - "name": "panel_2", - "type": "visualization", - "id": "42bd0a50-e847-11ec-8b84-87a2d75f23b1" - }, - { - "name": "panel_3", - "type": "visualization", - "id": "8e4a7890-e847-11ec-8b84-87a2d75f23b1" - }, - { - "name": "panel_4", - "type": "search", - "id": "37139ac0-e842-11ec-88e0-67bf497b7120" - } - ], - "migrationVersion": { - "dashboard": "7.9.3" - } - }, - { - "id": "6ccdb970-e842-11ec-88e0-67bf497b7120", - "type": "visualization", - "namespaces": [ - "default" - ], - "updated_at": "2022-06-09T22:51:27.963Z", - "version": "WzkxMywxXQ==", - "attributes": { - "title": "Malcolm Sensor Syslog - Host", - "visState": "{\"title\":\"Malcolm Sensor Syslog - Host\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"log.syslog.hostname\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":50,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"unknown\",\"customLabel\":\"Host\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", - "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" - }, - "savedSearchRefName": "search_0" - }, - "references": [ - { - "name": "search_0", - "type": "search", - "id": "37139ac0-e842-11ec-88e0-67bf497b7120" - } - ], - "migrationVersion": { - "visualization": "7.10.0" - } - }, - { - "id": "a6549790-e842-11ec-88e0-67bf497b7120", - "type": "visualization", - "namespaces": [ - "default" - ], - "updated_at": "2022-06-09T22:51:27.963Z", - "version": "WzkxNCwxXQ==", - "attributes": { - "title": "Malcolm Sensor Syslog - Program", - "visState": "{\"title\":\"Malcolm Sensor Syslog - Program\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"log.syslog.appname\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":200,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Program\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", - "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" - }, - "savedSearchRefName": "search_0" - }, - "references": [ - { - "name": "search_0", - "type": "search", - "id": "37139ac0-e842-11ec-88e0-67bf497b7120" - } - ], - "migrationVersion": { - "visualization": "7.10.0" - } - }, - { - "id": "42bd0a50-e847-11ec-8b84-87a2d75f23b1", - "type": "visualization", - "namespaces": [ - "default" - ], - "updated_at": "2022-06-09T22:55:30.164Z", - "version": "WzkyMSwxXQ==", - "attributes": { - "title": "Malcolm Sensor Syslog - Severity", - "visState": "{\"title\":\"Malcolm Sensor Syslog - Severity\",\"type\":\"pie\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"log.syslog.severity.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":8,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"segment\"}],\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"isDonut\":true,\"labels\":{\"show\":true,\"values\":true,\"last_level\":true,\"truncate\":100}}}", - "uiStateJSON": "{\"vis\":{\"legendOpen\":true}}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" - }, - "savedSearchRefName": "search_0" - }, - "references": [ - { - "name": "search_0", - "type": "search", - "id": "37139ac0-e842-11ec-88e0-67bf497b7120" - } - ], - "migrationVersion": { - "visualization": "7.10.0" - } - }, - { - "id": "8e4a7890-e847-11ec-8b84-87a2d75f23b1", - "type": "visualization", - "namespaces": [ - "default" - ], - "updated_at": "2022-06-09T22:57:36.921Z", - "version": "WzkyMiwxXQ==", - "attributes": { - "title": "Malcolm Sensor Syslog - Facility", - "visState": "{\"title\":\"Malcolm Sensor Syslog - Facility\",\"type\":\"horizontal_bar\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"log.syslog.facility.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":25,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Facility\"},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"log.syslog.facility.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":25,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Facility\"},\"schema\":\"group\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":200},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":75,\"filter\":true,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}", - "uiStateJSON": "{\"vis\":{\"legendOpen\":false}}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" - }, - "savedSearchRefName": "search_0" - }, - "references": [ - { - "name": "search_0", - "type": "search", - "id": "37139ac0-e842-11ec-88e0-67bf497b7120" - } - ], - "migrationVersion": { - "visualization": "7.10.0" - } - }, - { - "id": "37139ac0-e842-11ec-88e0-67bf497b7120", - "type": "search", - "namespaces": [ - "default" - ], - "updated_at": "2022-06-09T22:51:27.963Z", - "version": "WzkxNSwxXQ==", - "attributes": { - "title": "Malcolm Sensor Syslog - Logs", - "description": "", - "hits": 0, - "columns": [ - "log.syslog.hostname", - "log.syslog.severity.name", - "log.syslog.facility.name", - "log.syslog.appname", - "event.original" - ], - "sort": [], - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"log.syslog:*\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" - } - }, - "references": [ - { - "name": "kibanaSavedObjectMeta.searchSourceJSON.index", - "type": "index-pattern", - "id": "malcolm_beats_*" - } - ], - "migrationVersion": { - "search": "7.9.3" - } - } - ] -} \ No newline at end of file diff --git a/dashboards/dashboards/beats/f6600310-9943-11ee-a029-e973f4774355.json b/dashboards/dashboards/beats/f6600310-9943-11ee-a029-e973f4774355.json new file mode 100644 index 000000000..6a4e2047e --- /dev/null +++ b/dashboards/dashboards/beats/f6600310-9943-11ee-a029-e973f4774355.json @@ -0,0 +1,251 @@ +{ + "version": "2.11.1", + "objects": [ + { + "id": "f6600310-9943-11ee-a029-e973f4774355", + "type": "dashboard", + "namespaces": [ + "default" + ], + "updated_at": "2023-12-14T22:33:38.334Z", + "version": "WzkzOCwxXQ==", + "attributes": { + "title": "Malcolm Sensor Journald Logs", + "hits": 0, + "description": "", + "panelsJSON": "[{\"embeddableConfig\":{\"vis\":{\"legendOpen\":true}},\"gridData\":{\"h\":19,\"i\":\"b514b4e8-689b-465e-8335-ca20c20d46fe\",\"w\":14,\"x\":0,\"y\":0},\"panelIndex\":\"b514b4e8-689b-465e-8335-ca20c20d46fe\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_0\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":19,\"i\":\"7a7cfec2-8688-45a7-9790-66b3f0e9fd7e\",\"w\":11,\"x\":14,\"y\":0},\"panelIndex\":\"7a7cfec2-8688-45a7-9790-66b3f0e9fd7e\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_1\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":19,\"i\":\"2143906d-7adb-4de7-8484-2f87c8c98332\",\"w\":23,\"x\":25,\"y\":0},\"panelIndex\":\"2143906d-7adb-4de7-8484-2f87c8c98332\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_2\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":18,\"i\":\"bb4d56fd-b110-4d58-b6aa-e4189bdba918\",\"w\":24,\"x\":0,\"y\":19},\"panelIndex\":\"bb4d56fd-b110-4d58-b6aa-e4189bdba918\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_3\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":18,\"i\":\"4e4780cd-add6-4dbe-95ac-d11afbcd630c\",\"w\":24,\"x\":24,\"y\":19},\"panelIndex\":\"4e4780cd-add6-4dbe-95ac-d11afbcd630c\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_4\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":32,\"i\":\"77bbae8a-66cd-4e30-9b90-c7ab7c271320\",\"w\":48,\"x\":0,\"y\":37},\"panelIndex\":\"77bbae8a-66cd-4e30-9b90-c7ab7c271320\",\"version\":\"2.11.1\",\"panelRefName\":\"panel_5\"}]", + "optionsJSON": "{\"hidePanelTitles\":false,\"useMargins\":true}", + "version": 1, + "timeRestore": false, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}" + } + }, + "references": [ + { + "name": "panel_0", + "type": "visualization", + "id": "82e154f0-99e5-11ee-a12e-a134fdba98ea" + }, + { + "name": "panel_1", + "type": "visualization", + "id": "5a3cb5c0-99e6-11ee-a12e-a134fdba98ea" + }, + { + "name": "panel_2", + "type": "visualization", + "id": "02c189b0-99e6-11ee-a12e-a134fdba98ea" + }, + { + "name": "panel_3", + "type": "visualization", + "id": "81e6b660-99e7-11ee-a12e-a134fdba98ea" + }, + { + "name": "panel_4", + "type": "visualization", + "id": "9601eb20-99e6-11ee-a12e-a134fdba98ea" + }, + { + "name": "panel_5", + "type": "search", + "id": "c79c1e60-9943-11ee-a029-e973f4774355" + } + ], + "migrationVersion": { + "dashboard": "7.9.3" + } + }, + { + "id": "82e154f0-99e5-11ee-a12e-a134fdba98ea", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2023-12-14T22:33:38.334Z", + "version": "WzkzOSwxXQ==", + "attributes": { + "title": "Malcolm Sensor Journald - Logger", + "visState": "{\"title\":\"Malcolm Sensor Journald - Logger\",\"type\":\"pie\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"log.logger\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"segment\"}],\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"isDonut\":true,\"labels\":{\"show\":true,\"values\":true,\"last_level\":true,\"truncate\":100}}}", + "uiStateJSON": "{\"vis\":{\"legendOpen\":false}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "name": "search_0", + "type": "search", + "id": "c79c1e60-9943-11ee-a029-e973f4774355" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "5a3cb5c0-99e6-11ee-a12e-a134fdba98ea", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2023-12-14T22:50:36.981Z", + "version": "Wzk1NSwxXQ==", + "attributes": { + "title": "Malcolm Sensor Journald - Process UID", + "visState": "{\"title\":\"Malcolm Sensor Journald - Process UID\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.systemd.hostname\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Systemd Host\"},\"schema\":\"bucket\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"process.user.id\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Process UID\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", + "uiStateJSON": "{\"vis\":{\"sortColumn\":{\"colIndex\":1,\"direction\":\"desc\"}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "name": "search_0", + "type": "search", + "id": "c79c1e60-9943-11ee-a029-e973f4774355" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "02c189b0-99e6-11ee-a12e-a134fdba98ea", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2023-12-14T22:33:38.334Z", + "version": "Wzk0MSwxXQ==", + "attributes": { + "title": "Malcolm Sensor Journald - Logs by Host Over Time", + "visState": "{\"title\":\"Malcolm Sensor Journald - Logs by Host Over Time\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"now-15m\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.systemd.hostname\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Unknown\",\"customLabel\":\"Journald Host\"},\"schema\":\"group\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"square root\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"times\":[],\"addTimeMarker\":true,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "name": "search_0", + "type": "search", + "id": "c79c1e60-9943-11ee-a029-e973f4774355" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "81e6b660-99e7-11ee-a12e-a134fdba98ea", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2023-12-14T22:33:38.334Z", + "version": "Wzk0MiwxXQ==", + "attributes": { + "title": "Malcolm Sensor Journald - Systemd Unit", + "visState": "{\"title\":\"Malcolm Sensor Journald - Systemd Unit\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.systemd.systemd_unit\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Systemd Unit\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.systemd.systemd_user_unit\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"Systemd User Unit\"},\"schema\":\"bucket\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"miscbeat.systemd.user_unit\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"-\",\"customLabel\":\"User Unit\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\"}}", + "uiStateJSON": "{\"vis\":{\"sortColumn\":{\"colIndex\":3,\"direction\":\"desc\"}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "name": "search_0", + "type": "search", + "id": "c79c1e60-9943-11ee-a029-e973f4774355" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "9601eb20-99e6-11ee-a12e-a134fdba98ea", + "type": "visualization", + "namespaces": [ + "default" + ], + "updated_at": "2023-12-14T22:33:38.334Z", + "version": "Wzk0MywxXQ==", + "attributes": { + "title": "Malcolm Sensor Journald - Process Name", + "visState": "{\"title\":\"Malcolm Sensor Journald - Process Name\",\"type\":\"tagcloud\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"process.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":50,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"segment\"}],\"params\":{\"scale\":\"square root\",\"orientation\":\"single\",\"minFontSize\":18,\"maxFontSize\":72,\"showLabel\":false}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}" + }, + "savedSearchRefName": "search_0" + }, + "references": [ + { + "name": "search_0", + "type": "search", + "id": "c79c1e60-9943-11ee-a029-e973f4774355" + } + ], + "migrationVersion": { + "visualization": "7.10.0" + } + }, + { + "id": "c79c1e60-9943-11ee-a029-e973f4774355", + "type": "search", + "namespaces": [ + "default" + ], + "updated_at": "2023-12-14T22:33:38.334Z", + "version": "Wzk0NCwxXQ==", + "attributes": { + "title": "Malcolm Sensor Journald - Logs", + "description": "", + "hits": 0, + "columns": [ + "@timestamp", + "miscbeat.systemd.hostname", + "process.name", + "process.pid", + "process.user.id", + "event.original", + "miscbeat.systemd.systemd_unit", + "miscbeat.systemd.systemd_user_unit", + "log.logger" + ], + "sort": [], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"query\":{\"query\":\"miscbeat.systemd:*\",\"language\":\"kuery\"},\"highlightAll\":true,\"version\":true,\"aggs\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"calendar_interval\":\"1w\",\"time_zone\":\"America/Denver\",\"min_doc_count\":1}}},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + } + }, + "references": [ + { + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern", + "id": "malcolm_beats_*" + } + ], + "migrationVersion": { + "search": "7.9.3" + } + } + ] +} \ No newline at end of file diff --git a/dashboards/scripts/create-arkime-sessions-index.sh b/dashboards/scripts/create-arkime-sessions-index.sh index c839d0fd9..a118dd46c 100755 --- a/dashboards/scripts/create-arkime-sessions-index.sh +++ b/dashboards/scripts/create-arkime-sessions-index.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. set -euo pipefail shopt -s nocasematch diff --git a/dashboards/supervisord.conf b/dashboards/supervisord.conf index 81e64278a..b6cee6932 100644 --- a/dashboards/supervisord.conf +++ b/dashboards/supervisord.conf @@ -1,4 +1,4 @@ -; Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +; Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. [unix_http_server] file=/tmp/supervisor.sock ; (the path to the socket file) diff --git a/dashboards/templates/composable/component/miscbeat.json b/dashboards/templates/composable/component/miscbeat.json index 62f272927..d3072b3c4 100644 --- a/dashboards/templates/composable/component/miscbeat.json +++ b/dashboards/templates/composable/component/miscbeat.json @@ -93,6 +93,58 @@ "properties": { } }, + "systemd": { + "properties": { + "audit_loginuid": { "type": "integer" }, + "audit_session": { "type": "integer" }, + "boot_id": { "type": "keyword" }, + "cpu_usage_nsec": { "type": "long" }, + "device": { "type": "keyword" }, + "glib_domain": { "type": "keyword" }, + "glib_old_log_api": { "type": "keyword" }, + "hostname": { "type": "keyword" }, + "invocation_id": { "type": "keyword" }, + "job_id": { "type": "integer" }, + "job_result": { "type": "keyword" }, + "job_type": { "type": "keyword" }, + "kernel_device": { "type": "keyword" }, + "kernel_subsystem": { "type": "keyword" }, + "leader": { "type": "keyword" }, + "machine_id": { "type": "keyword" }, + "message": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, + "message_id": { "type": "keyword" }, + "nm_device": { "type": "keyword" }, + "nm_log_domains": { "type": "keyword" }, + "nm_log_level": { "type": "keyword" }, + "priority": { "type": "integer" }, + "runtime_scope": { "type": "keyword" }, + "selinux_context": { "type": "keyword" }, + "session_id": { "type": "keyword" }, + "source_monotonic_timestamp": { "type": "long" }, + "source_realtime_timestamp" : { "type": "date" }, + "stream_id": { "type": "keyword" }, + "syslog_raw": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, + "syslog_timestamp": { "type": "keyword" }, + "systemd_cgroup": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text" } } }, + "systemd_invocation_id": { "type": "keyword" }, + "systemd_session": { "type": "keyword" }, + "systemd_slice": { "type": "keyword" }, + "systemd_unit": { "type": "keyword" }, + "systemd_user_slice": { "type": "keyword" }, + "systemd_user_unit": { "type": "keyword" }, + "timestamp_boottime": { "type": "float" }, + "timestamp_monotonic": { "type": "float" }, + "udev_devnode": { "type": "keyword" }, + "udev_sysname": { "type": "keyword" }, + "unit": { "type": "keyword" }, + "user_id": { "type": "keyword" }, + "user_invocation_id": { "type": "keyword" }, + "user_unit": { "type": "keyword" }, + "userspace_usec": { "type": "integer" }, + "wp_object": { "type": "keyword" }, + "wp_object_type": { "type": "keyword" } + } + }, "thermal": { "properties": { "name": { "type": "keyword"}, diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml index 1963a0f74..697718674 100644 --- a/docker-compose-standalone.yml +++ b/docker-compose-standalone.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. version: '3.7' @@ -12,7 +12,7 @@ x-logging: services: opensearch: - image: ghcr.io/idaholab/malcolm/opensearch:23.12.0 + image: ghcr.io/idaholab/malcolm/opensearch:23.12.1 # Technically the "hedgehog" profile doesn't have OpenSearch, but in that case # OPENSEARCH_PRIMARY will be set to remote, which means the container will # start but not actually run OpenSearch. It's included in both profiles to @@ -36,7 +36,10 @@ services: soft: -1 hard: -1 cap_add: + # IPC_LOCK - to lock memory, preventing swapping - IPC_LOCK + # SYS_RESOURCE - for increasing memlock limits + - SYS_RESOURCE volumes: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro @@ -51,7 +54,7 @@ services: retries: 3 start_period: 180s dashboards-helper: - image: ghcr.io/idaholab/malcolm/dashboards-helper:23.12.0 + image: ghcr.io/idaholab/malcolm/dashboards-helper:23.12.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -81,7 +84,7 @@ services: retries: 3 start_period: 30s dashboards: - image: ghcr.io/idaholab/malcolm/dashboards:23.12.0 + image: ghcr.io/idaholab/malcolm/dashboards:23.12.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -109,7 +112,7 @@ services: retries: 3 start_period: 210s logstash: - image: ghcr.io/idaholab/malcolm/logstash-oss:23.12.0 + image: ghcr.io/idaholab/malcolm/logstash-oss:23.12.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -123,7 +126,10 @@ services: soft: -1 hard: -1 cap_add: + # IPC_LOCK - to lock memory, preventing swapping - IPC_LOCK + # SYS_RESOURCE - for increasing memlock limits + - SYS_RESOURCE env_file: - ./config/process.env - ./config/ssl.env @@ -153,7 +159,7 @@ services: retries: 3 start_period: 600s filebeat: - image: ghcr.io/idaholab/malcolm/filebeat-oss:23.12.0 + image: ghcr.io/idaholab/malcolm/filebeat-oss:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -188,7 +194,7 @@ services: retries: 3 start_period: 60s arkime: - image: ghcr.io/idaholab/malcolm/arkime:23.12.0 + image: ghcr.io/idaholab/malcolm/arkime:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -204,48 +210,66 @@ services: - ./config/upload-common.env - ./config/auth.env - ./config/arkime.env + - ./config/arkime-offline.env - ./config/arkime-secret.env environment: VIRTUAL_HOST : 'arkime.malcolm.local' - ulimits: - memlock: - soft: -1 - hard: -1 depends_on: - opensearch - ports: - - "127.0.0.1:8005:8005" volumes: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro + - ./arkime/rules:/opt/arkime/rules:ro - ./pcap:/data/pcap - - ./arkime-logs:/opt/arkime/logs - - ./arkime-raw:/opt/arkime/raw healthcheck: test: ["CMD", "curl", "--insecure", "--silent", "--fail", "https://localhost:8005/_ns_/nstest.html"] interval: 90s timeout: 30s retries: 3 start_period: 210s - zeek: - image: ghcr.io/idaholab/malcolm/zeek:23.12.0 + arkime-live: + image: ghcr.io/idaholab/malcolm/arkime:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" stdin_open: false tty: true - hostname: zeek - networks: - - default - ulimits: - memlock: - soft: -1 - hard: -1 + network_mode: host cap_add: + # IPC_LOCK - to lock memory, preventing swapping - IPC_LOCK + # SYS_RESOURCE - for increasing memlock limits + - SYS_RESOURCE + # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW - - SYS_ADMIN + # SYS_NICE - to set process nice values, real-time scheduling policies, I/O scheduling + - SYS_NICE + env_file: + - ./config/process.env + - ./config/ssl.env + - ./config/opensearch.env + - ./config/upload-common.env + - ./config/pcap-capture.env + - ./config/auth.env + - ./config/arkime.env + - ./config/arkime-live.env + - ./config/arkime-secret.env + volumes: + - ./nginx/ca-trust:/var/local/ca-trust:ro + - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro + - ./arkime/rules:/opt/arkime/rules:ro + - ./pcap:/data/pcap + zeek: + image: ghcr.io/idaholab/malcolm/zeek:23.12.1 + profiles: ["malcolm", "hedgehog"] + logging: *default-logging + restart: "no" + stdin_open: false + tty: true + hostname: zeek + networks: + - default env_file: - ./config/process.env - ./config/ssl.env @@ -269,22 +293,19 @@ services: retries: 3 start_period: 60s zeek-live: - image: ghcr.io/idaholab/malcolm/zeek:23.12.0 + image: ghcr.io/idaholab/malcolm/zeek:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" stdin_open: false tty: true network_mode: host - ulimits: - memlock: - soft: -1 - hard: -1 cap_add: - - IPC_LOCK + # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW - - SYS_ADMIN + # SYS_NICE - to set process nice values, real-time scheduling policies, I/O scheduling + - SYS_NICE env_file: - ./config/process.env - ./config/ssl.env @@ -300,7 +321,7 @@ services: - ./zeek/intel:/opt/zeek/share/zeek/site/intel - ./zeek/custom:/opt/zeek/share/zeek/site/custom:ro suricata: - image: ghcr.io/idaholab/malcolm/suricata:23.12.0 + image: ghcr.io/idaholab/malcolm/suricata:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -309,15 +330,6 @@ services: hostname: suricata networks: - default - ulimits: - memlock: - soft: -1 - hard: -1 - cap_add: - - IPC_LOCK - - NET_ADMIN - - NET_RAW - - SYS_ADMIN env_file: - ./config/process.env - ./config/ssl.env @@ -337,22 +349,23 @@ services: retries: 3 start_period: 120s suricata-live: - image: ghcr.io/idaholab/malcolm/suricata:23.12.0 + image: ghcr.io/idaholab/malcolm/suricata:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" stdin_open: false tty: true network_mode: host - ulimits: - memlock: - soft: -1 - hard: -1 cap_add: + # IPC_LOCK - to lock memory, preventing swapping - IPC_LOCK + # SYS_RESOURCE - for increasing memlock limits + - SYS_RESOURCE + # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW - - SYS_ADMIN + # SYS_NICE - to set process nice values and scheduling policies for capture + - SYS_NICE env_file: - ./config/process.env - ./config/ssl.env @@ -366,7 +379,7 @@ services: - ./suricata/rules:/opt/suricata/rules:ro - ./suricata/include-configs:/opt/suricata/include-configs:ro file-monitor: - image: ghcr.io/idaholab/malcolm/file-monitor:23.12.0 + image: ghcr.io/idaholab/malcolm/file-monitor:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -394,7 +407,7 @@ services: retries: 3 start_period: 60s pcap-capture: - image: ghcr.io/idaholab/malcolm/pcap-capture:23.12.0 + image: ghcr.io/idaholab/malcolm/pcap-capture:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -406,10 +419,15 @@ services: soft: -1 hard: -1 cap_add: + # IPC_LOCK - to lock memory, preventing swapping - IPC_LOCK + # SYS_RESOURCE - for increasing memlock limits + - SYS_RESOURCE + # SYS_ADMIN - for netsniff-ng to set the disc I/O scheduler policy + - SYS_ADMIN + # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW - - SYS_ADMIN env_file: - ./config/process.env - ./config/ssl.env @@ -418,7 +436,7 @@ services: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./pcap/upload:/pcap pcap-monitor: - image: ghcr.io/idaholab/malcolm/pcap-monitor:23.12.0 + image: ghcr.io/idaholab/malcolm/pcap-monitor:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -446,7 +464,7 @@ services: retries: 3 start_period: 90s upload: - image: ghcr.io/idaholab/malcolm/file-upload:23.12.0 + image: ghcr.io/idaholab/malcolm/file-upload:23.12.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -473,7 +491,7 @@ services: retries: 3 start_period: 60s htadmin: - image: ghcr.io/idaholab/malcolm/htadmin:23.12.0 + image: ghcr.io/idaholab/malcolm/htadmin:23.12.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -500,7 +518,7 @@ services: retries: 3 start_period: 60s freq: - image: ghcr.io/idaholab/malcolm/freq:23.12.0 + image: ghcr.io/idaholab/malcolm/freq:23.12.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -524,7 +542,7 @@ services: retries: 3 start_period: 60s netbox: - image: ghcr.io/idaholab/malcolm/netbox:23.12.0 + image: ghcr.io/idaholab/malcolm/netbox:23.12.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -558,7 +576,7 @@ services: retries: 3 start_period: 120s netbox-postgres: - image: ghcr.io/idaholab/malcolm/postgresql:23.12.0 + image: ghcr.io/idaholab/malcolm/postgresql:23.12.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -584,7 +602,7 @@ services: retries: 3 start_period: 45s netbox-redis: - image: ghcr.io/idaholab/malcolm/redis:23.12.0 + image: ghcr.io/idaholab/malcolm/redis:23.12.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -614,7 +632,7 @@ services: retries: 3 start_period: 45s netbox-redis-cache: - image: ghcr.io/idaholab/malcolm/redis:23.12.0 + image: ghcr.io/idaholab/malcolm/redis:23.12.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -643,7 +661,7 @@ services: retries: 3 start_period: 45s api: - image: ghcr.io/idaholab/malcolm/api:23.12.0 + image: ghcr.io/idaholab/malcolm/api:23.12.1 profiles: ["malcolm"] logging: *default-logging command: gunicorn --bind 0:5000 manage:app @@ -670,7 +688,7 @@ services: retries: 3 start_period: 60s nginx-proxy: - image: ghcr.io/idaholab/malcolm/nginx-proxy:23.12.0 + image: ghcr.io/idaholab/malcolm/nginx-proxy:23.12.1 profiles: ["malcolm"] logging: *default-logging restart: "no" diff --git a/docker-compose.yml b/docker-compose.yml index 85958b908..3bc8b9014 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. version: '3.7' @@ -15,7 +15,7 @@ services: build: context: . dockerfile: Dockerfiles/opensearch.Dockerfile - image: ghcr.io/idaholab/malcolm/opensearch:23.12.0 + image: ghcr.io/idaholab/malcolm/opensearch:23.12.1 # Technically the "hedgehog" profile doesn't have OpenSearch, but in that case # OPENSEARCH_PRIMARY will be set to remote, which means the container will # start but not actually run OpenSearch. It's included in both profiles to @@ -39,7 +39,10 @@ services: soft: -1 hard: -1 cap_add: + # IPC_LOCK - to lock memory, preventing swapping - IPC_LOCK + # SYS_RESOURCE - for increasing memlock limits + - SYS_RESOURCE volumes: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro @@ -57,7 +60,7 @@ services: build: context: . dockerfile: Dockerfiles/dashboards-helper.Dockerfile - image: ghcr.io/idaholab/malcolm/dashboards-helper:23.12.0 + image: ghcr.io/idaholab/malcolm/dashboards-helper:23.12.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -90,7 +93,7 @@ services: build: context: . dockerfile: Dockerfiles/dashboards.Dockerfile - image: ghcr.io/idaholab/malcolm/dashboards:23.12.0 + image: ghcr.io/idaholab/malcolm/dashboards:23.12.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -121,7 +124,7 @@ services: build: context: . dockerfile: Dockerfiles/logstash.Dockerfile - image: ghcr.io/idaholab/malcolm/logstash-oss:23.12.0 + image: ghcr.io/idaholab/malcolm/logstash-oss:23.12.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -135,7 +138,10 @@ services: soft: -1 hard: -1 cap_add: + # IPC_LOCK - to lock memory, preventing swapping - IPC_LOCK + # SYS_RESOURCE - for increasing memlock limits + - SYS_RESOURCE env_file: - ./config/process.env - ./config/ssl.env @@ -172,7 +178,7 @@ services: build: context: . dockerfile: Dockerfiles/filebeat.Dockerfile - image: ghcr.io/idaholab/malcolm/filebeat-oss:23.12.0 + image: ghcr.io/idaholab/malcolm/filebeat-oss:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -210,7 +216,7 @@ services: build: context: . dockerfile: Dockerfiles/arkime.Dockerfile - image: ghcr.io/idaholab/malcolm/arkime:23.12.0 + image: ghcr.io/idaholab/malcolm/arkime:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -229,20 +235,13 @@ services: - ./config/arkime-secret.env environment: VIRTUAL_HOST : 'arkime.malcolm.local' - ulimits: - memlock: - soft: -1 - hard: -1 depends_on: - opensearch - ports: - - "127.0.0.1:8005:8005" volumes: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro - ./pcap:/data/pcap - - ./arkime-logs:/opt/arkime/logs - - ./arkime-raw:/opt/arkime/raw + - ./arkime/rules:/opt/arkime/rules:ro - ./arkime/etc/config.ini:/opt/arkime/etc/config.orig.ini:ro - ./arkime/etc/user_settings.json:/opt/arkime/etc/user_settings.json:ro - ./arkime/wise/source.zeeklogs.js:/opt/arkime/wiseService/source.zeeklogs.js:ro @@ -252,11 +251,50 @@ services: timeout: 30s retries: 3 start_period: 210s + arkime-live: + build: + context: . + dockerfile: Dockerfiles/arkime.Dockerfile + image: ghcr.io/idaholab/malcolm/arkime:23.12.1 + profiles: ["malcolm", "hedgehog"] + logging: *default-logging + restart: "no" + stdin_open: false + tty: true + network_mode: host + cap_add: + # IPC_LOCK - to lock memory, preventing swapping + - IPC_LOCK + # SYS_RESOURCE - for increasing memlock limits + - SYS_RESOURCE + # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets + - NET_ADMIN + - NET_RAW + # SYS_NICE - to set process nice values, real-time scheduling policies, I/O scheduling + - SYS_NICE + env_file: + - ./config/process.env + - ./config/ssl.env + - ./config/opensearch.env + - ./config/upload-common.env + - ./config/pcap-capture.env + - ./config/auth.env + - ./config/arkime.env + - ./config/arkime-live.env + - ./config/arkime-secret.env + volumes: + - ./nginx/ca-trust:/var/local/ca-trust:ro + - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro + - ./pcap:/data/pcap + - ./arkime/rules:/opt/arkime/rules:ro + - ./arkime/etc/config.ini:/opt/arkime/etc/config.orig.ini:ro + - ./arkime/etc/user_settings.json:/opt/arkime/etc/user_settings.json:ro + - ./arkime/wise/source.zeeklogs.js:/opt/arkime/wiseService/source.zeeklogs.js:ro zeek: build: context: . dockerfile: Dockerfiles/zeek.Dockerfile - image: ghcr.io/idaholab/malcolm/zeek:23.12.0 + image: ghcr.io/idaholab/malcolm/zeek:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -265,15 +303,6 @@ services: hostname: zeek networks: - default - ulimits: - memlock: - soft: -1 - hard: -1 - cap_add: - - IPC_LOCK - - NET_ADMIN - - NET_RAW - - SYS_ADMIN env_file: - ./config/process.env - ./config/ssl.env @@ -301,22 +330,19 @@ services: build: context: . dockerfile: Dockerfiles/zeek.Dockerfile - image: ghcr.io/idaholab/malcolm/zeek:23.12.0 + image: ghcr.io/idaholab/malcolm/zeek:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" stdin_open: false tty: true network_mode: host - ulimits: - memlock: - soft: -1 - hard: -1 cap_add: - - IPC_LOCK + # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW - - SYS_ADMIN + # SYS_NICE - to set process nice values, real-timescheduling policies, I/O scheduling + - SYS_NICE env_file: - ./config/process.env - ./config/ssl.env @@ -336,7 +362,7 @@ services: build: context: . dockerfile: Dockerfiles/suricata.Dockerfile - image: ghcr.io/idaholab/malcolm/suricata:23.12.0 + image: ghcr.io/idaholab/malcolm/suricata:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -345,15 +371,6 @@ services: hostname: suricata networks: - default - ulimits: - memlock: - soft: -1 - hard: -1 - cap_add: - - IPC_LOCK - - NET_ADMIN - - NET_RAW - - SYS_ADMIN env_file: - ./config/process.env - ./config/ssl.env @@ -376,22 +393,23 @@ services: build: context: . dockerfile: Dockerfiles/suricata.Dockerfile - image: ghcr.io/idaholab/malcolm/suricata:23.12.0 + image: ghcr.io/idaholab/malcolm/suricata:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" stdin_open: false tty: true network_mode: host - ulimits: - memlock: - soft: -1 - hard: -1 cap_add: + # IPC_LOCK - to lock memory, preventing swapping - IPC_LOCK + # SYS_RESOURCE - for increasing memlock limits + - SYS_RESOURCE + # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW - - SYS_ADMIN + # SYS_NICE - to set process nice values, real-timescheduling policies, I/O scheduling + - SYS_NICE env_file: - ./config/process.env - ./config/ssl.env @@ -408,7 +426,7 @@ services: build: context: . dockerfile: Dockerfiles/file-monitor.Dockerfile - image: ghcr.io/idaholab/malcolm/file-monitor:23.12.0 + image: ghcr.io/idaholab/malcolm/file-monitor:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -439,7 +457,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-capture.Dockerfile - image: ghcr.io/idaholab/malcolm/pcap-capture:23.12.0 + image: ghcr.io/idaholab/malcolm/pcap-capture:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -451,10 +469,15 @@ services: soft: -1 hard: -1 cap_add: + # IPC_LOCK - to lock memory, preventing swapping - IPC_LOCK + # SYS_RESOURCE - for increasing memlock limits + - SYS_RESOURCE + # SYS_ADMIN - for netsniff-ng to set the disc I/O scheduler policy + - SYS_ADMIN + # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW - - SYS_ADMIN env_file: - ./config/process.env - ./config/ssl.env @@ -466,7 +489,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-monitor.Dockerfile - image: ghcr.io/idaholab/malcolm/pcap-monitor:23.12.0 + image: ghcr.io/idaholab/malcolm/pcap-monitor:23.12.1 profiles: ["malcolm", "hedgehog"] logging: *default-logging restart: "no" @@ -497,7 +520,7 @@ services: build: context: . dockerfile: Dockerfiles/file-upload.Dockerfile - image: ghcr.io/idaholab/malcolm/file-upload:23.12.0 + image: ghcr.io/idaholab/malcolm/file-upload:23.12.1 profiles: ["malcolm"] logging: *default-logging restart: "no" @@ -524,7 +547,7 @@ services: retries: 3 start_period: 60s htadmin: - image: ghcr.io/idaholab/malcolm/htadmin:23.12.0 + image: ghcr.io/idaholab/malcolm/htadmin:23.12.1 profiles: ["malcolm"] logging: *default-logging build: @@ -554,7 +577,7 @@ services: retries: 3 start_period: 60s freq: - image: ghcr.io/idaholab/malcolm/freq:23.12.0 + image: ghcr.io/idaholab/malcolm/freq:23.12.1 profiles: ["malcolm"] logging: *default-logging build: @@ -581,7 +604,7 @@ services: retries: 3 start_period: 60s netbox: - image: ghcr.io/idaholab/malcolm/netbox:23.12.0 + image: ghcr.io/idaholab/malcolm/netbox:23.12.1 profiles: ["malcolm"] logging: *default-logging build: @@ -618,7 +641,7 @@ services: retries: 3 start_period: 120s netbox-postgres: - image: ghcr.io/idaholab/malcolm/postgresql:23.12.0 + image: ghcr.io/idaholab/malcolm/postgresql:23.12.1 profiles: ["malcolm"] logging: *default-logging build: @@ -647,7 +670,7 @@ services: retries: 3 start_period: 45s netbox-redis: - image: ghcr.io/idaholab/malcolm/redis:23.12.0 + image: ghcr.io/idaholab/malcolm/redis:23.12.1 profiles: ["malcolm"] logging: *default-logging build: @@ -680,7 +703,7 @@ services: retries: 3 start_period: 45s netbox-redis-cache: - image: ghcr.io/idaholab/malcolm/redis:23.12.0 + image: ghcr.io/idaholab/malcolm/redis:23.12.1 profiles: ["malcolm"] logging: *default-logging build: @@ -712,7 +735,7 @@ services: retries: 3 start_period: 45s api: - image: ghcr.io/idaholab/malcolm/api:23.12.0 + image: ghcr.io/idaholab/malcolm/api:23.12.1 profiles: ["malcolm"] logging: *default-logging build: @@ -745,7 +768,7 @@ services: build: context: . dockerfile: Dockerfiles/nginx.Dockerfile - image: ghcr.io/idaholab/malcolm/nginx-proxy:23.12.0 + image: ghcr.io/idaholab/malcolm/nginx-proxy:23.12.1 profiles: ["malcolm"] logging: *default-logging restart: "no" diff --git a/docs/README.md b/docs/README.md index a1f428dc0..560f87caa 100644 --- a/docs/README.md +++ b/docs/README.md @@ -50,6 +50,7 @@ Malcolm can also easily be deployed locally on an ordinary consumer workstation * [Live analysis](live-analysis.md#LiveAnalysis) - [Using a network sensor appliance](live-analysis.md#Hedgehog) - [Monitoring local network interfaces](live-analysis.md#LocalPCAP) + + ["Hedgehog" run profile](live-analysis.md#Profiles) - [Manually forwarding logs from an external source](live-analysis.md#ExternalForward) * [Arkime](arkime.md#Arkime) - [Zeek log integration](arkime.md#ArkimeZeek) diff --git a/docs/contributing-local-modifications.md b/docs/contributing-local-modifications.md index 53e32eddf..8f2491351 100644 --- a/docs/contributing-local-modifications.md +++ b/docs/contributing-local-modifications.md @@ -42,8 +42,6 @@ arkime: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./.opensearch.primary.curlrc:/var/local/curlrc/.opensearch.primary.curlrc:ro - ./pcap:/data/pcap - - ./arkime-logs:/opt/arkime/logs - - ./arkime-raw:/opt/arkime/raw zeek: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./pcap:/pcap diff --git a/docs/contributing-pcap.md b/docs/contributing-pcap.md index d8f0edd61..f9ed21c54 100644 --- a/docs/contributing-pcap.md +++ b/docs/contributing-pcap.md @@ -1,6 +1,6 @@ # PCAP processors -When a PCAP is uploaded (either through Malcolm's [upload web interface](upload.md#Upload) or just copied manually into the `./pcap/upload` directory), the `pcap-monitor` container has a script that picks up those PCAP files and publishes to a [ZeroMQ](https://zeromq.org/) topic that can be subscribed to by any other process that wants to analyze that PCAP. In Malcolm (at the time of the [v23.12.0 release]({{ site.github.repository_url }}/releases/tag/v23.12.0)), there are three such ZeroMQ topics: the `zeek`, `suricata` and `arkime` containers. These actually share the [same script]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/pcap_processor.py) to run the PCAP through Zeek, Suricata, and Arkime, respectively. For an example to follow, the `zeek` container is the less complicated of the two. To integrate a new PCAP processing tool into Malcolm (named `cooltool` for this example) the process would entail: +When a PCAP is uploaded (either through Malcolm's [upload web interface](upload.md#Upload) or just copied manually into the `./pcap/upload` directory), the `pcap-monitor` container has a script that picks up those PCAP files and publishes to a [ZeroMQ](https://zeromq.org/) topic that can be subscribed to by any other process that wants to analyze that PCAP. In Malcolm (at the time of the [v23.12.1 release]({{ site.github.repository_url }}/releases/tag/v23.12.1)), there are three such ZeroMQ topics: the `zeek`, `suricata` and `arkime` containers. These actually share the [same script]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/pcap_processor.py) to run the PCAP through Zeek, Suricata, and Arkime, respectively. For an example to follow, the `zeek` container is the less complicated of the two. To integrate a new PCAP processing tool into Malcolm (named `cooltool` for this example) the process would entail: 1. Define the service as instructed in the [Adding a new service](contributing-new-image.md#NewImage) section * Note how the existing `zeek` and `arkime` services use [bind mounts](contributing-local-modifications.md#Bind) to access the local `./pcap` directory diff --git a/docs/development.md b/docs/development.md index 8207e3582..c4a5a05fa 100644 --- a/docs/development.md +++ b/docs/development.md @@ -8,7 +8,6 @@ Checking out the [Malcolm source code]({{ site.github.repository_url }}/tree/{{ * `api` - code and configuration for the `api` container, which provides a REST API to query Malcolm * `arkime` - code and configuration for the `arkime` container that processes PCAP files using `capture`, which serves the Viewer application -* `arkime-logs` - an initially empty directory to which the `arkime` container will write some debug log files * `config` - a directory containing the environment variable files that define Malcolm's configuration * `dashboards` - code and configuration for the `dashboards` container for creating additional ad-hoc visualizations and dashboards beyond that which is provided by Arkime Viewer * `Dockerfiles` - a directory containing build instructions for Malcolm's docker images diff --git a/docs/download.md b/docs/download.md index 1aca19bca..588fc90ef 100644 --- a/docs/download.md +++ b/docs/download.md @@ -16,7 +16,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | ISO | SHA256 | |---|---| -| [malcolm-23.12.0.iso](/iso/malcolm-23.12.0.iso) (5.1GiB) | [`3e836d09cd79a4e3f54c6fc365b032385312ad885b8483a0df156b59175d4909`](/iso/malcolm-23.12.0.iso.sha256.txt) | +| [malcolm-23.12.1.iso](/iso/malcolm-23.12.1.iso) (5.2GiB) | [`a3dd7dbcaa78322f1ae62b93efd4d95e3644a1b52b65ba24dd1bccf4ac6b173a`](/iso/malcolm-23.12.1.iso.sha256.txt) | ## Hedgehog Linux @@ -26,7 +26,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | ISO | SHA256 | |---|---| -| [hedgehog-23.12.0.iso](/iso/hedgehog-23.12.0.iso) (2.4GiB) | [`835160cc0d2e3608754736989088d912c17372c49764244742e0572af9295d4b`](/iso/hedgehog-23.12.0.iso.sha256.txt) | +| [hedgehog-23.12.1.iso](/iso/hedgehog-23.12.1.iso) (2.4GiB) | [`009e67d61ae6e8ffa53e8e134091263b91c0f7a442a0717594434761db60b1b5`](/iso/hedgehog-23.12.1.iso.sha256.txt) | ## Warning diff --git a/docs/hedgehog-iso-build.md b/docs/hedgehog-iso-build.md index eb05c0ccb..a97d0d283 100644 --- a/docs/hedgehog-iso-build.md +++ b/docs/hedgehog-iso-build.md @@ -29,7 +29,7 @@ Building the ISO may take 90 minutes or more depending on your system. As the bu ``` … -Finished, created "/sensor-build/hedgehog-23.12.0.iso" +Finished, created "/sensor-build/hedgehog-23.12.1.iso" … ``` diff --git a/docs/hedgehog-upgrade.md b/docs/hedgehog-upgrade.md index 6c7e69e29..f106bb030 100644 --- a/docs/hedgehog-upgrade.md +++ b/docs/hedgehog-upgrade.md @@ -208,9 +208,9 @@ commands: ``` chown root:netdev /usr/sbin/netsniff-ng && \ - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip CAP_SYS_ADMIN+eip' /usr/sbin/netsniff-ng + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /usr/sbin/netsniff-ng chown root:netdev /opt/zeek/bin/zeek && \ - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /opt/zeek/bin/zeek + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /opt/zeek/bin/zeek chown root:netdev /sbin/ethtool && \ setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool chown root:netdev /opt/zeek/bin/capstats && \ @@ -218,7 +218,7 @@ chown root:netdev /opt/zeek/bin/capstats && \ chown root:netdev /usr/bin/tcpdump && \ setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /usr/bin/tcpdump chown root:netdev /opt/arkime/bin/capture && \ - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /opt/arkime/bin/capture + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /opt/arkime/bin/capture ln -s -f /opt/zeek/bin/zeek /usr/local/bin/ ln -s -f /usr/sbin/netsniff-ng /usr/local/bin/ @@ -233,9 +233,9 @@ example: ``` root@hedgehog:/tmp# chown root:netdev /usr/sbin/netsniff-ng && \ -> setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip CAP_SYS_ADMIN+eip' /usr/sbin/netsniff-ng +> setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /usr/sbin/netsniff-ng root@hedgehog:/tmp# chown root:netdev /opt/zeek/bin/zeek && \ -> setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /opt/zeek/bin/zeek +> setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /opt/zeek/bin/zeek root@hedgehog:/tmp# chown root:netdev /sbin/ethtool && \ > setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool root@hedgehog:/tmp# chown root:netdev /opt/zeek/bin/capstats && \ @@ -243,7 +243,7 @@ root@hedgehog:/tmp# chown root:netdev /opt/zeek/bin/capstats && \ root@hedgehog:/tmp# chown root:netdev /usr/bin/tcpdump && \ > setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /usr/bin/tcpdump root@hedgehog:/tmp# chown root:netdev /opt/arkime/bin/capture && \ -> setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /opt/arkime/bin/capture +> setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /opt/arkime/bin/capture root@hedgehog:/tmp# ln -s -f /opt/zeek/bin/zeek /usr/local/bin/ root@hedgehog:/tmp# ln -s -f /usr/sbin/netsniff-ng /usr/local/bin/ root@hedgehog:/tmp# ln -s -f /usr/bin/tcpdump /usr/local/bin/ diff --git a/docs/kubernetes.md b/docs/kubernetes.md index da570cb96..b1b351d50 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -272,28 +272,28 @@ agent2 | agent2 | 192.168.56.12 | agent2 | k3s | 6000m | agent1 | agent1 | 192.168.56.11 | agent1 | k3s | 6000m | 861.34m | 14.36% | 19.55Gi | 9.29Gi | 61.28Gi | 11 | Pod Name | State | Pod IP | Pod Kind | Worker Node | CPU Usage | Memory Usage | Container Name:Restarts | Container Image | -api-deployment-6f4686cf59-bn286 | Running | 10.42.2.14 | ReplicaSet | agent1 | 0.11m | 59.62Mi | api-container:0 | api:23.12.0 | -file-monitor-deployment-855646bd75-vk7st | Running | 10.42.2.16 | ReplicaSet | agent1 | 8.47m | 1.46Gi | file-monitor-container:0 | file-monitor:23.12.0 | -zeek-live-deployment-64b69d4b6f-947vr | Running | 10.42.2.17 | ReplicaSet | agent1 | 0.02m | 12.44Mi | zeek-live-container:0 | zeek:23.12.0 | -dashboards-helper-deployment-69dc54f6b6-ln4sq | Running | 10.42.2.15 | ReplicaSet | agent1 | 10.77m | 38.43Mi | dashboards-helper-container:0 | dashboards-helper:23.12.0 | -upload-deployment-586568844b-4jnk9 | Running | 10.42.2.18 | ReplicaSet | agent1 | 0.15m | 29.78Mi | upload-container:0 | file-upload:23.12.0 | -filebeat-deployment-6ff8bc444f-t7h49 | Running | 10.42.2.20 | ReplicaSet | agent1 | 2.84m | 70.71Mi | filebeat-container:0 | filebeat-oss:23.12.0 | -zeek-offline-deployment-844f4865bd-g2sdm | Running | 10.42.2.21 | ReplicaSet | agent1 | 0.17m | 41.92Mi | zeek-offline-container:0 | zeek:23.12.0 | -logstash-deployment-6fbc9fdcd5-hwx8s | Running | 10.42.2.22 | ReplicaSet | agent1 | 85.55m | 2.91Gi | logstash-container:0 | logstash-oss:23.12.0 | -netbox-deployment-cdcff4977-hbbw5 | Running | 10.42.2.23 | ReplicaSet | agent1 | 807.64m | 702.86Mi | netbox-container:0 | netbox:23.12.0 | -suricata-offline-deployment-6ccdb89478-z5696 | Running | 10.42.2.19 | ReplicaSet | agent1 | 0.22m | 34.88Mi | suricata-offline-container:0 | suricata:23.12.0 | -dashboards-deployment-69b5465db-vz88g | Running | 10.42.1.14 | ReplicaSet | agent2 | 0.94m | 100.12Mi | dashboards-container:0 | dashboards:23.12.0 | -netbox-redis-cache-deployment-5f77d47b8b-z7t2z | Running | 10.42.1.15 | ReplicaSet | agent2 | 3.57m | 7.36Mi | netbox-redis-cache-container:0 | redis:23.12.0 | -suricata-live-deployment-6494c77759-9rlnt | Running | 10.42.1.16 | ReplicaSet | agent2 | 0.02m | 9.69Mi | suricata-live-container:0 | suricata:23.12.0 | -freq-deployment-cfd84fd97-dnngf | Running | 10.42.1.17 | ReplicaSet | agent2 | 0.2m | 26.36Mi | freq-container:0 | freq:23.12.0 | -arkime-deployment-56999cdd66-s98pp | Running | 10.42.1.18 | ReplicaSet | agent2 | 4.15m | 113.07Mi | arkime-container:0 | arkime:23.12.0 | -pcap-monitor-deployment-594ff674c4-fsm7m | Running | 10.42.1.19 | ReplicaSet | agent2 | 1.24m | 48.44Mi | pcap-monitor-container:0 | pcap-monitor:23.12.0 | -pcap-capture-deployment-7c8bf6957-jzpzn | Running | 10.42.1.20 | ReplicaSet | agent2 | 0.02m | 9.64Mi | pcap-capture-container:0 | pcap-capture:23.12.0 | -netbox-postgres-deployment-5879b8dffc-kkt56 | Running | 10.42.1.21 | ReplicaSet | agent2 | 70.91m | 33.02Mi | netbox-postgres-container:0 | postgresql:23.12.0 | -htadmin-deployment-6fc46888b9-sq6ln | Running | 10.42.1.23 | ReplicaSet | agent2 | 0.14m | 30.53Mi | htadmin-container:0 | htadmin:23.12.0 | -netbox-redis-deployment-5bcd8f6c96-j5xpf | Running | 10.42.1.24 | ReplicaSet | agent2 | 1.46m | 7.34Mi | netbox-redis-container:0 | redis:23.12.0 | -nginx-proxy-deployment-69fcc4968d-f68tq | Running | 10.42.1.22 | ReplicaSet | agent2 | 0.31m | 22.63Mi | nginx-proxy-container:0 | nginx-proxy:23.12.0 | -opensearch-deployment-75498799f6-4zmwd | Running | 10.42.1.25 | ReplicaSet | agent2 | 89.8m | 11.03Gi | opensearch-container:0 | opensearch:23.12.0 | +api-deployment-6f4686cf59-bn286 | Running | 10.42.2.14 | ReplicaSet | agent1 | 0.11m | 59.62Mi | api-container:0 | api:23.12.1 | +file-monitor-deployment-855646bd75-vk7st | Running | 10.42.2.16 | ReplicaSet | agent1 | 8.47m | 1.46Gi | file-monitor-container:0 | file-monitor:23.12.1 | +zeek-live-deployment-64b69d4b6f-947vr | Running | 10.42.2.17 | ReplicaSet | agent1 | 0.02m | 12.44Mi | zeek-live-container:0 | zeek:23.12.1 | +dashboards-helper-deployment-69dc54f6b6-ln4sq | Running | 10.42.2.15 | ReplicaSet | agent1 | 10.77m | 38.43Mi | dashboards-helper-container:0 | dashboards-helper:23.12.1 | +upload-deployment-586568844b-4jnk9 | Running | 10.42.2.18 | ReplicaSet | agent1 | 0.15m | 29.78Mi | upload-container:0 | file-upload:23.12.1 | +filebeat-deployment-6ff8bc444f-t7h49 | Running | 10.42.2.20 | ReplicaSet | agent1 | 2.84m | 70.71Mi | filebeat-container:0 | filebeat-oss:23.12.1 | +zeek-offline-deployment-844f4865bd-g2sdm | Running | 10.42.2.21 | ReplicaSet | agent1 | 0.17m | 41.92Mi | zeek-offline-container:0 | zeek:23.12.1 | +logstash-deployment-6fbc9fdcd5-hwx8s | Running | 10.42.2.22 | ReplicaSet | agent1 | 85.55m | 2.91Gi | logstash-container:0 | logstash-oss:23.12.1 | +netbox-deployment-cdcff4977-hbbw5 | Running | 10.42.2.23 | ReplicaSet | agent1 | 807.64m | 702.86Mi | netbox-container:0 | netbox:23.12.1 | +suricata-offline-deployment-6ccdb89478-z5696 | Running | 10.42.2.19 | ReplicaSet | agent1 | 0.22m | 34.88Mi | suricata-offline-container:0 | suricata:23.12.1 | +dashboards-deployment-69b5465db-vz88g | Running | 10.42.1.14 | ReplicaSet | agent2 | 0.94m | 100.12Mi | dashboards-container:0 | dashboards:23.12.1 | +netbox-redis-cache-deployment-5f77d47b8b-z7t2z | Running | 10.42.1.15 | ReplicaSet | agent2 | 3.57m | 7.36Mi | netbox-redis-cache-container:0 | redis:23.12.1 | +suricata-live-deployment-6494c77759-9rlnt | Running | 10.42.1.16 | ReplicaSet | agent2 | 0.02m | 9.69Mi | suricata-live-container:0 | suricata:23.12.1 | +freq-deployment-cfd84fd97-dnngf | Running | 10.42.1.17 | ReplicaSet | agent2 | 0.2m | 26.36Mi | freq-container:0 | freq:23.12.1 | +arkime-deployment-56999cdd66-s98pp | Running | 10.42.1.18 | ReplicaSet | agent2 | 4.15m | 113.07Mi | arkime-container:0 | arkime:23.12.1 | +pcap-monitor-deployment-594ff674c4-fsm7m | Running | 10.42.1.19 | ReplicaSet | agent2 | 1.24m | 48.44Mi | pcap-monitor-container:0 | pcap-monitor:23.12.1 | +pcap-capture-deployment-7c8bf6957-jzpzn | Running | 10.42.1.20 | ReplicaSet | agent2 | 0.02m | 9.64Mi | pcap-capture-container:0 | pcap-capture:23.12.1 | +netbox-postgres-deployment-5879b8dffc-kkt56 | Running | 10.42.1.21 | ReplicaSet | agent2 | 70.91m | 33.02Mi | netbox-postgres-container:0 | postgresql:23.12.1 | +htadmin-deployment-6fc46888b9-sq6ln | Running | 10.42.1.23 | ReplicaSet | agent2 | 0.14m | 30.53Mi | htadmin-container:0 | htadmin:23.12.1 | +netbox-redis-deployment-5bcd8f6c96-j5xpf | Running | 10.42.1.24 | ReplicaSet | agent2 | 1.46m | 7.34Mi | netbox-redis-container:0 | redis:23.12.1 | +nginx-proxy-deployment-69fcc4968d-f68tq | Running | 10.42.1.22 | ReplicaSet | agent2 | 0.31m | 22.63Mi | nginx-proxy-container:0 | nginx-proxy:23.12.1 | +opensearch-deployment-75498799f6-4zmwd | Running | 10.42.1.25 | ReplicaSet | agent2 | 89.8m | 11.03Gi | opensearch-container:0 | opensearch:23.12.1 | ``` The other control scripts (`stop`, `restart`, `logs`, etc.) work in a similar manner as in a Docker-based deployment. One notable difference is the `wipe` script: data on PersistentVolume storage cannot be deleted by `wipe`. It must be deleted manually on the storage media underlying the PersistentVolumes. @@ -307,8 +307,6 @@ Here is a basic step-by-step example illustrating how to deploy Malcolm with Kub ``` $ ls -l total 45,056 -drwxr-xr-x 2 user user 6 Apr 24 14:35 arkime-logs -drwxr-xr-x 2 user user 6 Apr 24 14:35 arkime-raw drwxr-xr-x 2 user user 4,096 Apr 24 14:35 config drwxr-xr-x 3 user user 19 Apr 24 14:35 filebeat drwxr-xr-x 2 user user 6 Apr 24 14:35 htadmin @@ -377,7 +375,7 @@ Enter index threshold (e.g., 250GB, 1TB, 60%, etc.): 250G Determine oldest indices by name (instead of creation time)? (Y / n): y -Should Arkime delete PCAP files based on available storage (see https://arkime.com/faq#pcap-deletion)? (y / N): y +Should Arkime delete uploaded PCAP files based on available storage (see https://arkime.com/faq#pcap-deletion)? (y / N): y Enter PCAP deletion threshold in gigabytes or as a percentage (e.g., 500, 10%, etc.): 10% @@ -557,28 +555,28 @@ agent1 | agent1 | 192.168.56.11 | agent1 | k3s | 6000m | agent2 | agent2 | 192.168.56.12 | agent2 | k3s | 6000m | 552.71m | 9.21% | 19.55Gi | 13.27Gi | 61.28Gi | 12 | Pod Name | State | Pod IP | Pod Kind | Worker Node | CPU Usage | Memory Usage | Container Name:Restarts | Container Image | -netbox-redis-cache-deployment-5f77d47b8b-jr9nt | Running | 10.42.2.6 | ReplicaSet | agent2 | 1.89m | 7.24Mi | netbox-redis-cache-container:0 | redis:23.12.0 | -netbox-redis-deployment-5bcd8f6c96-bkzmh | Running | 10.42.2.5 | ReplicaSet | agent2 | 1.62m | 7.52Mi | netbox-redis-container:0 | redis:23.12.0 | -dashboards-helper-deployment-69dc54f6b6-ks7ps | Running | 10.42.2.4 | ReplicaSet | agent2 | 12.95m | 40.75Mi | dashboards-helper-container:0 | dashboards-helper:23.12.0 | -freq-deployment-cfd84fd97-5bwp6 | Running | 10.42.2.8 | ReplicaSet | agent2 | 0.11m | 26.33Mi | freq-container:0 | freq:23.12.0 | -pcap-capture-deployment-7c8bf6957-hkvkn | Running | 10.42.2.12 | ReplicaSet | agent2 | 0.02m | 9.21Mi | pcap-capture-container:0 | pcap-capture:23.12.0 | -nginx-proxy-deployment-69fcc4968d-m57rz | Running | 10.42.2.10 | ReplicaSet | agent2 | 0.91m | 22.72Mi | nginx-proxy-container:0 | nginx-proxy:23.12.0 | -htadmin-deployment-6fc46888b9-vpt7l | Running | 10.42.2.7 | ReplicaSet | agent2 | 0.16m | 30.21Mi | htadmin-container:0 | htadmin:23.12.0 | -opensearch-deployment-75498799f6-5v92w | Running | 10.42.2.13 | ReplicaSet | agent2 | 139.2m | 10.86Gi | opensearch-container:0 | opensearch:23.12.0 | -zeek-live-deployment-64b69d4b6f-fcb6n | Running | 10.42.2.9 | ReplicaSet | agent2 | 0.02m | 109.55Mi | zeek-live-container:0 | zeek:23.12.0 | -dashboards-deployment-69b5465db-kgsqk | Running | 10.42.2.3 | ReplicaSet | agent2 | 14.98m | 108.85Mi | dashboards-container:0 | dashboards:23.12.0 | -arkime-deployment-56999cdd66-xxpw9 | Running | 10.42.2.11 | ReplicaSet | agent2 | 208.95m | 78.42Mi | arkime-container:0 | arkime:23.12.0 | -api-deployment-6f4686cf59-xt9md | Running | 10.42.1.3 | ReplicaSet | agent1 | 0.14m | 56.88Mi | api-container:0 | api:23.12.0 | -netbox-postgres-deployment-5879b8dffc-lb4qm | Running | 10.42.1.6 | ReplicaSet | agent1 | 141.2m | 48.02Mi | netbox-postgres-container:0 | postgresql:23.12.0 | -pcap-monitor-deployment-594ff674c4-fwq7g | Running | 10.42.1.12 | ReplicaSet | agent1 | 3.93m | 46.44Mi | pcap-monitor-container:0 | pcap-monitor:23.12.0 | -suricata-offline-deployment-6ccdb89478-j5fgj | Running | 10.42.1.10 | ReplicaSet | agent1 | 10.42m | 35.12Mi | suricata-offline-container:0 | suricata:23.12.0 | -suricata-live-deployment-6494c77759-rpt48 | Running | 10.42.1.8 | ReplicaSet | agent1 | 0.01m | 9.62Mi | suricata-live-container:0 | suricata:23.12.0 | -netbox-deployment-cdcff4977-7ns2q | Running | 10.42.1.7 | ReplicaSet | agent1 | 830.47m | 530.7Mi | netbox-container:0 | netbox:23.12.0 | -zeek-offline-deployment-844f4865bd-7x68b | Running | 10.42.1.9 | ReplicaSet | agent1 | 1.44m | 43.66Mi | zeek-offline-container:0 | zeek:23.12.0 | -filebeat-deployment-6ff8bc444f-pdgzj | Running | 10.42.1.11 | ReplicaSet | agent1 | 0.78m | 75.25Mi | filebeat-container:0 | filebeat-oss:23.12.0 | -file-monitor-deployment-855646bd75-nbngq | Running | 10.42.1.4 | ReplicaSet | agent1 | 1.69m | 1.46Gi | file-monitor-container:0 | file-monitor:23.12.0 | -upload-deployment-586568844b-9s7f5 | Running | 10.42.1.13 | ReplicaSet | agent1 | 0.14m | 29.62Mi | upload-container:0 | file-upload:23.12.0 | -logstash-deployment-6fbc9fdcd5-2hhx8 | Running | 10.42.1.5 | ReplicaSet | agent1 | 3236.29m | 357.36Mi | logstash-container:0 | logstash-oss:23.12.0 | +netbox-redis-cache-deployment-5f77d47b8b-jr9nt | Running | 10.42.2.6 | ReplicaSet | agent2 | 1.89m | 7.24Mi | netbox-redis-cache-container:0 | redis:23.12.1 | +netbox-redis-deployment-5bcd8f6c96-bkzmh | Running | 10.42.2.5 | ReplicaSet | agent2 | 1.62m | 7.52Mi | netbox-redis-container:0 | redis:23.12.1 | +dashboards-helper-deployment-69dc54f6b6-ks7ps | Running | 10.42.2.4 | ReplicaSet | agent2 | 12.95m | 40.75Mi | dashboards-helper-container:0 | dashboards-helper:23.12.1 | +freq-deployment-cfd84fd97-5bwp6 | Running | 10.42.2.8 | ReplicaSet | agent2 | 0.11m | 26.33Mi | freq-container:0 | freq:23.12.1 | +pcap-capture-deployment-7c8bf6957-hkvkn | Running | 10.42.2.12 | ReplicaSet | agent2 | 0.02m | 9.21Mi | pcap-capture-container:0 | pcap-capture:23.12.1 | +nginx-proxy-deployment-69fcc4968d-m57rz | Running | 10.42.2.10 | ReplicaSet | agent2 | 0.91m | 22.72Mi | nginx-proxy-container:0 | nginx-proxy:23.12.1 | +htadmin-deployment-6fc46888b9-vpt7l | Running | 10.42.2.7 | ReplicaSet | agent2 | 0.16m | 30.21Mi | htadmin-container:0 | htadmin:23.12.1 | +opensearch-deployment-75498799f6-5v92w | Running | 10.42.2.13 | ReplicaSet | agent2 | 139.2m | 10.86Gi | opensearch-container:0 | opensearch:23.12.1 | +zeek-live-deployment-64b69d4b6f-fcb6n | Running | 10.42.2.9 | ReplicaSet | agent2 | 0.02m | 109.55Mi | zeek-live-container:0 | zeek:23.12.1 | +dashboards-deployment-69b5465db-kgsqk | Running | 10.42.2.3 | ReplicaSet | agent2 | 14.98m | 108.85Mi | dashboards-container:0 | dashboards:23.12.1 | +arkime-deployment-56999cdd66-xxpw9 | Running | 10.42.2.11 | ReplicaSet | agent2 | 208.95m | 78.42Mi | arkime-container:0 | arkime:23.12.1 | +api-deployment-6f4686cf59-xt9md | Running | 10.42.1.3 | ReplicaSet | agent1 | 0.14m | 56.88Mi | api-container:0 | api:23.12.1 | +netbox-postgres-deployment-5879b8dffc-lb4qm | Running | 10.42.1.6 | ReplicaSet | agent1 | 141.2m | 48.02Mi | netbox-postgres-container:0 | postgresql:23.12.1 | +pcap-monitor-deployment-594ff674c4-fwq7g | Running | 10.42.1.12 | ReplicaSet | agent1 | 3.93m | 46.44Mi | pcap-monitor-container:0 | pcap-monitor:23.12.1 | +suricata-offline-deployment-6ccdb89478-j5fgj | Running | 10.42.1.10 | ReplicaSet | agent1 | 10.42m | 35.12Mi | suricata-offline-container:0 | suricata:23.12.1 | +suricata-live-deployment-6494c77759-rpt48 | Running | 10.42.1.8 | ReplicaSet | agent1 | 0.01m | 9.62Mi | suricata-live-container:0 | suricata:23.12.1 | +netbox-deployment-cdcff4977-7ns2q | Running | 10.42.1.7 | ReplicaSet | agent1 | 830.47m | 530.7Mi | netbox-container:0 | netbox:23.12.1 | +zeek-offline-deployment-844f4865bd-7x68b | Running | 10.42.1.9 | ReplicaSet | agent1 | 1.44m | 43.66Mi | zeek-offline-container:0 | zeek:23.12.1 | +filebeat-deployment-6ff8bc444f-pdgzj | Running | 10.42.1.11 | ReplicaSet | agent1 | 0.78m | 75.25Mi | filebeat-container:0 | filebeat-oss:23.12.1 | +file-monitor-deployment-855646bd75-nbngq | Running | 10.42.1.4 | ReplicaSet | agent1 | 1.69m | 1.46Gi | file-monitor-container:0 | file-monitor:23.12.1 | +upload-deployment-586568844b-9s7f5 | Running | 10.42.1.13 | ReplicaSet | agent1 | 0.14m | 29.62Mi | upload-container:0 | file-upload:23.12.1 | +logstash-deployment-6fbc9fdcd5-2hhx8 | Running | 10.42.1.5 | ReplicaSet | agent1 | 3236.29m | 357.36Mi | logstash-container:0 | logstash-oss:23.12.1 | ``` View container logs for the Malcolm deployment with `./scripts/logs` (if **[stern](https://github.com/stern/stern)** present in `$PATH`): diff --git a/docs/live-analysis.md b/docs/live-analysis.md index 4e8dd7453..1389384db 100644 --- a/docs/live-analysis.md +++ b/docs/live-analysis.md @@ -3,6 +3,7 @@ * [Live analysis](#LiveAnalysis) - [Using a network sensor appliance](#Hedgehog) - [Monitoring local network interfaces](#LocalPCAP) + + ["Hedgehog" run profile](#Profiles) - [Manually forwarding logs from an external source](#ExternalForward) ## Using a network sensor appliance @@ -18,16 +19,22 @@ Please see the [Hedgehog Linux README](hedgehog.md) for more information. ## Monitoring local network interfaces -Malcolm's `pcap-capture`, `suricata-live` and `zeek-live` containers can monitor one or more local network interfaces, specified by the `PCAP_IFACE` environment variable in [`pcap-capture.env`](malcolm-config.md#MalcolmConfigEnvVars). These containers are started with additional privileges (`IPC_LOCK`, `NET_ADMIN`, `NET_RAW`, and `SYS_ADMIN`) to allow opening network interfaces in promiscuous mode for capture. +The options for monitoring traffic on local network interfaces can be [configured](malcolm-hedgehog-e2e-iso-install.md#MalcolmConfig) by running [`./scripts/configure`](malcolm-config.md#ConfigAndTuning). -The instances of Zeek and Suricata (in the `suricata-live` and `zeek-live` containers when the `SURICATA_LIVE_CAPTURE` and `ZEEK_LIVE_CAPTURE` [environment variables](malcolm-config.md#MalcolmConfigEnvVars) are set to `true`, respectively) analyze traffic on-the-fly and generate log files containing network session metadata. These log files are in turn scanned by [Filebeat](https://www.elastic.co/products/beats/filebeat) and forwarded to [Logstash](https://www.elastic.co/products/logstash) for enrichment and indexing into the [OpenSearch](https://opensearch.org/) document store. +Malcolm's `pcap-capture`, `suricata-live` and `zeek-live` containers can monitor one or more local network interfaces, specified by the `PCAP_IFACE` environment variable in [`pcap-capture.env`](malcolm-config.md#MalcolmConfigEnvVars). These containers are started with additional privileges to allow opening network interfaces in promiscuous mode for capture. -In contrast, the `pcap-capture` container buffers traffic to PCAP files and periodically rotates these files for processing (by Arkime's `capture` utlity in the `arkime` container) according to the thresholds defined by the `PCAP_ROTATE_MEGABYTES` and `PCAP_ROTATE_MINUTES` environment variables in [`pcap-capture.env`](malcolm-config.md#MalcolmConfigEnvVars). If for some reason (e.g., a low resources environment) you also want Zeek and Suricata to process these intermediate PCAP files rather than monitoring the network interfaces directly, you can set `SURICATA_ROTATED_PCAP`/`ZEEK_ROTATED_PCAP` to `true` and `SURICATA_LIVE_CAPTURE`/`ZEEK_LIVE_CAPTURE` to false. +The instances of Zeek and Suricata (in the `suricata-live` and `zeek-live` containers when the `SURICATA_LIVE_CAPTURE` and `ZEEK_LIVE_CAPTURE` [environment variables](malcolm-config.md#MalcolmConfigEnvVars) are set to `true`, respectively) analyze traffic on-the-fly and generate log files containing network session metadata. These log files are in turn scanned by [Filebeat](https://www.elastic.co/products/beats/filebeat) and forwarded to [Logstash](https://www.elastic.co/products/logstash) for enrichment and indexing into the [OpenSearch](https://opensearch.org/) document store. -These various options for monitoring traffic on local network interfaces can also be configured by running [`./scripts/configure`](malcolm-config.md#ConfigAndTuning). +In contrast, the `pcap-capture` container buffers traffic to PCAP files and periodically rotates these files for processing (by Arkime's `capture` utlity in the `arkime` container) according to the thresholds defined by the `PCAP_ROTATE_MEGABYTES` and `PCAP_ROTATE_MINUTES` environment variables in [`pcap-capture.env`](malcolm-config.md#MalcolmConfigEnvVars). If for some reason (e.g., a low resources environment) you also want Zeek and Suricata to process these intermediate PCAP files rather than monitoring the network interfaces directly, you can set `SURICATA_ROTATED_PCAP`/`ZEEK_ROTATED_PCAP` to `true` and `SURICATA_LIVE_CAPTURE`/`ZEEK_LIVE_CAPTURE` to false. The only exception to this behavior (i.e., the creation of intermediate PCAP files by `netsniff-ng` or `tcpdump` in the `pcap-capture` which are periodically rolled over for processing by Arkime) is when running the ["Hedgehog" run profile](#Profiles) or when using [a remote OpenSearch or Elasticsearch instance](opensearch-instances.md#OpenSearchInstance). In either of these configurations, users may choose to have Arkime's `capture` tool monitor live traffic on the network interface without using the intermediate PCAP file. Note that Microsoft Windows and Apple macOS platforms currently run Docker inside of a virtualized environment. Live traffic capture and analysis on those platforms would require additional configuration of virtual interfaces and port forwarding in Docker, which is outside of the scope of this document. +### "Hedgehog" run profile + +Another configuration for monitoring local network interfaces is to use the `hedgehog` run profile. During [Malcolm configuration](malcolm-hedgehog-e2e-iso-install.md#MalcolmConfig) users are prompted "**Run with Malcolm (all containers) or Hedgehog (capture only) profile?**" Docker Compose can use [profiles](https://docs.docker.com/compose/profiles/) to selectively start services. While the `malcolm` run profile runs all of Malcolm's containers (OpenSearch, Dashboards, LogStash, etc.), the `hedgehog` profile runs *only* the containers necessary for traffic capture. + +When configuring the `hedgehog` profile, users must provide connection details for another Malcolm instance to which to forward its network traffic logs. + ## Manually forwarding logs from an external source Malcolm's Logstash instance can also be configured to accept logs from a [remote forwarder](https://www.elastic.co/products/beats/filebeat) by running [`./scripts/configure`](malcolm-config.md#ConfigAndTuning) and answering "yes" to "`Expose Logstash port to external hosts?`" Enabling encrypted transport of these log files is discussed in [Configure authentication](authsetup.md#AuthSetup) and the description of the `BEATS_SSL` environment variable in [`beats-common.env`](malcolm-config.md#MalcolmConfigEnvVars). diff --git a/docs/malcolm-config.md b/docs/malcolm-config.md index fc2fe24c2..94ee44959 100644 --- a/docs/malcolm-config.md +++ b/docs/malcolm-config.md @@ -9,7 +9,7 @@ Run `./scripts/configure` and answer the questions to configure Malcolm. For an Although the configuration script automates many of the following configuration and tuning parameters, some environment variables of particular interest are listed here for reference. * **`arkime.env`** and **`arkime-secret.env`** - settings for [Arkime](https://arkime.com/) - - `ARKIME_ANALYZE_PCAP_THREADS` – the number of threads available to Arkime for analyzing PCAP files (default `1`) + - `ARKIME_AUTO_ANALYZE_PCAP_THREADS` – the number of threads available to Arkime for analyzing PCAP files (default `1`) - `ARKIME_PASSWORD_SECRET` - the password hash secret for the Arkime viewer cluster (see `passwordSecret` in [Arkime INI Settings](https://arkime.com/settings)) used to secure the connection used when Arkime viewer retrieves a PCAP payload for display in its user interface - `MANAGE_PCAP_FILES` – if set to `true`, all PCAP files imported into Malcolm will be marked as available for deletion by Arkime if available storage space becomes too low (default `false`) - `MAXMIND_GEOIP_DB_LICENSE_KEY` - Malcolm uses MaxMind's free GeoLite2 databases for GeoIP lookups. As of December 30, 2019, these databases are [no longer available](https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-geolite2-databases/) for download via a public URL. Instead, they must be downloaded using a MaxMind license key (available without charge [from MaxMind](https://www.maxmind.com/en/geolite2/signup)). The license key can be specified here for GeoIP database downloads during build- and run-time. @@ -69,12 +69,12 @@ Although the configuration script automates many of the following configuration - `SURICATA_UPDATE_RULES` – if set to `true`, Suricata signatures will periodically be updated (default `false`) - `SURICATA_LIVE_CAPTURE` - if set to `true`, Suricata will monitor live traffic on the local interface(s) defined by `PCAP_FILTER` - `SURICATA_ROTATED_PCAP` - if set to `true`, Suricata can analyze PCAP files captured by `netsniff-ng` or `tcpdump` (see `PCAP_ENABLE_NETSNIFF` and `PCAP_ENABLE_TCPDUMP`, as well as `SURICATA_AUTO_ANALYZE_PCAP_FILES`); if `SURICATA_LIVE_CAPTURE` is `true`, this should be `false`; otherwise Suricata will see duplicate traffic + - `SURICATA_DISABLE_ICS_ALL` - if set to `true`, this variable can be used to disable Malcolm's [built-in Suricata rules for Operational Technology/Industrial Control Systems (OT/ICS) vulnerabilities and exploits]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/suricata/rules-default/OT) - `SURICATA_…` - the [`suricata` container entrypoint script]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/suricata_config_populate.py) can use **many** more environment variables to tweak [suricata.yaml](https://github.com/OISF/suricata/blob/master/suricata.yaml.in); in that script, `DEFAULT_VARS` defines those variables (albeit without the `SURICATA_` prefix you must add to each for use) Note that for some variables (e.g., something with a sequence like `HOME_NET`) Suricata wants values to be quoted. To accomplish that in the `suricata.env` file, use outer single quotes with inner double quotes, like this: + `SURICATA_HOME_NET='"[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]"'` * **`upload-common.env`** - settings for dealing with PCAP files [uploaded](upload.md#Upload) to Malcolm for analysis - `AUTO_TAG` – if set to `true`, Malcolm will automatically create Arkime sessions and Zeek logs with tags based on the filename, as described in [Tagging](upload.md#Tagging) (default `true`) - `PCAP_NODE_NAME` - specifies the node name to associate with network traffic metadata - - `PCAP_NODE_HOST` - specifies the node host or IP address to associate with network traffic metadata (optional, defaults to the value of `PCAP_NODE_NAME`; generally this value *does not* need to be specified) * **`zeek.env`**, **`zeek-secret.env`**, **`zeek-live.env`** and **`zeek-offline.env`** - settings for [Zeek](https://www.zeek.org/index.html) and for scanning [extracted files](file-scanning.md#ZeekFileExtraction) Zeek observes in network traffic - `EXTRACTED_FILE_CAPA_VERBOSE` – if set to `true`, all Capa rule hits will be logged; otherwise (`false`) only [MITRE ATT&CK® technique](https://attack.mitre.org/techniques) classifications will be logged - `EXTRACTED_FILE_ENABLE_CAPA` – if set to `true`, [Zeek-extracted files](file-scanning.md#ZeekFileExtraction) determined to be PE (portable executable) files will be scanned with [Capa](https://github.com/fireeye/capa) diff --git a/docs/malcolm-hedgehog-e2e-iso-install.md b/docs/malcolm-hedgehog-e2e-iso-install.md index a537f9f65..cc756826a 100644 --- a/docs/malcolm-hedgehog-e2e-iso-install.md +++ b/docs/malcolm-hedgehog-e2e-iso-install.md @@ -177,7 +177,7 @@ The [configuration and tuning](malcolm-config.md#ConfigAndTuning) wizard's quest - Choose **Y** to proceed to the following related questions about managing the data storage used by Malcolm. - **Delete the oldest indices when the database exceeds a certain size?** - Most of the configuration around OpenSearch [Index State Management](https://opensearch.org/docs/latest/im-plugin/ism/index/) and [Snapshot Management](https://opensearch.org/docs/latest/opensearch/snapshots/sm-dashboards/) can be done in OpenSearch Dashboards. In addition to (or instead of) the OpenSearch index state management operations, Malcolm can also be configured to delete the oldest network session metadata indices when the database exceeds a certain size to prevent filling up all available storage with OpenSearch indices. - - **Should Arkime delete PCAP files based on available storage?** + - **Should Arkime delete uploaded PCAP files based on available storage?** - Answering **Y** allows Arkime to prune (delete) old PCAP files based on available disk space (see https://arkime.com/faq#pcap-deletion). - **Enter PCAP deletion threshold in gigabytes or as a percentage (e.g., 500, 10%, etc.)** - If [Arkime PCAP-deletion](https://arkime.com/faq#pcap-deletion) is enabled, Arkime will delete PCAP files when **free space** is lower than this value, specified as integer gigabytes (e.g., `500`) or a percentage (e.g., `10%`) @@ -572,9 +572,9 @@ Despite configuring capture and/or forwarder services as described in previous s * **AUTOSTART_FILEBEAT** - [filebeat](#Hedgehogfilebeat) Zeek and Suricata log forwarder * **AUTOSTART_FLUENTBIT_AIDE** - [Fluent Bit](https://fluentbit.io/) agent [monitoring](https://docs.fluentbit.io/manual/pipeline/inputs/exec) [AIDE](https://aide.github.io/) file system integrity checks * **AUTOSTART_FLUENTBIT_AUDITLOG** - [Fluent Bit](https://fluentbit.io/) agent [monitoring](https://docs.fluentbit.io/manual/pipeline/inputs/tail) [auditd](https://man7.org/linux/man-pages/man8/auditd.8.html) logs -* *AUTOSTART_FLUENTBIT_KMSG* - [Fluent Bit](https://fluentbit.io/) agent [monitoring](https://docs.fluentbit.io/manual/pipeline/inputs/kernel-logs) the Linux kernel log buffer (these are generally reflected in syslog as well, which may make this agent redundant) +* *AUTOSTART_FLUENTBIT_KMSG* - [Fluent Bit](https://fluentbit.io/) agent [monitoring](https://docs.fluentbit.io/manual/pipeline/inputs/kernel-logs) the Linux kernel log buffer (these are generally reflected in the Systemd log as well, which may make this agent redundant) * **AUTOSTART_FLUENTBIT_METRICS** - [Fluent Bit](https://fluentbit.io/) agent for collecting [various](https://docs.fluentbit.io/manual/pipeline/inputs) system resource and performance metrics -* **AUTOSTART_FLUENTBIT_SYSLOG** - [Fluent Bit](https://fluentbit.io/) agent [monitoring](https://docs.fluentbit.io/manual/pipeline/inputs/syslog) Linux syslog messages +* **AUTOSTART_FLUENTBIT_SYSTEMD** - [Fluent Bit](https://fluentbit.io/) agent [monitoring](https://docs.fluentbit.io/manual/pipeline/inputs/systemd) log messages from the Linux Journald daemon * **AUTOSTART_FLUENTBIT_THERMAL** - [Fluent Bit](https://fluentbit.io/) agent [monitoring](https://docs.fluentbit.io/manual/pipeline/inputs/thermal) system temperatures (only applicable on actual hardware, not if Hedgehog is running on a virtual machine) * **AUTOSTART_MISCBEAT** - [filebeat](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-tcp.html) forwarder which sends system metrics collected by [Fluent Bit](https://fluentbit.io/) to a remote Logstash instance (e.g., [Malcolm]({{ site.github.repository_url }})'s) * *AUTOSTART_NETSNIFF* - [netsniff-ng](http://netsniff-ng.org/) PCAP engine for saving packet capture (PCAP) files @@ -617,7 +617,7 @@ fluentbit-metrics:disk RUNNING pid 6468, uptime 0:03:17 fluentbit-metrics:mem RUNNING pid 6472, uptime 0:03:17 fluentbit-metrics:mem_p RUNNING pid 6473, uptime 0:03:17 fluentbit-metrics:netif RUNNING pid 6474, uptime 0:03:17 -fluentbit-syslog RUNNING pid 6478, uptime 0:03:17 +fluentbit-systemd RUNNING pid 6478, uptime 0:03:17 fluentbit-thermal RUNNING pid 6480, uptime 0:03:17 netsniff:netsniff-enp1s0 STOPPED Not started prune:prune-pcap RUNNING pid 6484, uptime 0:03:17 diff --git a/docs/malcolm-iso.md b/docs/malcolm-iso.md index 309c48309..640276d77 100644 --- a/docs/malcolm-iso.md +++ b/docs/malcolm-iso.md @@ -41,7 +41,7 @@ Building the ISO may take 30 minutes or more depending on the system. As the bui ``` … -Finished, created "/malcolm-build/malcolm-iso/malcolm-23.12.0.iso" +Finished, created "/malcolm-build/malcolm-iso/malcolm-23.12.1.iso" … ``` diff --git a/docs/quickstart.md b/docs/quickstart.md index 0a6accdb5..91adf3d93 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -54,25 +54,25 @@ You can then observe the images have been retrieved by running `docker images`: ``` $ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -ghcr.io/idaholab/malcolm/api 23.12.0 xxxxxxxxxxxx 3 days ago 158MB -ghcr.io/idaholab/malcolm/arkime 23.12.0 xxxxxxxxxxxx 3 days ago 816MB -ghcr.io/idaholab/malcolm/dashboards 23.12.0 xxxxxxxxxxxx 3 days ago 1.02GB -ghcr.io/idaholab/malcolm/dashboards-helper 23.12.0 xxxxxxxxxxxx 3 days ago 184MB -ghcr.io/idaholab/malcolm/file-monitor 23.12.0 xxxxxxxxxxxx 3 days ago 588MB -ghcr.io/idaholab/malcolm/file-upload 23.12.0 xxxxxxxxxxxx 3 days ago 259MB -ghcr.io/idaholab/malcolm/filebeat-oss 23.12.0 xxxxxxxxxxxx 3 days ago 624MB -ghcr.io/idaholab/malcolm/freq 23.12.0 xxxxxxxxxxxx 3 days ago 132MB -ghcr.io/idaholab/malcolm/htadmin 23.12.0 xxxxxxxxxxxx 3 days ago 242MB -ghcr.io/idaholab/malcolm/logstash-oss 23.12.0 xxxxxxxxxxxx 3 days ago 1.35GB -ghcr.io/idaholab/malcolm/netbox 23.12.0 xxxxxxxxxxxx 3 days ago 1.01GB -ghcr.io/idaholab/malcolm/nginx-proxy 23.12.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/opensearch 23.12.0 xxxxxxxxxxxx 3 days ago 1.17GB -ghcr.io/idaholab/malcolm/pcap-capture 23.12.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/pcap-monitor 23.12.0 xxxxxxxxxxxx 3 days ago 213MB -ghcr.io/idaholab/malcolm/postgresql 23.12.0 xxxxxxxxxxxx 3 days ago 268MB -ghcr.io/idaholab/malcolm/redis 23.12.0 xxxxxxxxxxxx 3 days ago 34.2MB -ghcr.io/idaholab/malcolm/suricata 23.12.0 xxxxxxxxxxxx 3 days ago 278MB -ghcr.io/idaholab/malcolm/zeek 23.12.0 xxxxxxxxxxxx 3 days ago 1GB +ghcr.io/idaholab/malcolm/api 23.12.1 xxxxxxxxxxxx 3 days ago 158MB +ghcr.io/idaholab/malcolm/arkime 23.12.1 xxxxxxxxxxxx 3 days ago 816MB +ghcr.io/idaholab/malcolm/dashboards 23.12.1 xxxxxxxxxxxx 3 days ago 1.02GB +ghcr.io/idaholab/malcolm/dashboards-helper 23.12.1 xxxxxxxxxxxx 3 days ago 184MB +ghcr.io/idaholab/malcolm/file-monitor 23.12.1 xxxxxxxxxxxx 3 days ago 588MB +ghcr.io/idaholab/malcolm/file-upload 23.12.1 xxxxxxxxxxxx 3 days ago 259MB +ghcr.io/idaholab/malcolm/filebeat-oss 23.12.1 xxxxxxxxxxxx 3 days ago 624MB +ghcr.io/idaholab/malcolm/freq 23.12.1 xxxxxxxxxxxx 3 days ago 132MB +ghcr.io/idaholab/malcolm/htadmin 23.12.1 xxxxxxxxxxxx 3 days ago 242MB +ghcr.io/idaholab/malcolm/logstash-oss 23.12.1 xxxxxxxxxxxx 3 days ago 1.35GB +ghcr.io/idaholab/malcolm/netbox 23.12.1 xxxxxxxxxxxx 3 days ago 1.01GB +ghcr.io/idaholab/malcolm/nginx-proxy 23.12.1 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/opensearch 23.12.1 xxxxxxxxxxxx 3 days ago 1.17GB +ghcr.io/idaholab/malcolm/pcap-capture 23.12.1 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/pcap-monitor 23.12.1 xxxxxxxxxxxx 3 days ago 213MB +ghcr.io/idaholab/malcolm/postgresql 23.12.1 xxxxxxxxxxxx 3 days ago 268MB +ghcr.io/idaholab/malcolm/redis 23.12.1 xxxxxxxxxxxx 3 days ago 34.2MB +ghcr.io/idaholab/malcolm/suricata 23.12.1 xxxxxxxxxxxx 3 days ago 278MB +ghcr.io/idaholab/malcolm/zeek 23.12.1 xxxxxxxxxxxx 3 days ago 1GB ``` ### Import from pre-packaged tarballs diff --git a/docs/ubuntu-install-example.md b/docs/ubuntu-install-example.md index 9278935e1..389a0ad39 100644 --- a/docs/ubuntu-install-example.md +++ b/docs/ubuntu-install-example.md @@ -179,7 +179,7 @@ Scan extracted PE files with Capa? (y / N): y Lookup extracted file hashes with VirusTotal? (y / N): n -Download updated file scanner signatures periodically? (n/Y): y +Download updated file scanner signatures periodically? (Y / n): n Should Malcolm run and maintain an instance of NetBox, an infrastructure resource modeling tool? (y / N): n @@ -190,7 +190,9 @@ Should Malcolm capture live network traffic? 2 Specify capture interface(s) (comma-separated): eth0 -Enable dark mode for OpenSearch Dashboards? (n/Y): y +Enable dark mode for OpenSearch Dashboards? (Y / n): y + +Pull Malcolm Docker images (y / N): y Malcolm has been installed to /home/user/Malcolm. See README.md for more information. Scripts for starting and stopping Malcolm and changing authentication-related settings can be found in /home/user/Malcolm/scripts. @@ -227,7 +229,7 @@ As an alternative to manually copying the files to the sensor, Malcolm can facil In this example, rather than [building Malcolm from scratch](development.md#Build), images may be pulled from [GitHub](https://github.com/orgs/idaholab/packages?repo_name=Malcolm): ``` -user@host:~/Malcolm$ docker compose pull +user@host:~/Malcolm$ docker compose --profile malcolm pull Pulling api ... done Pulling arkime ... done Pulling dashboards ... done @@ -250,25 +252,25 @@ Pulling zeek ... done user@host:~/Malcolm$ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -ghcr.io/idaholab/malcolm/api 23.12.0 xxxxxxxxxxxx 3 days ago 158MB -ghcr.io/idaholab/malcolm/arkime 23.12.0 xxxxxxxxxxxx 3 days ago 816MB -ghcr.io/idaholab/malcolm/dashboards 23.12.0 xxxxxxxxxxxx 3 days ago 1.02GB -ghcr.io/idaholab/malcolm/dashboards-helper 23.12.0 xxxxxxxxxxxx 3 days ago 184MB -ghcr.io/idaholab/malcolm/file-monitor 23.12.0 xxxxxxxxxxxx 3 days ago 588MB -ghcr.io/idaholab/malcolm/file-upload 23.12.0 xxxxxxxxxxxx 3 days ago 259MB -ghcr.io/idaholab/malcolm/filebeat-oss 23.12.0 xxxxxxxxxxxx 3 days ago 624MB -ghcr.io/idaholab/malcolm/freq 23.12.0 xxxxxxxxxxxx 3 days ago 132MB -ghcr.io/idaholab/malcolm/htadmin 23.12.0 xxxxxxxxxxxx 3 days ago 242MB -ghcr.io/idaholab/malcolm/logstash-oss 23.12.0 xxxxxxxxxxxx 3 days ago 1.35GB -ghcr.io/idaholab/malcolm/netbox 23.12.0 xxxxxxxxxxxx 3 days ago 1.01GB -ghcr.io/idaholab/malcolm/nginx-proxy 23.12.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/opensearch 23.12.0 xxxxxxxxxxxx 3 days ago 1.17GB -ghcr.io/idaholab/malcolm/pcap-capture 23.12.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/pcap-monitor 23.12.0 xxxxxxxxxxxx 3 days ago 213MB -ghcr.io/idaholab/malcolm/postgresql 23.12.0 xxxxxxxxxxxx 3 days ago 268MB -ghcr.io/idaholab/malcolm/redis 23.12.0 xxxxxxxxxxxx 3 days ago 34.2MB -ghcr.io/idaholab/malcolm/suricata 23.12.0 xxxxxxxxxxxx 3 days ago 278MB -ghcr.io/idaholab/malcolm/zeek 23.12.0 xxxxxxxxxxxx 3 days ago 1GB +ghcr.io/idaholab/malcolm/api 23.12.1 xxxxxxxxxxxx 3 days ago 158MB +ghcr.io/idaholab/malcolm/arkime 23.12.1 xxxxxxxxxxxx 3 days ago 816MB +ghcr.io/idaholab/malcolm/dashboards 23.12.1 xxxxxxxxxxxx 3 days ago 1.02GB +ghcr.io/idaholab/malcolm/dashboards-helper 23.12.1 xxxxxxxxxxxx 3 days ago 184MB +ghcr.io/idaholab/malcolm/file-monitor 23.12.1 xxxxxxxxxxxx 3 days ago 588MB +ghcr.io/idaholab/malcolm/file-upload 23.12.1 xxxxxxxxxxxx 3 days ago 259MB +ghcr.io/idaholab/malcolm/filebeat-oss 23.12.1 xxxxxxxxxxxx 3 days ago 624MB +ghcr.io/idaholab/malcolm/freq 23.12.1 xxxxxxxxxxxx 3 days ago 132MB +ghcr.io/idaholab/malcolm/htadmin 23.12.1 xxxxxxxxxxxx 3 days ago 242MB +ghcr.io/idaholab/malcolm/logstash-oss 23.12.1 xxxxxxxxxxxx 3 days ago 1.35GB +ghcr.io/idaholab/malcolm/netbox 23.12.1 xxxxxxxxxxxx 3 days ago 1.01GB +ghcr.io/idaholab/malcolm/nginx-proxy 23.12.1 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/opensearch 23.12.1 xxxxxxxxxxxx 3 days ago 1.17GB +ghcr.io/idaholab/malcolm/pcap-capture 23.12.1 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/pcap-monitor 23.12.1 xxxxxxxxxxxx 3 days ago 213MB +ghcr.io/idaholab/malcolm/postgresql 23.12.1 xxxxxxxxxxxx 3 days ago 268MB +ghcr.io/idaholab/malcolm/redis 23.12.1 xxxxxxxxxxxx 3 days ago 34.2MB +ghcr.io/idaholab/malcolm/suricata 23.12.1 xxxxxxxxxxxx 3 days ago 278MB +ghcr.io/idaholab/malcolm/zeek 23.12.1 xxxxxxxxxxxx 3 days ago 1GB ``` Finally, start Malcolm. When Malcolm starts it will stream informational and debug messages to the console until it has completed initializing. diff --git a/file-monitor/docker-entrypoint.sh b/file-monitor/docker-entrypoint.sh index eb9955a1c..0027ecfd6 100755 --- a/file-monitor/docker-entrypoint.sh +++ b/file-monitor/docker-entrypoint.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. if [[ -z $EXTRACTED_FILE_ENABLE_CLAMAV ]]; then EXTRACTED_FILE_ENABLE_CLAMAV=false diff --git a/file-monitor/supervisord.conf b/file-monitor/supervisord.conf index d39508348..78cf4d79b 100644 --- a/file-monitor/supervisord.conf +++ b/file-monitor/supervisord.conf @@ -1,4 +1,4 @@ -; Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +; Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. [unix_http_server] file=/tmp/supervisor.sock ; (the path to the socket file) diff --git a/file-upload/docker-entrypoint.sh b/file-upload/docker-entrypoint.sh index 85dacd97f..f4b3c3afc 100755 --- a/file-upload/docker-entrypoint.sh +++ b/file-upload/docker-entrypoint.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. if [[ -z $MALCOLM_USERNAME || -z $MALCOLM_PASSWORD ]]; then echo "Please set the SSH username and (openssl-encrypted then base64-encoded) password by adding the following arguments to docker run/create:" diff --git a/file-upload/supervisord.conf b/file-upload/supervisord.conf index f85dac136..5a6f22aa2 100644 --- a/file-upload/supervisord.conf +++ b/file-upload/supervisord.conf @@ -1,4 +1,4 @@ -; Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +; Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. [unix_http_server] file=/tmp/supervisor.sock ; (the path to the socket file) diff --git a/filebeat/filebeat-nginx.yml b/filebeat/filebeat-nginx.yml index 7bd453f6c..2d1247d13 100644 --- a/filebeat/filebeat-nginx.yml +++ b/filebeat/filebeat-nginx.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. name: "${PCAP_NODE_NAME:malcolm}" diff --git a/filebeat/filebeat-tcp.yml b/filebeat/filebeat-tcp.yml index 6a208eedc..dd5353579 100644 --- a/filebeat/filebeat-tcp.yml +++ b/filebeat/filebeat-tcp.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. name: "${PCAP_NODE_NAME:malcolm}" diff --git a/filebeat/filebeat.yml b/filebeat/filebeat.yml index 046c1947f..f4429083a 100644 --- a/filebeat/filebeat.yml +++ b/filebeat/filebeat.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. name: "${PCAP_NODE_NAME:malcolm}" diff --git a/filebeat/scripts/filebeat-clean-zeeklogs-processed-folder.py b/filebeat/scripts/filebeat-clean-zeeklogs-processed-folder.py index 736e7c0e7..2484f9c39 100755 --- a/filebeat/scripts/filebeat-clean-zeeklogs-processed-folder.py +++ b/filebeat/scripts/filebeat-clean-zeeklogs-processed-folder.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. import os @@ -9,7 +9,6 @@ import errno import time import fcntl -import fnmatch import magic import json import pprint diff --git a/filebeat/scripts/filebeat-process-zeek-folder-functions.sh b/filebeat/scripts/filebeat-process-zeek-folder-functions.sh index 806e0e44c..17cd2e8ff 100755 --- a/filebeat/scripts/filebeat-process-zeek-folder-functions.sh +++ b/filebeat/scripts/filebeat-process-zeek-folder-functions.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. function in_array() { local haystack="${1}[@]" diff --git a/filebeat/scripts/filebeat-process-zeek-folder.sh b/filebeat/scripts/filebeat-process-zeek-folder.sh index 7b0c89c3c..825e26609 100755 --- a/filebeat/scripts/filebeat-process-zeek-folder.sh +++ b/filebeat/scripts/filebeat-process-zeek-folder.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # for files (sort -V (natural)) under /zeek that: diff --git a/filebeat/scripts/filebeat-watch-zeeklogs-uploads-folder.py b/filebeat/scripts/filebeat-watch-zeeklogs-uploads-folder.py index 7139361b6..aca0a0893 100755 --- a/filebeat/scripts/filebeat-watch-zeeklogs-uploads-folder.py +++ b/filebeat/scripts/filebeat-watch-zeeklogs-uploads-folder.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ################################################################################################### # Monitor a directory for PCAP files for processing (by publishing their filenames to a ZMQ socket) diff --git a/filebeat/scripts/zeek-log-field-bitmap.py b/filebeat/scripts/zeek-log-field-bitmap.py index 8e05898ea..6ee960c40 100755 --- a/filebeat/scripts/zeek-log-field-bitmap.py +++ b/filebeat/scripts/zeek-log-field-bitmap.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ################################################################################################### # parse the fields names from the header of of the log file and compare them to the diff --git a/filebeat/supervisord.conf b/filebeat/supervisord.conf index 705ebc304..24c402bf6 100644 --- a/filebeat/supervisord.conf +++ b/filebeat/supervisord.conf @@ -1,4 +1,4 @@ -; Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +; Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. [unix_http_server] file=/tmp/supervisor.sock ; (the path to the socket file) diff --git a/freq-server/supervisord.conf b/freq-server/supervisord.conf index e1f73dff7..0fc672422 100644 --- a/freq-server/supervisord.conf +++ b/freq-server/supervisord.conf @@ -1,4 +1,4 @@ -; Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +; Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. [unix_http_server] file=/tmp/supervisor.sock ; (the path to the socket file) diff --git a/htadmin/supervisord.conf b/htadmin/supervisord.conf index c93a6d9b0..c0aba0f9d 100644 --- a/htadmin/supervisord.conf +++ b/htadmin/supervisord.conf @@ -1,4 +1,4 @@ -; Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +; Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. [unix_http_server] file=/tmp/supervisor.sock ; (the path to the socket file) diff --git a/kubernetes/03-opensearch.yml b/kubernetes/03-opensearch.yml index 2f0ff97b0..97ef53dfe 100644 --- a/kubernetes/03-opensearch.yml +++ b/kubernetes/03-opensearch.yml @@ -30,14 +30,16 @@ spec: spec: containers: - name: opensearch-container - image: ghcr.io/idaholab/malcolm/opensearch:23.12.0 + image: ghcr.io/idaholab/malcolm/opensearch:23.12.1 imagePullPolicy: Always stdin: false tty: true securityContext: capabilities: add: + # IPC_LOCK - to lock memory, preventing swapping - IPC_LOCK + # SYS_RESOURCE - for increasing memlock limits - SYS_RESOURCE ports: - name: opensearch @@ -69,7 +71,7 @@ spec: subPath: "opensearch" initContainers: - name: opensearch-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/04-dashboards.yml b/kubernetes/04-dashboards.yml index 4817da142..8481d5d6c 100644 --- a/kubernetes/04-dashboards.yml +++ b/kubernetes/04-dashboards.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: dashboards-container - image: ghcr.io/idaholab/malcolm/dashboards:23.12.0 + image: ghcr.io/idaholab/malcolm/dashboards:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/05-upload.yml b/kubernetes/05-upload.yml index c2152f0d9..44b281c0a 100644 --- a/kubernetes/05-upload.yml +++ b/kubernetes/05-upload.yml @@ -34,7 +34,7 @@ spec: spec: containers: - name: upload-container - image: ghcr.io/idaholab/malcolm/file-upload:23.12.0 + image: ghcr.io/idaholab/malcolm/file-upload:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -73,7 +73,7 @@ spec: subPath: "upload" initContainers: - name: upload-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/06-pcap-monitor.yml b/kubernetes/06-pcap-monitor.yml index 9a2287e41..fb7a356c5 100644 --- a/kubernetes/06-pcap-monitor.yml +++ b/kubernetes/06-pcap-monitor.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: pcap-monitor-container - image: ghcr.io/idaholab/malcolm/pcap-monitor:23.12.0 + image: ghcr.io/idaholab/malcolm/pcap-monitor:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -70,7 +70,7 @@ spec: name: pcap-monitor-zeek-volume initContainers: - name: pcap-monitor-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/07-arkime.yml b/kubernetes/07-arkime.yml index 863c46f73..70cdf1e56 100644 --- a/kubernetes/07-arkime.yml +++ b/kubernetes/07-arkime.yml @@ -30,15 +30,10 @@ spec: spec: containers: - name: arkime-container - image: ghcr.io/idaholab/malcolm/arkime:23.12.0 + image: ghcr.io/idaholab/malcolm/arkime:23.12.1 imagePullPolicy: Always stdin: false tty: true - securityContext: - capabilities: - add: - - IPC_LOCK - - SYS_RESOURCE ports: - name: http protocol: TCP @@ -56,6 +51,8 @@ spec: name: upload-common-env - configMapRef: name: arkime-env + - configMapRef: + name: arkime-offline-env - secretRef: name: arkime-secret-env env: @@ -76,14 +73,13 @@ spec: name: arkime-var-local-catrust-volume - mountPath: /var/local/curlrc/secretmap name: arkime-opensearch-curlrc-secret-volume + - mountPath: "/opt/arkime/rules/configmap" + name: arkime-rules-volume - mountPath: "/data/pcap" name: arkime-pcap-volume - - name: arkime-runtime-logs-volume - mountPath: /opt/arkime/logs - subPath: "arkime" initContainers: - name: arkime-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -92,12 +88,10 @@ spec: name: process-env env: - name: PUSER_MKDIR - value: "/data/pcap:processed;/data/runtime-logs:arkime" + value: "/data/pcap:processed" volumeMounts: - name: arkime-pcap-volume mountPath: "/data/pcap" - - name: arkime-runtime-logs-volume - mountPath: "/data/runtime-logs" volumes: - name: arkime-var-local-catrust-volume configMap: @@ -105,9 +99,9 @@ spec: - name: arkime-opensearch-curlrc-secret-volume secret: secretName: opensearch-curlrc + - name: arkime-rules-volume + configMap: + name: arkime-rules - name: arkime-pcap-volume persistentVolumeClaim: claimName: pcap-claim - - name: arkime-runtime-logs-volume - persistentVolumeClaim: - claimName: runtime-logs-claim \ No newline at end of file diff --git a/kubernetes/08-api.yml b/kubernetes/08-api.yml index c158439fc..34b6a62a3 100644 --- a/kubernetes/08-api.yml +++ b/kubernetes/08-api.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: api-container - image: ghcr.io/idaholab/malcolm/api:23.12.0 + image: ghcr.io/idaholab/malcolm/api:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/09-dashboards-helper.yml b/kubernetes/09-dashboards-helper.yml index a210c1b8d..35ecffa08 100644 --- a/kubernetes/09-dashboards-helper.yml +++ b/kubernetes/09-dashboards-helper.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: dashboards-helper-container - image: ghcr.io/idaholab/malcolm/dashboards-helper:23.12.0 + image: ghcr.io/idaholab/malcolm/dashboards-helper:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/10-zeek.yml b/kubernetes/10-zeek.yml index 3e5c25046..60041dd52 100644 --- a/kubernetes/10-zeek.yml +++ b/kubernetes/10-zeek.yml @@ -16,18 +16,10 @@ spec: spec: containers: - name: zeek-offline-container - image: ghcr.io/idaholab/malcolm/zeek:23.12.0 + image: ghcr.io/idaholab/malcolm/zeek:23.12.1 imagePullPolicy: Always stdin: false tty: true - securityContext: - capabilities: - add: - - IPC_LOCK - - NET_ADMIN - - NET_RAW - - SYS_ADMIN - - SYS_RESOURCE envFrom: - configMapRef: name: process-env @@ -63,16 +55,16 @@ spec: - mountPath: "/zeek/upload" name: zeek-offline-zeek-volume subPath: "upload" - - mountPath: "/opt/zeek/share/zeek/site/custom" + - mountPath: "/opt/zeek/share/zeek/site/custom/configmap" name: zeek-offline-custom-volume - - mountPath: "/opt/zeek/share/zeek/site/intel-preseed" + - mountPath: "/opt/zeek/share/zeek/site/intel-preseed/configmap" name: zeek-offline-intel-preseed-volume - mountPath: "/opt/zeek/share/zeek/site/intel" name: zeek-offline-intel-volume subPath: "zeek/intel" initContainers: - name: zeek-offline-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/11-suricata.yml b/kubernetes/11-suricata.yml index b7ed63050..41c126f2e 100644 --- a/kubernetes/11-suricata.yml +++ b/kubernetes/11-suricata.yml @@ -16,18 +16,10 @@ spec: spec: containers: - name: suricata-offline-container - image: ghcr.io/idaholab/malcolm/suricata:23.12.0 + image: ghcr.io/idaholab/malcolm/suricata:23.12.1 imagePullPolicy: Always stdin: false tty: true - securityContext: - capabilities: - add: - - IPC_LOCK - - NET_ADMIN - - NET_RAW - - SYS_ADMIN - - SYS_RESOURCE envFrom: - configMapRef: name: process-env @@ -63,7 +55,7 @@ spec: name: suricata-offline-custom-configs-volume initContainers: - name: suricata-offline-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/12-file-monitor.yml b/kubernetes/12-file-monitor.yml index a8e9afd94..8f8140a47 100644 --- a/kubernetes/12-file-monitor.yml +++ b/kubernetes/12-file-monitor.yml @@ -33,7 +33,7 @@ spec: spec: containers: - name: file-monitor-container - image: ghcr.io/idaholab/malcolm/file-monitor:23.12.0 + image: ghcr.io/idaholab/malcolm/file-monitor:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -81,7 +81,7 @@ spec: name: file-monitor-yara-rules-custom-volume initContainers: - name: file-monitor-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/13-filebeat.yml b/kubernetes/13-filebeat.yml index c38697906..9681fbb7b 100644 --- a/kubernetes/13-filebeat.yml +++ b/kubernetes/13-filebeat.yml @@ -33,7 +33,7 @@ spec: spec: containers: - name: filebeat-container - image: ghcr.io/idaholab/malcolm/filebeat-oss:23.12.0 + image: ghcr.io/idaholab/malcolm/filebeat-oss:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -83,7 +83,7 @@ spec: subPath: "nginx" initContainers: - name: filebeat-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/14-logstash.yml b/kubernetes/14-logstash.yml index 4b2d03bb7..6f6ee2b2a 100644 --- a/kubernetes/14-logstash.yml +++ b/kubernetes/14-logstash.yml @@ -49,14 +49,16 @@ spec: # topologyKey: "kubernetes.io/hostname" containers: - name: logstash-container - image: ghcr.io/idaholab/malcolm/logstash-oss:23.12.0 + image: ghcr.io/idaholab/malcolm/logstash-oss:23.12.1 imagePullPolicy: Always stdin: false tty: true securityContext: capabilities: add: + # IPC_LOCK - to lock memory, preventing swapping - IPC_LOCK + # SYS_RESOURCE - for increasing memlock limits - SYS_RESOURCE ports: - name: lumberjack @@ -113,7 +115,7 @@ spec: subPath: "logstash" initContainers: - name: logstash-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/15-netbox-redis.yml b/kubernetes/15-netbox-redis.yml index 6fc358ecc..6062e0098 100644 --- a/kubernetes/15-netbox-redis.yml +++ b/kubernetes/15-netbox-redis.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-redis-container - image: ghcr.io/idaholab/malcolm/redis:23.12.0 + image: ghcr.io/idaholab/malcolm/redis:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -83,7 +83,7 @@ spec: subPath: netbox/redis initContainers: - name: netbox-redis-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/16-netbox-redis-cache.yml b/kubernetes/16-netbox-redis-cache.yml index d8c7dc2f5..091e49ada 100644 --- a/kubernetes/16-netbox-redis-cache.yml +++ b/kubernetes/16-netbox-redis-cache.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-redis-cache-container - image: ghcr.io/idaholab/malcolm/redis:23.12.0 + image: ghcr.io/idaholab/malcolm/redis:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/17-netbox-postgres.yml b/kubernetes/17-netbox-postgres.yml index 8bd333ede..2d04687ba 100644 --- a/kubernetes/17-netbox-postgres.yml +++ b/kubernetes/17-netbox-postgres.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-postgres-container - image: ghcr.io/idaholab/malcolm/postgresql:23.12.0 + image: ghcr.io/idaholab/malcolm/postgresql:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -74,7 +74,7 @@ spec: subPath: netbox/postgres initContainers: - name: netbox-postgres-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/18-netbox.yml b/kubernetes/18-netbox.yml index 8ca9d1fde..83f9db69b 100644 --- a/kubernetes/18-netbox.yml +++ b/kubernetes/18-netbox.yml @@ -36,7 +36,7 @@ spec: spec: containers: - name: netbox-container - image: ghcr.io/idaholab/malcolm/netbox:23.12.0 + image: ghcr.io/idaholab/malcolm/netbox:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -88,7 +88,7 @@ spec: subPath: netbox/media initContainers: - name: netbox-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/19-htadmin.yml b/kubernetes/19-htadmin.yml index d402c9e1b..c4a9b3ebb 100644 --- a/kubernetes/19-htadmin.yml +++ b/kubernetes/19-htadmin.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: htadmin-container - image: ghcr.io/idaholab/malcolm/htadmin:23.12.0 + image: ghcr.io/idaholab/malcolm/htadmin:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -63,7 +63,7 @@ spec: subPath: "htadmin" initContainers: - name: htadmin-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/20-pcap-capture.yml b/kubernetes/20-pcap-capture.yml index d82fd6274..66014d9c8 100644 --- a/kubernetes/20-pcap-capture.yml +++ b/kubernetes/20-pcap-capture.yml @@ -16,18 +16,22 @@ spec: spec: containers: - name: pcap-capture-container - image: ghcr.io/idaholab/malcolm/pcap-capture:23.12.0 + image: ghcr.io/idaholab/malcolm/pcap-capture:23.12.1 imagePullPolicy: Always stdin: false tty: true securityContext: capabilities: add: + # IPC_LOCK required for some of the memory optimizations netsniff-ng does - IPC_LOCK + # SYS_RESOURCE - for increasing memlock limits + - SYS_RESOURCE + # SYS_ADMIN - for netsniff-ng to set the disc I/O scheduler policy + - SYS_ADMIN + # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW - - SYS_ADMIN - - SYS_RESOURCE envFrom: - configMapRef: name: process-env @@ -46,7 +50,7 @@ spec: subPath: "upload" initContainers: - name: pcap-capture-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/21-zeek-live.yml b/kubernetes/21-zeek-live.yml index 725a21b10..c63a9acb8 100644 --- a/kubernetes/21-zeek-live.yml +++ b/kubernetes/21-zeek-live.yml @@ -16,18 +16,18 @@ spec: spec: containers: - name: zeek-live-container - image: ghcr.io/idaholab/malcolm/zeek:23.12.0 + image: ghcr.io/idaholab/malcolm/zeek:23.12.1 imagePullPolicy: Always stdin: false tty: true securityContext: capabilities: add: - - IPC_LOCK + # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW - - SYS_ADMIN - - SYS_RESOURCE + # SYS_NICE - to set process nice values, real-time scheduling policies, I/O scheduling + - SYS_NICE envFrom: - configMapRef: name: process-env @@ -43,9 +43,6 @@ spec: name: zeek-live-env - configMapRef: name: pcap-capture-env - env: - - name: ZEEK_DISABLED - value: "true" volumeMounts: - mountPath: /var/local/ca-trust/configmap name: zeek-live-var-local-catrust-volume @@ -55,16 +52,16 @@ spec: - mountPath: "/zeek/upload" name: zeek-live-zeek-volume subPath: "upload" - - mountPath: "/opt/zeek/share/zeek/site/custom" + - mountPath: "/opt/zeek/share/zeek/site/custom/configmap" name: zeek-live-custom-volume - - mountPath: "/opt/zeek/share/zeek/site/intel-preseed" + - mountPath: "/opt/zeek/share/zeek/site/intel-preseed/configmap" name: zeek-live-intel-preseed-volume - mountPath: "/opt/zeek/share/zeek/site/intel" name: zeek-live-intel-volume subPath: "zeek/intel" initContainers: - name: zeek-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/22-suricata-live.yml b/kubernetes/22-suricata-live.yml index 19e5763c8..1b373130f 100644 --- a/kubernetes/22-suricata-live.yml +++ b/kubernetes/22-suricata-live.yml @@ -16,18 +16,22 @@ spec: spec: containers: - name: suricata-live-container - image: ghcr.io/idaholab/malcolm/suricata:23.12.0 + image: ghcr.io/idaholab/malcolm/suricata:23.12.1 imagePullPolicy: Always stdin: false tty: true securityContext: capabilities: add: + # IPC_LOCK - to lock memory, preventing swapping - IPC_LOCK + # SYS_RESOURCE - for increasing memlock limits + - SYS_RESOURCE + # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets - NET_ADMIN - NET_RAW - - SYS_ADMIN - - SYS_RESOURCE + # SYS_NICE - to set process nice values, real-time scheduling policies, I/O scheduling + - SYS_NICE envFrom: - configMapRef: name: process-env @@ -41,9 +45,6 @@ spec: name: suricata-live-env - configMapRef: name: pcap-capture-env - env: - - name: SURICATA_DISABLED - value: "true" volumeMounts: - mountPath: /var/local/ca-trust/configmap name: suricata-live-var-local-catrust-volume @@ -55,7 +56,7 @@ spec: name: suricata-live-custom-configs-volume initContainers: - name: suricata-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/23-arkime-live.yml b/kubernetes/23-arkime-live.yml new file mode 100644 index 000000000..41f508378 --- /dev/null +++ b/kubernetes/23-arkime-live.yml @@ -0,0 +1,90 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: arkime-live-deployment + namespace: malcolm +spec: + selector: + matchLabels: + name: arkime-live-deployment + replicas: 1 + template: + metadata: + labels: + name: arkime-live-deployment + spec: + containers: + - name: arkime-live-container + image: ghcr.io/idaholab/malcolm/arkime:23.12.1 + imagePullPolicy: Always + stdin: false + tty: true + securityContext: + capabilities: + add: + # IPC_LOCK - to lock memory, preventing swapping + - IPC_LOCK + # SYS_RESOURCE - for increasing memlock limits + - SYS_RESOURCE + # NET_ADMIN and NET_RAW - to turn on promiscuous mode and capture raw packets + - NET_ADMIN + - NET_RAW + # SYS_NICE - to set process nice values, real-time scheduling policies, I/O scheduling + - SYS_NICE + envFrom: + - configMapRef: + name: process-env + - configMapRef: + name: ssl-env + - configMapRef: + name: opensearch-env + - secretRef: + name: auth-env + - configMapRef: + name: upload-common-env + - configMapRef: + name: pcap-capture-env + - configMapRef: + name: arkime-env + - configMapRef: + name: arkime-live-env + - secretRef: + name: arkime-secret-env + volumeMounts: + - mountPath: /var/local/ca-trust/configmap + name: arkime-live-var-local-catrust-volume + - mountPath: /var/local/curlrc/secretmap + name: arkime-live-opensearch-curlrc-secret-volume + - mountPath: "/opt/arkime/rules/configmap" + name: arkime-live-rules-volume + - mountPath: "/data/pcap" + name: arkime-live-pcap-volume + initContainers: + - name: arkime-live-dirinit-container + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 + imagePullPolicy: Always + stdin: false + tty: true + envFrom: + - configMapRef: + name: process-env + env: + - name: PUSER_MKDIR + value: "/data/pcap:arkime-live" + volumeMounts: + - name: arkime-live-pcap-volume + mountPath: "/data/pcap" + volumes: + - name: arkime-live-var-local-catrust-volume + configMap: + name: var-local-catrust + - name: arkime-live-opensearch-curlrc-secret-volume + secret: + secretName: opensearch-curlrc + - name: arkime-live-rules-volume + configMap: + name: arkime-rules + - name: arkime-live-pcap-volume + persistentVolumeClaim: + claimName: pcap-claim diff --git a/kubernetes/23-freq.yml b/kubernetes/24-freq.yml similarity index 96% rename from kubernetes/23-freq.yml rename to kubernetes/24-freq.yml index 5173b8d2a..32e834237 100644 --- a/kubernetes/23-freq.yml +++ b/kubernetes/24-freq.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: freq-container - image: ghcr.io/idaholab/malcolm/freq:23.12.0 + image: ghcr.io/idaholab/malcolm/freq:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/98-nginx-proxy.yml b/kubernetes/98-nginx-proxy.yml index 1f293bd64..c8e208b56 100644 --- a/kubernetes/98-nginx-proxy.yml +++ b/kubernetes/98-nginx-proxy.yml @@ -39,7 +39,7 @@ spec: spec: containers: - name: nginx-proxy-container - image: ghcr.io/idaholab/malcolm/nginx-proxy:23.12.0 + image: ghcr.io/idaholab/malcolm/nginx-proxy:23.12.1 imagePullPolicy: Always stdin: false tty: true @@ -95,7 +95,7 @@ spec: subPath: "nginx" initContainers: - name: nginx-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:23.12.0 + image: ghcr.io/idaholab/malcolm/dirinit:23.12.1 imagePullPolicy: Always stdin: false tty: true diff --git a/logstash/pipelines/beats/11_beats_logs.conf b/logstash/pipelines/beats/11_beats_logs.conf index e4cd573f9..e991be9f8 100644 --- a/logstash/pipelines/beats/11_beats_logs.conf +++ b/logstash/pipelines/beats/11_beats_logs.conf @@ -2,7 +2,7 @@ # Malcolm and Hedgehog Linux itself (i.e., not captured # network traffic metadata, but operational metadata) # -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ####################### filter { @@ -545,6 +545,101 @@ filter { } # if ([miscbeat][syslog]) + if ([miscbeat][systemd]) { + #------------------------------------------------- + # systemd - https://docs.fluentbit.io/manual/pipeline/inputs/systemd + # see also - https://www.freedesktop.org/software/systemd/man/latest/systemd.journal-fields.html + + if ([miscbeat][systemd][source_realtime_timestamp]) { + # microseconds to milliseconds + ruby { + id => "ruby_miscbeat_systemd_source_realtime_timestamp_calc" + code => " + event.set('[miscbeat][systemd][source_realtime_timestamp]', + (event.get('[miscbeat][systemd][source_realtime_timestamp]').to_i / 1000)) + " + } + # milliseconds to date + date { + id => "date_beats_miscbeat_systemd_source_realtime_timestamp" + match => [ "[miscbeat][systemd][source_realtime_timestamp]", "UNIX_MS" ] + target => "[miscbeat][systemd][source_realtime_timestamp]" + } + } + + # rename fields according to ECS + # https://www.elastic.co/guide/en/ecs/current/ecs-process.html + # https://www.elastic.co/guide/en/ecs/current/ecs-log.html + mutate { id => "mutate_rename_miscbeat_systemd_log_fields" + # todo: translate this to CAP_BPF, CAP_SYS_ADMIN, etc. + rename => { "[miscbeat][systemd][cap_effective]" => "[process][thread][capabilities][effective]" } + rename => { "[miscbeat][systemd][cmdline]" => "[process][command_line]" } + rename => { "[miscbeat][systemd][code_file]" => "[log][origin][file][name]" } + rename => { "[miscbeat][systemd][code_func]" => "[log][origin][function]" } + rename => { "[miscbeat][systemd][code_line]" => "[log][origin][file][line]" } + rename => { "[miscbeat][systemd][comm]" => "[process][name]" } + rename => { "[miscbeat][systemd][exe]" => "[process][executable]" } + rename => { "[miscbeat][systemd][gid]" => "[process][group_leader][pid]" } + rename => { "[miscbeat][systemd][pid]" => "[process][pid]" } + rename => { "[miscbeat][systemd][syslog_facility]" => "[log][syslog][facility][code]" } + rename => { "[miscbeat][systemd][syslog_identifier]" => "[log][syslog][appname]" } + rename => { "[miscbeat][systemd][syslog_pid]" => "[log][syslog][procid]" } + rename => { "[miscbeat][systemd][systemd_owner_uid]" => "[process][real_user][id]" } + rename => { "[miscbeat][systemd][tid]" => "[process][thread][id]" } + rename => { "[miscbeat][systemd][uid]" => "[process][user][id]" } + rename => { "[miscbeat][systemd][transport]" => "[log][logger]" } + } + if ([log][syslog][facility][code]) { + translate { + id => "translate_sensor_systemd_syslog_facility_code" + source => "[log][syslog][facility][code]" + target => "[log][syslog][facility][name]" + dictionary_path => "/etc/syslog_facility_codes.yaml" + } + } + + # store raw message text as event.original + if ([miscbeat][systemd][syslog_raw]) { + mutate { id => "mutate_rename_miscbeat_systemd_syslog_raw" + rename => { "[miscbeat][systemd][syslog_raw]" => "[event][original]" } } + } else if ([miscbeat][systemd][message]) { + mutate { id => "mutate_rename_miscbeat_systemd_message" + rename => { "[miscbeat][systemd][message]" => "[event][original]" } } + } + + # + if ([miscbeat][systemd][errno]) { + mutate { id => "mutate_rename_miscbeat_systemd_errno" + rename => { "[miscbeat][systemd][errno]" => "[event][result]" } } + } + + if (![event][hash]) { + fingerprint { + id => "fingerprint_malcolm_miscbeat_systemd" + source => [ "[host][name]", + "[event][module]", + "[process][pid]", + "[process][user][id]", + "[process][thread][id]", + "[process][group_leader][pid]", + "[miscbeat][systemd][unit]", + "[miscbeat][systemd][comm]", + "[miscbeat][systemd][exe]", + "[miscbeat][systemd][cmdline]", + "[miscbeat][systemd][machine_id]", + "[event][original]", + "[miscbeat][systemd][message_id]", + "[@timestamp]" ] + concatenate_sources => true + # uses event.hash + ecs_compatibility => "v8" + method => "MURMUR3_128" + base64encode => true + } + } + + } # if ([miscbeat][systemd]) + if ([miscbeat][thermal]) { #------------------------------------------------- # thermal - https://docs.fluentbit.io/manual/pipeline/inputs/thermal diff --git a/logstash/pipelines/enrichment/20_enriched_to_ecs.conf b/logstash/pipelines/enrichment/20_enriched_to_ecs.conf index 24552d683..5a46456a6 100644 --- a/logstash/pipelines/enrichment/20_enriched_to_ecs.conf +++ b/logstash/pipelines/enrichment/20_enriched_to_ecs.conf @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. filter { diff --git a/logstash/pipelines/enrichment/23_severity.conf b/logstash/pipelines/enrichment/23_severity.conf index 2f794df35..6f8506c32 100644 --- a/logstash/pipelines/enrichment/23_severity.conf +++ b/logstash/pipelines/enrichment/23_severity.conf @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. filter { diff --git a/logstash/pipelines/enrichment/96_make_unique.conf b/logstash/pipelines/enrichment/96_make_unique.conf index 22848bd4c..913f54514 100644 --- a/logstash/pipelines/enrichment/96_make_unique.conf +++ b/logstash/pipelines/enrichment/96_make_unique.conf @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # take array fields that are already generic (i.e., ECS or Arkime) and deduplicate them. # there is also a little bit of light normalization that happens here diff --git a/logstash/pipelines/enrichment/97_arkimize.conf b/logstash/pipelines/enrichment/97_arkimize.conf index e5a88aa8c..384f9a455 100644 --- a/logstash/pipelines/enrichment/97_arkimize.conf +++ b/logstash/pipelines/enrichment/97_arkimize.conf @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # take fields that are already generic (i.e., ECS or whatever) and copy them # to their Arkime equivalents if applicable diff --git a/logstash/pipelines/enrichment/98_finalize.conf b/logstash/pipelines/enrichment/98_finalize.conf index d9a64f309..f292b9202 100644 --- a/logstash/pipelines/enrichment/98_finalize.conf +++ b/logstash/pipelines/enrichment/98_finalize.conf @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # final adjustments before forwarding diff --git a/logstash/pipelines/suricata/19_severity.conf b/logstash/pipelines/suricata/19_severity.conf index 5c3043642..19d8db1eb 100644 --- a/logstash/pipelines/suricata/19_severity.conf +++ b/logstash/pipelines/suricata/19_severity.conf @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. filter { diff --git a/logstash/pipelines/zeek/10_zeek_prep.conf b/logstash/pipelines/zeek/10_zeek_prep.conf index 48c5a2d1e..6e0785a35 100644 --- a/logstash/pipelines/zeek/10_zeek_prep.conf +++ b/logstash/pipelines/zeek/10_zeek_prep.conf @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. filter { diff --git a/logstash/pipelines/zeek/11_zeek_parse.conf b/logstash/pipelines/zeek/11_zeek_parse.conf index f6809839d..249450c3d 100644 --- a/logstash/pipelines/zeek/11_zeek_parse.conf +++ b/logstash/pipelines/zeek/11_zeek_parse.conf @@ -10,7 +10,7 @@ # - get filters where in != out # $ docker compose exec logstash curl -XGET http://localhost:9600/_node/stats/pipelines | jq -r '.. | .filters? // empty | .[] | objects | select (.events.in != .events.out) | [.id, .events.in, .events.out, .events.duration_in_millis] | join (";")' # -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ####################### filter { diff --git a/logstash/pipelines/zeek/12_zeek_mutate.conf b/logstash/pipelines/zeek/12_zeek_mutate.conf index 83ed11143..9eebdebc8 100644 --- a/logstash/pipelines/zeek/12_zeek_mutate.conf +++ b/logstash/pipelines/zeek/12_zeek_mutate.conf @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. filter { diff --git a/logstash/pipelines/zeek/13_zeek_normalize.conf b/logstash/pipelines/zeek/13_zeek_normalize.conf index c4e74423b..48d97d88d 100644 --- a/logstash/pipelines/zeek/13_zeek_normalize.conf +++ b/logstash/pipelines/zeek/13_zeek_normalize.conf @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. filter { diff --git a/logstash/pipelines/zeek/14_zeek_convert.conf b/logstash/pipelines/zeek/14_zeek_convert.conf index 51e9b2dfb..3103b0d62 100644 --- a/logstash/pipelines/zeek/14_zeek_convert.conf +++ b/logstash/pipelines/zeek/14_zeek_convert.conf @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. filter { diff --git a/logstash/pipelines/zeek/19_severity.conf b/logstash/pipelines/zeek/19_severity.conf index 5851cf21e..aedf410ae 100644 --- a/logstash/pipelines/zeek/19_severity.conf +++ b/logstash/pipelines/zeek/19_severity.conf @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. filter { diff --git a/logstash/scripts/logstash-start.sh b/logstash/scripts/logstash-start.sh index be558a9d8..35df19c66 100755 --- a/logstash/scripts/logstash-start.sh +++ b/logstash/scripts/logstash-start.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. set -e diff --git a/logstash/supervisord.conf b/logstash/supervisord.conf index f9734ba3f..a9b91b0bf 100644 --- a/logstash/supervisord.conf +++ b/logstash/supervisord.conf @@ -1,4 +1,4 @@ -; Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +; Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. [inet_http_server] port=0.0.0.0:9001 diff --git a/malcolm-iso/Dockerfile b/malcolm-iso/Dockerfile index 7790c54a8..e371b85b0 100644 --- a/malcolm-iso/Dockerfile +++ b/malcolm-iso/Dockerfile @@ -1,6 +1,6 @@ FROM ghcr.io/mmguero/qemu-live-iso:latest -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' diff --git a/malcolm-iso/build.sh b/malcolm-iso/build.sh index 2f7fe76c4..feb2a0f99 100755 --- a/malcolm-iso/build.sh +++ b/malcolm-iso/build.sh @@ -92,8 +92,7 @@ if [ -d "$WORKDIR" ]; then # grab things from the Malcolm parent directory into /etc/skel so the user's got it set up in their home/Malcolm dir pushd "$SCRIPT_PATH/.." >/dev/null 2>&1 MALCOLM_DEST_DIR="$WORKDIR/work/$IMAGE_NAME-Live-Build/config/includes.chroot/etc/skel/Malcolm" - mkdir -p "$MALCOLM_DEST_DIR/arkime-logs/" - mkdir -p "$MALCOLM_DEST_DIR/arkime-raw/" + mkdir -p "$MALCOLM_DEST_DIR/arkime/rules/" mkdir -p "$MALCOLM_DEST_DIR/config/" mkdir -p "$MALCOLM_DEST_DIR/filebeat/certs/" mkdir -p "$MALCOLM_DEST_DIR/htadmin/" @@ -108,6 +107,7 @@ if [ -d "$WORKDIR" ]; then mkdir -p "$MALCOLM_DEST_DIR/kubernetes/" mkdir -p "$MALCOLM_DEST_DIR/opensearch-backup/" mkdir -p "$MALCOLM_DEST_DIR/opensearch/nodes/" + mkdir -p "$MALCOLM_DEST_DIR/pcap/arkime-live/" mkdir -p "$MALCOLM_DEST_DIR/pcap/processed/" mkdir -p "$MALCOLM_DEST_DIR/pcap/upload/tmp/spool/" mkdir -p "$MALCOLM_DEST_DIR/pcap/upload/variants/" @@ -147,6 +147,7 @@ if [ -d "$WORKDIR" ]; then cp ./scripts/malcolm_utils.py "$MALCOLM_DEST_DIR/scripts/" cp ./kubernetes/*.* "$MALCOLM_DEST_DIR/kubernetes/" grep -v "^#" ./kubernetes/.gitignore | xargs -r -I XXX rm -f "$MALCOLM_DEST_DIR/kubernetes/XXX" + cp ./arkime/rules/*.yml "$MALCOLM_DEST_DIR/arkime/rules/" cp ./logstash/certs/*.conf "$MALCOLM_DEST_DIR/logstash/certs/" cp ./logstash/maps/malcolm_severity.yaml "$MALCOLM_DEST_DIR/logstash/maps/" cp -r ./netbox/config/ "$MALCOLM_DEST_DIR/netbox/" diff --git a/malcolm-iso/config/hooks/normal/0168-firefox-install.hook.chroot b/malcolm-iso/config/hooks/normal/0168-firefox-install.hook.chroot index 18c4941ab..98b7a4782 100755 --- a/malcolm-iso/config/hooks/normal/0168-firefox-install.hook.chroot +++ b/malcolm-iso/config/hooks/normal/0168-firefox-install.hook.chroot @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. export LC_ALL=C.UTF-8 export LANG=C.UTF-8 diff --git a/malcolm-iso/config/hooks/normal/0911-get-stig-scripts.hook.chroot b/malcolm-iso/config/hooks/normal/0911-get-stig-scripts.hook.chroot index 32dceebc4..4ba6b95fb 100755 --- a/malcolm-iso/config/hooks/normal/0911-get-stig-scripts.hook.chroot +++ b/malcolm-iso/config/hooks/normal/0911-get-stig-scripts.hook.chroot @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # clone harbian-audit and clean up some stuff we don't need mkdir -p /opt diff --git a/malcolm-iso/config/hooks/normal/0991-security-performance.hook.chroot b/malcolm-iso/config/hooks/normal/0991-security-performance.hook.chroot index 9d160f065..9c0131952 100755 --- a/malcolm-iso/config/hooks/normal/0991-security-performance.hook.chroot +++ b/malcolm-iso/config/hooks/normal/0991-security-performance.hook.chroot @@ -31,6 +31,7 @@ UFW_ALLOW_RULES=( for i in ${UFW_ALLOW_RULES[@]}; do ufw allow "$i" done +ufw allow proto tcp from 172.29.0.0/16 to any port 8005 # docker (disallow overriding firewall) mkdir -p /etc/docker/ @@ -121,14 +122,6 @@ echo "umask 077" >> /etc/profile echo "export UMASK=077" >> /etc/profile echo "export PYTHONDONTWRITEBYTECODE=1" >> /etc/profile -# enable cron logging -sed -r -i "s@^#(cron\.\*\s+.*/var/log/cron\.log)@\1@" /etc/rsyslog.conf - -# enable rsyslog forwarding to localhost:9514 over UDP (for filebeat syslog input) -echo >> /etc/rsyslog.conf -echo '*.* @127.0.0.1:9514' >> /etc/rsyslog.conf -echo >> /etc/rsyslog.conf - # put sudoers log into its own logfile awk 'FNR==NR{ if (/^Defaults/) p=NR; next} 1; FNR==p{ print "Defaults\t!syslog\nDefaults\tlogfile=/var/log/sudo.log" }' /etc/sudoers /etc/sudoers > /tmp/newsudoers mv /tmp/newsudoers /etc/sudoers && chmod 440 /etc/sudoers diff --git a/malcolm-iso/config/includes.binary/install/preseed_base.cfg b/malcolm-iso/config/includes.binary/install/preseed_base.cfg index 1d18bfce4..f25b47593 100644 --- a/malcolm-iso/config/includes.binary/install/preseed_base.cfg +++ b/malcolm-iso/config/includes.binary/install/preseed_base.cfg @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. d-i hw-detect/load_firmware boolean true d-i clock-setup/utc boolean true diff --git a/malcolm-iso/config/includes.binary/install/preseed_multipar.cfg b/malcolm-iso/config/includes.binary/install/preseed_multipar.cfg index 43624ac41..96d68233c 100644 --- a/malcolm-iso/config/includes.binary/install/preseed_multipar.cfg +++ b/malcolm-iso/config/includes.binary/install/preseed_multipar.cfg @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. d-i debian-installer/locale string en_US.UTF-8 d-i console-setup/ask_detect boolean false diff --git a/malcolm-iso/config/includes.binary/install/preseed_vmware.cfg b/malcolm-iso/config/includes.binary/install/preseed_vmware.cfg index 7d433121d..792d0c7a6 100644 --- a/malcolm-iso/config/includes.binary/install/preseed_vmware.cfg +++ b/malcolm-iso/config/includes.binary/install/preseed_vmware.cfg @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. d-i debian-installer/locale string en_US.UTF-8 d-i console-setup/ask_detect boolean false diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/systemd-localhost-malcolm.service b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/systemd-localhost-malcolm.service new file mode 100644 index 000000000..ed79496b0 --- /dev/null +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/systemd-localhost-malcolm.service @@ -0,0 +1,12 @@ +[Unit] +AssertPathExists=%h/Malcolm/filebeat/certs/client.key +After=network.target + +[Service] +ExecStart=/opt/fluent-bit/bin/fluent-bit -R /etc/fluent-bit/parsers.conf -i systemd -p Read_From_Tail=On -p Lowercase=On -p Strip_Underscores=On -o tcp://localhost:5045 -p tls=on -p tls.verify=off -p tls.ca_file=%h/Malcolm/filebeat/certs/ca.crt -p tls.crt_file=%h/Malcolm/filebeat/certs/client.crt -p tls.key_file=%h/Malcolm/filebeat/certs/client.key -p format=json_lines -F nest -p Operation=nest -p Nested_under=systemd -p WildCard='*' -m '*' -F record_modifier -p 'Record=module systemd' -m '*' -F modify -p 'Add=host.name %H' -m '*' -F nest -p 'Operation=nest' -p 'Wildcard=host.*' -p 'Nest_under=host' -p 'Remove_prefix=host.' -m '*' -f 1 +Restart=on-failure +PrivateTmp=false +NoNewPrivileges=false + +[Install] +WantedBy=default.target diff --git a/malcolm-iso/vagrant/Vagrantfile b/malcolm-iso/vagrant/Vagrantfile index 0558db616..ecc7852a2 100644 --- a/malcolm-iso/vagrant/Vagrantfile +++ b/malcolm-iso/vagrant/Vagrantfile @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. unless Vagrant.has_plugin?("vagrant-sshfs") raise 'vagrant-sshfs plugin is not installed!' diff --git a/netbox/scripts/netbox_init.py b/netbox/scripts/netbox_init.py index ac93af265..874776681 100755 --- a/netbox/scripts/netbox_init.py +++ b/netbox/scripts/netbox_init.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. import argparse import glob diff --git a/netbox/scripts/netbox_library_import.py b/netbox/scripts/netbox_library_import.py index 034ec0306..fa514c738 100644 --- a/netbox/scripts/netbox_library_import.py +++ b/netbox/scripts/netbox_library_import.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # adapted from minitriga/Netbox-Device-Type-Library-Import (MIT License) # Copyright (c) 2021 Alexander Gittings diff --git a/netbox/supervisord.conf b/netbox/supervisord.conf index 1703592bc..4bb797473 100644 --- a/netbox/supervisord.conf +++ b/netbox/supervisord.conf @@ -1,4 +1,4 @@ -; Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +; Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. [inet_http_server] port=0.0.0.0:9001 diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 05a138c45..e6c109bcd 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. daemon off; @@ -145,14 +145,6 @@ http { client_max_body_size 50G; } - # Logstash statistics - location ~* ^/logstash\b(.*) { - include /etc/nginx/nginx_auth_rt.conf; - proxy_pass http://logstash-stats/_node/stats$1; - proxy_redirect off; - proxy_set_header Host arkime.malcolm.local; - } - # Arkime -> Dashboards shortcut location ~* ^/idark2dash(.*) { include /etc/nginx/nginx_auth_rt.conf; @@ -253,6 +245,34 @@ http { proxy_set_header Authorization ""; } + # passthrough Logstash statistics from the Malcolm API + location /mapi/logstash/ { + include /etc/nginx/nginx_auth_rt.conf; + proxy_pass http://logstash-stats/_node/stats/; + proxy_redirect off; + } + + # passthrough OpenSearch from the Malcolm API + location /mapi/opensearch/ { + include /etc/nginx/nginx_auth_rt.conf; + proxy_pass http://opensearch/; + proxy_redirect off; + proxy_set_header Host os.malcolm.local; + client_max_body_size 50m; + } + + # passthrough NetBox from the Malcolm API + location /mapi/netbox/ { + include /etc/nginx/nginx_auth_rt.conf; + proxy_pass http://netbox/netbox/api/; + proxy_redirect off; + proxy_set_header Host netbox.malcolm.local; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Remote-Auth $authenticated_user; + } + # Malcolm API location /mapi { include /etc/nginx/nginx_auth_rt.conf; diff --git a/nginx/nginx_readonly.conf b/nginx/nginx_readonly.conf index 61e6b0012..4bb3705fc 100644 --- a/nginx/nginx_readonly.conf +++ b/nginx/nginx_readonly.conf @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. daemon off; @@ -177,6 +177,19 @@ http { proxy_set_header Authorization ""; } + # passthrough NetBox from the Malcolm API + location /mapi/netbox/ { + limit_except GET { deny all; } + include /etc/nginx/nginx_auth_rt.conf; + proxy_pass http://netbox/netbox/api/; + proxy_redirect off; + proxy_set_header Host netbox.malcolm.local; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Remote-Auth $authenticated_user; + } + # Malcolm API location /mapi { proxy_pass http://api; diff --git a/nginx/supervisord.conf b/nginx/supervisord.conf index a03b1f7e3..4fa4d85ad 100644 --- a/nginx/supervisord.conf +++ b/nginx/supervisord.conf @@ -1,4 +1,4 @@ -; Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +; Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. [unix_http_server] file=/tmp/supervisor.sock ; (the path to the socket file) diff --git a/pcap-capture/scripts/netsniff-roll.sh b/pcap-capture/scripts/netsniff-roll.sh index ca2ab23f9..1fd48530a 100755 --- a/pcap-capture/scripts/netsniff-roll.sh +++ b/pcap-capture/scripts/netsniff-roll.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. lastmod(){ expr $(date +%s) - $(stat -c %X "$1") diff --git a/pcap-capture/scripts/supervisor.sh b/pcap-capture/scripts/supervisor.sh index 46e612c8d..618e775ff 100755 --- a/pcap-capture/scripts/supervisor.sh +++ b/pcap-capture/scripts/supervisor.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. set -e diff --git a/pcap-capture/supervisord.conf b/pcap-capture/supervisord.conf index 07a7b1af2..1ee0eb6a3 100644 --- a/pcap-capture/supervisord.conf +++ b/pcap-capture/supervisord.conf @@ -1,4 +1,4 @@ -; Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +; Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. [unix_http_server] file=/tmp/supervisor.sock ; (the path to the socket file) diff --git a/pcap-monitor/scripts/watch-pcap-uploads-folder.py b/pcap-monitor/scripts/watch-pcap-uploads-folder.py index 2b7847f0e..8945d937e 100755 --- a/pcap-monitor/scripts/watch-pcap-uploads-folder.py +++ b/pcap-monitor/scripts/watch-pcap-uploads-folder.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ################################################################################################### # Monitor a directory for PCAP files for processing (by publishing their filenames to a ZMQ socket) diff --git a/pcap-monitor/supervisord.conf b/pcap-monitor/supervisord.conf index bb3e3bdc1..e1f7b6fc1 100644 --- a/pcap-monitor/supervisord.conf +++ b/pcap-monitor/supervisord.conf @@ -1,4 +1,4 @@ -; Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +; Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. [unix_http_server] diff --git a/arkime-raw/.gitignore b/pcap/arkime-live/.gitignore similarity index 100% rename from arkime-raw/.gitignore rename to pcap/arkime-live/.gitignore diff --git a/scripts/build.sh b/scripts/build.sh index a4ff97c27..36bd18b5c 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. if [ -z "$BASH_VERSION" ]; then echo "Wrong interpreter, please run \"$0\" with bash" diff --git a/scripts/control.py b/scripts/control.py index c6b39d6c9..3b90076a2 100755 --- a/scripts/control.py +++ b/scripts/control.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. import sys @@ -843,15 +843,13 @@ def stop(wipe=False): if wipe: # there is some overlap here among some of these containers, but it doesn't matter boundPathsToWipe = ( - BoundPath("arkime", "/opt/arkime/logs", True, None, None), - BoundPath("arkime", "/opt/arkime/raw", True, None, None), BoundPath("filebeat", "/zeek", True, None, None), BoundPath("file-monitor", "/zeek/logs", True, None, None), BoundPath("netbox", "/opt/netbox/netbox/media", True, None, ["."]), BoundPath("netbox-postgres", "/var/lib/postgresql/data", True, None, ["."]), BoundPath("netbox-redis", "/data", True, None, ["."]), BoundPath("opensearch", "/usr/share/opensearch/data", True, ["nodes"], None), - BoundPath("pcap-monitor", "/pcap", True, ["processed", "upload"], None), + BoundPath("pcap-monitor", "/pcap", True, ["arkime-live", "processed", "upload"], None), BoundPath("suricata", "/var/log/suricata", True, None, ["."]), BoundPath( "upload", @@ -995,8 +993,6 @@ def start(): if orchMode is OrchestrationFramework.DOCKER_COMPOSE: # make sure some directories exist before we start boundPathsToCreate = ( - BoundPath("arkime", "/opt/arkime/logs", False, None, None), - BoundPath("arkime", "/opt/arkime/raw", False, None, None), BoundPath("file-monitor", "/zeek/logs", False, None, None), BoundPath("nginx-proxy", "/var/local/ca-trust", False, None, None), BoundPath("netbox", "/opt/netbox/netbox/media", False, None, None), @@ -1004,7 +1000,7 @@ def start(): BoundPath("netbox-redis", "/data", False, None, None), BoundPath("opensearch", "/usr/share/opensearch/data", False, ["nodes"], None), BoundPath("opensearch", "/opt/opensearch/backup", False, None, None), - BoundPath("pcap-monitor", "/pcap", False, ["processed", "upload"], None), + BoundPath("pcap-monitor", "/pcap", False, ["arkime-live", "processed", "upload"], None), BoundPath("suricata", "/var/log/suricata", False, ["live"], None), BoundPath( "upload", diff --git a/scripts/demo/Vagrantfile b/scripts/demo/Vagrantfile index a4859d27a..6ba20e272 100644 --- a/scripts/demo/Vagrantfile +++ b/scripts/demo/Vagrantfile @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. class VagrantPlugins::ProviderVirtualBox::Action::Network def dhcp_server_matches_config?(dhcp_server, config) diff --git a/scripts/demo/amazon_linux_2_malcolm_demo_setup.sh b/scripts/demo/amazon_linux_2_malcolm_demo_setup.sh index ac074e5c0..41dd9bd90 100755 --- a/scripts/demo/amazon_linux_2_malcolm_demo_setup.sh +++ b/scripts/demo/amazon_linux_2_malcolm_demo_setup.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ################################################################################### # for setting up a Malcolm demo instance on an Amazon Linux 2 instance from scratch diff --git a/scripts/demo/reset_and_auto_populate.sh b/scripts/demo/reset_and_auto_populate.sh index 38dea9057..900b2a43a 100755 --- a/scripts/demo/reset_and_auto_populate.sh +++ b/scripts/demo/reset_and_auto_populate.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ############################################################################### # diff --git a/scripts/documentation_build.sh b/scripts/documentation_build.sh index d9a230878..7510d1b57 100755 --- a/scripts/documentation_build.sh +++ b/scripts/documentation_build.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. if [ -z "$BASH_VERSION" ]; then echo "Wrong interpreter, please run \"$0\" with bash" diff --git a/scripts/install.py b/scripts/install.py index 8408144ed..0b341bcae 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. import sys @@ -235,6 +235,7 @@ def __init__(self, orchMode, debug=False, configOnly=False): self.checkPackageCmds = [] self.installPackageCmds = [] self.requiredPackages = [] + self.dockerComposeCmd = None self.pipCmd = 'pip3' if not which(self.pipCmd, debug=self.debug): @@ -302,15 +303,16 @@ def install_required_packages(self): return self.install_package(self.requiredPackages) # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - def install_docker_images(self, docker_image_file): + def install_docker_images(self, docker_image_file, malcolm_install_path): result = False + composeFile = os.path.join(malcolm_install_path, 'docker-compose.yml') if self.orchMode is OrchestrationFramework.DOCKER_COMPOSE: if ( docker_image_file and os.path.isfile(docker_image_file) and InstallerYesOrNo( - f'Load Malcolm Docker images from {docker_image_file}', default=True, forceInteraction=True + f'Load Malcolm Docker images from {docker_image_file}?', default=True, forceInteraction=True ) ): ecode, out = self.run_process(['docker', 'load', '-q', '-i', docker_image_file], privileged=True) @@ -319,6 +321,31 @@ def install_docker_images(self, docker_image_file): else: eprint(f"Loading Malcolm Docker images failed: {out}") + elif ( + os.path.isfile(composeFile) + and self.dockerComposeCmd + and InstallerYesOrNo(f'Pull Malcolm Docker images?', default=False, forceInteraction=False) + ): + for priv in (False, True): + ecode, out = self.run_process( + [ + self.dockerComposeCmd, + '-f', + composeFile, + '--profile=malcolm', + 'pull', + '--quiet', + ], + privileged=priv, + ) + if ecode == 0: + break + + if ecode == 0: + result = True + else: + eprint(f"Pulling Malcolm Docker images failed: {out}") + return result # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -331,7 +358,7 @@ def install_malcolm_files(self, malcolm_install_file, default_config_dir): malcolm_install_file and os.path.isfile(malcolm_install_file) and InstallerYesOrNo( - f'Extract Malcolm runtime files from {malcolm_install_file}', default=True, forceInteraction=True + f'Extract Malcolm runtime files from {malcolm_install_file}?', default=True, forceInteraction=True ) ): # determine and create destination path for installation @@ -434,8 +461,10 @@ def tweak_malcolm_runtime(self, malcolm_install_path): defaultUid = '1000' defaultGid = '1000' if ((self.platform == PLATFORM_LINUX) or (self.platform == PLATFORM_MAC)) and (self.scriptUser == "root"): - defaultUid = str(os.stat(malcolm_install_path).st_uid) - defaultGid = str(os.stat(malcolm_install_path).st_gid) + if pathUid := os.stat(malcolm_install_path).st_uid: + defaultUid = str(pathUid) + if pathGid := os.stat(malcolm_install_path).st_gid: + defaultGid = str(pathGid) puid = defaultUid pgid = defaultGid @@ -470,7 +499,6 @@ def tweak_malcolm_runtime(self, malcolm_install_path): f'Enter the node name to associate with network traffic metadata', default=args.pcapNodeName, ) - pcapNodeHost = '' if self.orchMode is OrchestrationFramework.DOCKER_COMPOSE: # guestimate how much memory we should use based on total system memory @@ -626,18 +654,6 @@ def tweak_malcolm_runtime(self, malcolm_install_path): f'Enter Logstash host and port (e.g., 192.168.1.123:5044)', default=args.logstashHost, ) - pcapNodeHost = InstallerAskForString( - f"Enter this node's hostname or IP to associate with network traffic metadata", - default=args.pcapNodeHost, - ) - if not pcapNodeHost and not InstallerYesOrNo( - f'Node hostname or IP is required for Arkime session retrieval under the {malcolmProfile} profile. Are you sure?', - default=False, - ): - pcapNodeHost = InstallerAskForString( - f"Enter this node's hostname or IP to associate with network traffic metadata", - default=args.pcapNodeHost, - ) if (malcolmProfile == PROFILE_MALCOLM) and InstallerYesOrNo( 'Forward Logstash logs to a secondary remote document store?', @@ -963,6 +979,7 @@ def tweak_malcolm_runtime(self, malcolm_install_path): for pathToCreate in ( indexDirFull, indexSnapshotDirFull, + os.path.join(pcapDirFull, 'arkime-live'), os.path.join(pcapDirFull, 'processed'), os.path.join(pcapDirFull, os.path.join('upload', os.path.join('tmp', 'spool'))), os.path.join(pcapDirFull, os.path.join('upload', 'variants')), @@ -1029,7 +1046,7 @@ def tweak_malcolm_runtime(self, malcolm_install_path): (opensearchPrimaryMode != DatabaseMode.OpenSearchLocal) or (malcolmProfile != PROFILE_MALCOLM) or InstallerYesOrNo( - 'Should Arkime delete PCAP files based on available storage (see https://arkime.com/faq#pcap-deletion)?', + 'Should Arkime delete uploaded PCAP files based on available storage (see https://arkime.com/faq#pcap-deletion)?', default=args.arkimeManagePCAP, ) ) @@ -1044,6 +1061,7 @@ def tweak_malcolm_runtime(self, malcolm_install_path): if arkimeFreeSpaceGTmp: arkimeFreeSpaceG = arkimeFreeSpaceGTmp + autoArkime = InstallerYesOrNo('Automatically analyze all PCAP files with Arkime?', default=args.autoArkime) autoSuricata = InstallerYesOrNo( 'Automatically analyze all PCAP files with Suricata?', default=args.autoSuricata ) @@ -1052,14 +1070,14 @@ def tweak_malcolm_runtime(self, malcolm_install_path): ) autoZeek = InstallerYesOrNo('Automatically analyze all PCAP files with Zeek?', default=args.autoZeek) - zeekIcs = InstallerYesOrNo( + malcolmIcs = InstallerYesOrNo( 'Is Malcolm being used to monitor an Operational Technology/Industrial Control Systems (OT/ICS) network?', - default=args.zeekIcs, + default=args.malcolmIcs, ) zeekICSBestGuess = ( autoZeek - and zeekIcs + and malcolmIcs and InstallerYesOrNo( 'Should Malcolm use "best guess" to identify potential OT/ICS traffic with Zeek?', default=args.zeekICSBestGuess, @@ -1084,7 +1102,6 @@ def tweak_malcolm_runtime(self, malcolm_install_path): ) if self.orchMode is OrchestrationFramework.DOCKER_COMPOSE: if malcolmProfile == PROFILE_MALCOLM: - arkimeViewerOpen = False openPortsOptions = ('no', 'yes', 'customize') loopBreaker = CountUntilException(MaxAskForValueCount) while openPortsSelection not in [x[0] for x in openPortsOptions] and loopBreaker.increment(): @@ -1097,7 +1114,7 @@ def tweak_malcolm_runtime(self, malcolm_install_path): logstashOpen = False filebeatTcpOpen = False elif openPortsSelection == 'y': - opensearchOpen = True + opensearchOpen = opensearchPrimaryMode == DatabaseMode.OpenSearchLocal logstashOpen = True filebeatTcpOpen = True else: @@ -1116,16 +1133,12 @@ def tweak_malcolm_runtime(self, malcolm_install_path): openPortsSelection = 'n' logstashOpen = False filebeatTcpOpen = False - arkimeViewerOpen = InstallerYesOrNo( - 'Expose Arkime viewer to external hosts for PCAP payload retrieval?', - default=args.exposeArkimeViewer, - ) + else: opensearchOpen = opensearchPrimaryMode == DatabaseMode.OpenSearchLocal openPortsSelection = 'y' logstashOpen = True filebeatTcpOpen = True - arkimeViewerOpen = malcolmProfile == PROFILE_HEDGEHOG filebeatTcpFormat = 'json' filebeatTcpSourceField = 'message' @@ -1260,7 +1273,8 @@ def tweak_malcolm_runtime(self, malcolm_install_path): default=args.netboxLogstashAutoPopulate, ) and ( - InstallerYesOrNo( + args.acceptDefaultsNonInteractive + or InstallerYesOrNo( "Autopopulating NetBox's inventory is not recommended. Are you sure?", default=args.netboxLogstashAutoPopulate, ) @@ -1284,6 +1298,8 @@ def tweak_malcolm_runtime(self, malcolm_install_path): # input packet capture parameters pcapNetSniff = False pcapTcpDump = False + liveArkime = False + liveArkimeNodeHost = '' liveZeek = False liveSuricata = False pcapIface = 'lo' @@ -1309,32 +1325,47 @@ def tweak_malcolm_runtime(self, malcolm_install_path): choices=[(x, '', x == captureOptions[0]) for x in captureOptions], )[0] if captureSelection == 'y': - pcapNetSniff = True + liveArkime = (malcolmProfile == PROFILE_HEDGEHOG) or (opensearchPrimaryMode != DatabaseMode.OpenSearchLocal) + pcapNetSniff = not liveArkime liveSuricata = True liveZeek = True + tweakIface = True elif captureSelection == 'c': if InstallerYesOrNo( 'Should Malcolm capture live network traffic to PCAP files for analysis with Arkime?', - default=args.pcapNetSniff or args.pcapTcpDump or (malcolmProfile == PROFILE_HEDGEHOG), + default=args.pcapNetSniff + or args.pcapTcpDump + or args.liveArkime + or (malcolmProfile == PROFILE_HEDGEHOG), ): - pcapNetSniff = InstallerYesOrNo('Capture packets using netsniff-ng?', default=args.pcapNetSniff) - if not pcapNetSniff: - pcapTcpDump = InstallerYesOrNo('Capture packets using tcpdump?', default=args.pcapTcpDump) + liveArkime = (opensearchPrimaryMode != DatabaseMode.OpenSearchLocal) and ( + (malcolmProfile == PROFILE_HEDGEHOG) + or InstallerYesOrNo('Capture packets using Arkime capture?', default=args.liveArkime) + ) + pcapNetSniff = (not liveArkime) and InstallerYesOrNo( + 'Capture packets using netsniff-ng?', default=args.pcapNetSniff + ) + pcapTcpDump = ( + (not liveArkime) + and (not pcapNetSniff) + and InstallerYesOrNo('Capture packets using tcpdump?', default=args.pcapTcpDump) + ) liveSuricata = InstallerYesOrNo( 'Should Malcolm analyze live network traffic with Suricata?', default=args.liveSuricata ) liveZeek = InstallerYesOrNo('Should Malcolm analyze live network traffic with Zeek?', default=args.liveZeek) - if pcapNetSniff or pcapTcpDump or liveZeek or liveSuricata: + if pcapNetSniff or pcapTcpDump or liveArkime or liveZeek or liveSuricata: pcapFilter = InstallerAskForString( 'Capture filter (tcpdump-like filter expression; leave blank to capture all traffic)', default=args.pcapFilter, ) - tweakIface = InstallerYesOrNo( + # Arkime requires disabling NIC offloading: https://arkime.com/faq#arkime_requires_full_packet_captures_error + tweakIface = liveArkime or InstallerYesOrNo( 'Disable capture interface hardware offloading and adjust ring buffer sizes?', default=args.tweakIface, ) - if pcapNetSniff or pcapTcpDump or liveZeek or liveSuricata: + if pcapNetSniff or pcapTcpDump or liveArkime or liveZeek or liveSuricata: pcapIface = '' loopBreaker = CountUntilException(MaxAskForValueCount, 'Invalid capture interface(s)') while (len(pcapIface) <= 0) and loopBreaker.increment(): @@ -1342,12 +1373,33 @@ def tweak_malcolm_runtime(self, malcolm_install_path): 'Specify capture interface(s) (comma-separated)', default=args.pcapIface ) + if liveArkime: + liveArkimeNodeHost = InstallerAskForString( + f"Enter this node's hostname or IP to associate with network traffic metadata", + default=args.liveArkimeNodeHost, + ) + if ( + (not liveArkimeNodeHost) + and (not args.acceptDefaultsNonInteractive) + and ( + not InstallerYesOrNo( + f'With live Arkime capture node hostname or IP is required for viewer session retrieval. Are you sure?', + default=False, + ) + ) + ): + liveArkimeNodeHost = InstallerAskForString( + f"Enter this node's hostname or IP to associate with network traffic metadata", + default=args.liveArkimeNodeHost, + ) + if ( (malcolmProfile == PROFILE_HEDGEHOG) and (not pcapNetSniff) and (not pcapTcpDump) and (not liveZeek) and (not liveSuricata) + and (not liveArkime) ): InstallerDisplayMessage( f'Warning: Running with the {malcolmProfile} profile but no capture methods are enabled.', @@ -1393,6 +1445,30 @@ def tweak_malcolm_runtime(self, malcolm_install_path): 'ARKIME_FREESPACEG', arkimeFreeSpaceG, ), + # live traffic analysis with Arkime capture (only available with remote opensearch or elasticsearch) + EnvValue( + os.path.join(args.configDir, 'arkime-live.env'), + 'ARKIME_LIVE_CAPTURE', + TrueOrFalseNoQuote(liveArkime), + ), + # capture source "node host" for live Arkime capture + EnvValue( + os.path.join(args.configDir, 'arkime-live.env'), + 'ARKIME_LIVE_NODE_HOST', + liveArkimeNodeHost, + ), + # rotated captured PCAP analysis with Arkime (not live capture) + EnvValue( + os.path.join(args.configDir, 'arkime-offline.env'), + 'ARKIME_ROTATED_PCAP', + TrueOrFalseNoQuote(autoArkime and (not liveArkime)), + ), + # automatic uploaded pcap analysis with Arkime + EnvValue( + os.path.join(args.configDir, 'arkime-offline.env'), + 'ARKIME_AUTO_ANALYZE_PCAP_FILES', + TrueOrFalseNoQuote(autoArkime), + ), # authentication method: basic (true), ldap (false) or no_authentication EnvValue( os.path.join(args.configDir, 'auth-common.env'), @@ -1650,6 +1726,12 @@ def tweak_malcolm_runtime(self, malcolm_install_path): 'SURICATA_UPDATE_RULES', TrueOrFalseNoQuote(suricataRuleUpdate), ), + # disable/enable ICS analyzers + EnvValue( + os.path.join(args.configDir, 'suricata.env'), + 'SURICATA_DISABLE_ICS_ALL', + TrueOrFalseNoQuote(not malcolmIcs), + ), # live traffic analysis with Suricata EnvValue( os.path.join(args.configDir, 'suricata-live.env'), @@ -1674,12 +1756,6 @@ def tweak_malcolm_runtime(self, malcolm_install_path): 'PCAP_NODE_NAME', pcapNodeName, ), - # capture source "node host" for locally processed PCAP files - EnvValue( - os.path.join(args.configDir, 'upload-common.env'), - 'PCAP_NODE_HOST', - pcapNodeHost, - ), # zeek file extraction mode EnvValue( os.path.join(args.configDir, 'zeek.env'), @@ -1744,7 +1820,7 @@ def tweak_malcolm_runtime(self, malcolm_install_path): EnvValue( os.path.join(args.configDir, 'zeek.env'), 'ZEEK_DISABLE_ICS_ALL', - '' if zeekIcs else TrueOrFalseNoQuote(not zeekIcs), + '' if malcolmIcs else TrueOrFalseNoQuote(not malcolmIcs), ), # disable/enable ICS best guess EnvValue( @@ -1892,7 +1968,7 @@ def tweak_malcolm_runtime(self, malcolm_install_path): # whether or not to restart services automatically (on boot, etc.) line = f"{sectionIndents[currentSection] * 2}restart: {restartMode}" - elif currentService == 'arkime': + elif (currentService == 'arkime') or (currentService == 'arkime-live'): # stuff specifically in the arkime section if re.match(r'^\s*-.+:/data/pcap(:.+)?\s*$', line): # Arkime's reference to the PCAP directory @@ -1901,13 +1977,6 @@ def tweak_malcolm_runtime(self, malcolm_install_path): pcapDir, sectionIndents[currentSection] * 3, ) - elif re.match(r'^[\s#]*-\s*"([\d\.]+:)?\d+:\d+"\s*$', line): - # set bind IP based on whether it should be externally exposed or not - line = re.sub( - r'^([\s#]*-\s*")([\d\.]+:)?(\d+:\d+"\s*)$', - fr"\g<1>{'0.0.0.0' if arkimeViewerOpen else '127.0.0.1'}:\g<3>", - line, - ) elif currentService == 'filebeat': # stuff specifically in the filebeat section @@ -2591,24 +2660,28 @@ def install_docker_compose(self): if self.orchMode is OrchestrationFramework.DOCKER_COMPOSE: # first see if docker compose/docker-compose is already installed and runnable # (try non-root and root) - dockerComposeCmd = ('docker', 'compose') - err, out = self.run_process([dockerComposeCmd, 'version'], privileged=False) + tmpComposeCmd = ('docker', 'compose') + + for priv in (False, True): + err, out = self.run_process([tmpComposeCmd, 'version'], privileged=priv) + if err == 0: + break if err != 0: - err, out = self.run_process([dockerComposeCmd, 'version'], privileged=True) - if err != 0: - dockerComposeCmd = 'docker-compose' - if not which(dockerComposeCmd, debug=self.debug): - if os.path.isfile('/usr/libexec/docker/cli-plugins/docker-compose'): - dockerComposeCmd = '/usr/libexec/docker/cli-plugins/docker-compose' - elif os.path.isfile('/usr/local/bin/docker-compose'): - dockerComposeCmd = '/usr/local/bin/docker-compose' - err, out = self.run_process([dockerComposeCmd, 'version'], privileged=False) - if err != 0: - err, out = self.run_process([dockerComposeCmd, 'version'], privileged=True) + tmpComposeCmd = 'docker-compose' + if not which(tmpComposeCmd, debug=self.debug): + if os.path.isfile('/usr/libexec/docker/cli-plugins/docker-compose'): + tmpComposeCmd = '/usr/libexec/docker/cli-plugins/docker-compose' + elif os.path.isfile('/usr/local/bin/docker-compose'): + tmpComposeCmd = '/usr/local/bin/docker-compose' + for priv in (False, True): + err, out = self.run_process([tmpComposeCmd, 'version'], privileged=priv) + if err == 0: + break - if (err != 0) and InstallerYesOrNo( - 'docker compose failed, attempt to install docker compose?', default=True - ): + if err == 0: + self.dockerComposeCmd = tmpComposeCmd + + elif InstallerYesOrNo('docker compose failed, attempt to install docker compose?', default=True): if InstallerYesOrNo('Install docker compose directly from docker github?', default=True): # download docker-compose from github and put it in /usr/local/bin @@ -2622,7 +2695,7 @@ def install_docker_compose(self): unames.append(out[0].lower()) if len(unames) == 2: # download docker-compose from github and save it to a temporary file - tempFileName = os.path.join(self.tempDirName, dockerComposeCmd) + tempFileName = os.path.join(self.tempDirName, tmpComposeCmd) dockerComposeUrl = f"https://github.com/docker/compose/releases/download/v{DOCKER_COMPOSE_INSTALL_VERSION}/docker-compose-{unames[0]}-{unames[1]}" if DownloadToFile(dockerComposeUrl, tempFileName, debug=self.debug): os.chmod(tempFileName, 493) # 493 = 0o755, mark as executable @@ -2632,7 +2705,7 @@ def install_docker_compose(self): ) if err == 0: eprint("Download and installation of docker-compose apparently succeeded") - dockerComposeCmd = '/usr/local/bin/docker-compose' + tmpComposeCmd = '/usr/local/bin/docker-compose' else: raise Exception(f'Error copying {tempFileName} to /usr/local/bin: {out}') @@ -2656,11 +2729,13 @@ def install_docker_compose(self): eprint(f"Install docker-compose via pip failed with {err}, {out}") # see if docker-compose is now installed and runnable (try non-root and root) - err, out = self.run_process([dockerComposeCmd, 'version'], privileged=False) - if err != 0: - err, out = self.run_process([dockerComposeCmd, 'version'], privileged=True) + for priv in (False, True): + err, out = self.run_process([tmpComposeCmd, 'version'], privileged=priv) + if err == 0: + break if err == 0: + self.dockerComposeCmd = tmpComposeCmd result = True if self.debug: eprint('docker compose succeeded') @@ -3508,6 +3583,16 @@ def main(): ) analysisArgGroup = parser.add_argument_group('Analysis options') + analysisArgGroup.add_argument( + '--auto-arkime', + dest='autoArkime', + type=str2bool, + metavar="true|false", + nargs='?', + const=True, + default=True, + help="Automatically analyze all PCAP files with Arkime", + ) analysisArgGroup.add_argument( '--auto-suricata', dest='autoSuricata', @@ -3540,7 +3625,7 @@ def main(): ) analysisArgGroup.add_argument( '--zeek-ics', - dest='zeekIcs', + dest='malcolmIcs', type=str2bool, metavar="true|false", nargs='?', @@ -3754,11 +3839,30 @@ def main(): metavar="true|false", nargs='?', const=True, - default=False, + default=True, help="Disable capture interface hardware offloading and adjust ring buffer sizes", ) captureArgGroup.add_argument( '--live-capture-arkime', + dest='liveArkime', + type=str2bool, + metavar="true|false", + nargs='?', + const=True, + default=False, + help=f"Capture live network traffic with Arkime capture (not available with --opensearch {DATABASE_MODE_LABELS[DatabaseMode.OpenSearchLocal]})", + ) + captureArgGroup.add_argument( + '--live-capture-arkime-node-host', + dest='liveArkimeNodeHost', + required=False, + metavar='', + type=str, + default='', + help='The node hostname or IP address to associate with live network traffic observed by Arkime capture', + ) + captureArgGroup.add_argument( + '--live-capture-netsniff', dest='pcapNetSniff', type=str2bool, metavar="true|false", @@ -3768,7 +3872,7 @@ def main(): help="Capture live network traffic with netsniff-ng for Arkime", ) captureArgGroup.add_argument( - '--live-capture-arkime-tcpdump', + '--live-capture-tcpdump', dest='pcapTcpDump', type=str2bool, metavar="true|false", @@ -3806,15 +3910,6 @@ def main(): default=os.getenv('HOSTNAME', os.getenv('COMPUTERNAME', platform.node())).split('.')[0], help='The node name to associate with network traffic metadata', ) - captureArgGroup.add_argument( - '--node-host', - dest='pcapNodeHost', - required=False, - metavar='', - type=str, - default='', - help='The node hostname or IP address to associate with network traffic metadata', - ) try: parser.error = parser.exit @@ -3906,8 +4001,6 @@ def main(): success = installer.install_docker_compose() if hasattr(installer, 'tweak_system_files'): success = installer.tweak_system_files() - if (orchMode is OrchestrationFramework.DOCKER_COMPOSE) and hasattr(installer, 'install_docker_images'): - success = installer.install_docker_images(imageFile) if (orchMode is OrchestrationFramework.DOCKER_COMPOSE) and hasattr(installer, 'install_malcolm_files'): success, installPath = installer.install_malcolm_files(malcolmFile, args.configDir is None) @@ -3958,11 +4051,19 @@ def main(): if args.debug: eprint(f"Malcolm installation detected at {installPath}") - if (installPath is not None) and os.path.isdir(installPath) and hasattr(installer, 'tweak_malcolm_runtime'): - installer.tweak_malcolm_runtime(installPath) - eprint(f"\nMalcolm has been installed to {installPath}. See README.md for more information.") - eprint( - f"Scripts for starting and stopping Malcolm and changing authentication-related settings can be found in {os.path.join(installPath, 'scripts')}." + if (installPath is not None) and os.path.isdir(installPath): + if hasattr(installer, 'tweak_malcolm_runtime'): + installer.tweak_malcolm_runtime(installPath) + + if ( + (not args.configOnly) + and (orchMode is OrchestrationFramework.DOCKER_COMPOSE) + and hasattr(installer, 'install_docker_images') + ): + success = installer.install_docker_images(imageFile, installPath) + + InstallerDisplayMessage( + f"Malcolm has been installed to {installPath}. See README.md for more information.\nScripts for starting and stopping Malcolm and changing authentication-related settings can be found in {os.path.join(installPath, 'scripts')}." ) diff --git a/scripts/malcolm_appliance_packager.sh b/scripts/malcolm_appliance_packager.sh index 3f7256549..d4fb0f795 100755 --- a/scripts/malcolm_appliance_packager.sh +++ b/scripts/malcolm_appliance_packager.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. if [ -z "$BASH_VERSION" ]; then echo "Wrong interpreter, please run \"$0\" with bash" @@ -61,8 +61,7 @@ if mkdir "$DESTDIR"; then # ensure that if we "grabbed a lock", we release it (works for clean exit, SIGTERM, and SIGINT/Ctrl-C) trap "cleanup" EXIT - mkdir $VERBOSE -p "$DESTDIR/arkime-logs/" - mkdir $VERBOSE -p "$DESTDIR/arkime-raw/" + mkdir $VERBOSE -p "$DESTDIR/arkime/rules/" mkdir $VERBOSE -p "$DESTDIR/filebeat/certs/" mkdir $VERBOSE -p "$DESTDIR/htadmin/" mkdir $VERBOSE -p "$DESTDIR/logstash/certs/" @@ -75,6 +74,7 @@ if mkdir "$DESTDIR"; then mkdir $VERBOSE -p "$DESTDIR/nginx/certs/" mkdir $VERBOSE -p "$DESTDIR/opensearch-backup/" mkdir $VERBOSE -p "$DESTDIR/opensearch/nodes/" + mkdir $VERBOSE -p "$DESTDIR/pcap/arkime-live/" mkdir $VERBOSE -p "$DESTDIR/pcap/processed/" mkdir $VERBOSE -p "$DESTDIR/pcap/upload/tmp/spool" mkdir $VERBOSE -p "$DESTDIR/pcap/upload/variants/" @@ -103,6 +103,7 @@ if mkdir "$DESTDIR"; then cp $VERBOSE ./scripts/malcolm_kubernetes.py "$DESTDIR/scripts/" cp $VERBOSE ./scripts/malcolm_utils.py "$DESTDIR/scripts/" cp $VERBOSE ./README.md "$DESTDIR/" + cp $VERBOSE ./arkime/rules/*.yml "$DESTDIR/arkime/rules/" cp $VERBOSE ./logstash/certs/*.conf "$DESTDIR/logstash/certs/" cp $VERBOSE ./logstash/maps/malcolm_severity.yaml "$DESTDIR/logstash/maps/" cp $VERBOSE -r ./netbox/config/ "$DESTDIR/netbox/" diff --git a/scripts/malcolm_common.py b/scripts/malcolm_common.py index 843af368a..d8e95ded3 100644 --- a/scripts/malcolm_common.py +++ b/scripts/malcolm_common.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. import getpass import importlib @@ -705,6 +705,7 @@ def DownloadToFile(url, local_filename, debug=False): | GET\s+/(_cat/health|api/status|sessions2-|arkime_\w+).+HTTP/[\d\.].+\b200\b | GET\s+/\s+.+\b200\b.+ELB-HealthChecker | (GET|POST|PATCH)\s+/netbox/.+HTTP/[\d\.].+\b20[01]\b + | (GET|POST)\s+/(fields|get|valueActions|views|fieldActions)\b.+bytes\s+[\d\.]+\s+ms | loaded\s+config\s+'/etc/netbox/config/ | LOG:\s+checkpoint\s+(complete|starting) | "netbox"\s+application\s+started diff --git a/scripts/malcolm_kubernetes.py b/scripts/malcolm_kubernetes.py index 4bb7bc47c..7643f58d5 100644 --- a/scripts/malcolm_kubernetes.py +++ b/scripts/malcolm_kubernetes.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. import base64 import glob @@ -11,6 +11,7 @@ from collections import defaultdict from itertools import chain from io import StringIO +from pathlib import Path from malcolm_common import ( DotEnvDynamic, @@ -23,7 +24,6 @@ from malcolm_utils import ( deep_get, dictsearch, - eprint, get_iterable, file_contents, remove_suffix, @@ -37,6 +37,7 @@ MALCOLM_IMAGE_PREFIX = 'ghcr.io/idaholab/malcolm/' MALCOLM_DOTFILE_SECRET_KEY = 'K8S_SECRET' +MALCOLM_CONFIGMAP_DIR_REPLACER = '_MALDIR_' MALCOLM_CONFIGMAPS = { 'etc-nginx': [ @@ -107,6 +108,12 @@ 'path': os.path.join(MalcolmPath, os.path.join('logstash', 'logstash.keystore')), }, ], + 'arkime-rules': [ + { + 'secret': False, + 'path': os.path.join(MalcolmPath, os.path.join('arkime', 'rules')), + }, + ], 'yara-rules': [ { 'secret': False, @@ -197,6 +204,7 @@ MALCOLM_PROFILES_CONTAINERS[PROFILE_MALCOLM] = [ 'api', 'arkime', + 'arkime-live', 'dashboards', 'dashboards-helper', 'filebeat', @@ -220,6 +228,7 @@ ] MALCOLM_PROFILES_CONTAINERS[PROFILE_HEDGEHOG] = [ 'arkime', + 'arkime-live', 'file-monitor', 'filebeat', 'pcap-capture', @@ -702,6 +711,8 @@ def StartMalcolm(namespace, malcolmPath, configPath, profile=PROFILE_MALCOLM): results_dict['create_namespace']['error'] = str(x) # create configmaps from files + # files in nested directories will be created with a name like foo_MALDIR_bar_MALDIR_baz.txt + # and then renamed to foo/bar/baz.txt during container start up by docker-uid-gid-setup.sh results_dict['create_namespaced_config_map']['result'] = dict() results_dict['create_namespaced_secret']['result'] = dict() for configMapName, configMapFiles in MALCOLM_CONFIGMAPS.items(): @@ -723,20 +734,21 @@ def StartMalcolm(namespace, malcolmPath, configPath, profile=PROFILE_MALCOLM): else: dataMap[os.path.basename(fname)] = contents elif os.path.isdir(fname): - for subfname in glob.iglob( - os.path.join(os.path.join(fname, '**'), '*'), recursive=True - ): - if os.path.isfile(subfname): - contents = file_contents( - subfname, - binary_fallback=True, + for root, dirNames, fileNames in os.walk(fname): + for f in fileNames: + subfname = os.path.join(root, f) + relfname = str(Path(os.path.join(root, f)).relative_to(fname)).replace( + os.sep, MALCOLM_CONFIGMAP_DIR_REPLACER ) - if hasattr(contents, 'decode'): - binaryDataMap[os.path.basename(subfname)] = base64.b64encode( - contents - ).decode('utf-8') - else: - dataMap[os.path.basename(subfname)] = contents + if os.path.isfile(subfname): + contents = file_contents( + subfname, + binary_fallback=True, + ) + if hasattr(contents, 'decode'): + binaryDataMap[relfname] = base64.b64encode(contents).decode('utf-8') + else: + dataMap[relfname] = contents metadata = kubeImported.client.V1ObjectMeta( name=configMapName, namespace=namespace, diff --git a/scripts/malcolm_utils.py b/scripts/malcolm_utils.py index cab6703d1..febc46df1 100644 --- a/scripts/malcolm_utils.py +++ b/scripts/malcolm_utils.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. import contextlib import enum diff --git a/scripts/package_zeek_logs.sh b/scripts/package_zeek_logs.sh index b06d6dcff..8a484e4da 100755 --- a/scripts/package_zeek_logs.sh +++ b/scripts/package_zeek_logs.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # package up Zeek logs in a format more suitable for upload to Malcolm # diff --git a/scripts/third-party-environments/aws/ami/packer_vars.json.example b/scripts/third-party-environments/aws/ami/packer_vars.json.example index 88031bc28..f6a3d87fd 100644 --- a/scripts/third-party-environments/aws/ami/packer_vars.json.example +++ b/scripts/third-party-environments/aws/ami/packer_vars.json.example @@ -2,7 +2,7 @@ "aws_access_key": "XXXXXXXXXXXXXXXXXXXX", "aws_secret_key": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "instance_type": "t2.micro", - "malcolm_tag": "v23.12.0", + "malcolm_tag": "v23.12.1", "malcolm_repo": "idaholab/Malcolm", "malcolm_uid": "1000", "ssh_username": "ec2-user", diff --git a/scripts/third-party-environments/aws/ami/scripts/Malcolm_AMI_Setup.sh b/scripts/third-party-environments/aws/ami/scripts/Malcolm_AMI_Setup.sh index a1c4d1f80..dcc032fe8 100755 --- a/scripts/third-party-environments/aws/ami/scripts/Malcolm_AMI_Setup.sh +++ b/scripts/third-party-environments/aws/ami/scripts/Malcolm_AMI_Setup.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # Configure Amazon Linux 2 and install Malcolm diff --git a/scripts/third-party-logs/fluent-bit-setup.ps1 b/scripts/third-party-logs/fluent-bit-setup.ps1 index 2fa6f40dd..8a35575dd 100644 --- a/scripts/third-party-logs/fluent-bit-setup.ps1 +++ b/scripts/third-party-logs/fluent-bit-setup.ps1 @@ -5,7 +5,7 @@ # configuration of fluent-bit (https://fluentbit.io/) for forwarding logs to # an instance of Malcolm (https://github.com/idaholab/malcolm). # -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ############################################################################### $fluent_bit_version = '2.2' diff --git a/scripts/third-party-logs/fluent-bit-setup.sh b/scripts/third-party-logs/fluent-bit-setup.sh index 69499b873..6ab0b8366 100755 --- a/scripts/third-party-logs/fluent-bit-setup.sh +++ b/scripts/third-party-logs/fluent-bit-setup.sh @@ -7,7 +7,7 @@ # configuration of fluent-bit (https://fluentbit.io/) for forwarding logs to # an instance of Malcolm (https://github.com/idaholab/malcolm). # -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ############################################################################### # force bash diff --git a/scripts/zeek_script_to_malcolm_boilerplate.py b/scripts/zeek_script_to_malcolm_boilerplate.py index 11c9857d3..f254099e0 100755 --- a/scripts/zeek_script_to_malcolm_boilerplate.py +++ b/scripts/zeek_script_to_malcolm_boilerplate.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # # This script takes as input the filenames of one or more .zeek scripts which diff --git a/sensor-iso/Dockerfile b/sensor-iso/Dockerfile index 1510f7e1a..774c62213 100644 --- a/sensor-iso/Dockerfile +++ b/sensor-iso/Dockerfile @@ -1,6 +1,6 @@ FROM ghcr.io/mmguero/qemu-live-iso:latest -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' diff --git a/sensor-iso/arkime/Dockerfile b/sensor-iso/arkime/Dockerfile index f6d5ab354..80c6074ea 100644 --- a/sensor-iso/arkime/Dockerfile +++ b/sensor-iso/arkime/Dockerfile @@ -1,6 +1,6 @@ FROM debian:12-slim -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" diff --git a/sensor-iso/arkime/build-arkime-deb.sh b/sensor-iso/arkime/build-arkime-deb.sh index c20eee583..b556741ab 100755 --- a/sensor-iso/arkime/build-arkime-deb.sh +++ b/sensor-iso/arkime/build-arkime-deb.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ARKIME_URL="https://github.com/arkime/arkime.git" OUTPUT_DIR="/tmp" diff --git a/sensor-iso/arkime/build-docker-image.sh b/sensor-iso/arkime/build-docker-image.sh index 808a5641b..e071e492d 100755 --- a/sensor-iso/arkime/build-docker-image.sh +++ b/sensor-iso/arkime/build-docker-image.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # force-navigate to script directory SCRIPT_PATH="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" diff --git a/sensor-iso/build.sh b/sensor-iso/build.sh index 0bc69ef89..85630c9ab 100755 --- a/sensor-iso/build.sh +++ b/sensor-iso/build.sh @@ -5,7 +5,7 @@ IMAGE_PUBLISHER=idaholab IMAGE_VERSION=1.0.0 IMAGE_DISTRIBUTION=bookworm -BEATS_VER="8.11.1" +BEATS_VER="8.11.3" BEATS_OSS="-oss" BUILD_ERROR_CODE=1 @@ -106,12 +106,13 @@ if [ -d "$WORKDIR" ]; then # replace capture interface for now, it'll need to be automatically detected/configured on boot sed -i "s/CAPTURE_INTERFACE=.*/CAPTURE_INTERFACE=xxxx/g" ./config/includes.chroot/opt/sensor/sensor_ctl/control_vars.conf - # copy shared scripts + # copy shared scripts and files rsync -a "$SCRIPT_PATH/shared/bin/" ./config/includes.chroot/usr/local/bin/ mkdir -p ./config/includes.chroot/opt/zeek/bin/ mv ./config/includes.chroot/usr/local/bin/zeekdeploy.sh ./config/includes.chroot/opt/zeek/bin/ ln -s -r ./config/includes.chroot/usr/local/bin/malcolm_utils.py ./config/includes.chroot/opt/zeek/bin/ chown -R root:root ./config/includes.chroot/usr/local/bin/ ./config/includes.chroot/opt/zeek/bin/ + rsync -a "$SCRIPT_PATH/suricata/" ./config/includes.chroot/opt/sensor/sensor_ctl/suricata/ # write out some version stuff specific to this installation version echo "BUILD_ID=\"$(date +'%Y-%m-%d')-${IMAGE_VERSION}\"" > ./config/includes.chroot/opt/sensor/.os-info @@ -167,7 +168,7 @@ if [ -d "$WORKDIR" ]; then fi fi curl -s -S -L -o ipv4-address-space.csv "https://www.iana.org/assignments/ipv4-address-space/ipv4-address-space.csv" - curl -s -S -L -o oui.txt "https://gitlab.com/wireshark/wireshark/raw/release-4.0/manuf" + curl -s -S -L -o oui.txt "https://www.wireshark.org/download/automated/data/manuf" popd >/dev/null 2>&1 # clone and build Arkime .deb package in its own clean environment (rather than in hooks/) diff --git a/sensor-iso/build_via_vagrant.sh b/sensor-iso/build_via_vagrant.sh index 7fd7fd43d..fa9a2f84a 100755 --- a/sensor-iso/build_via_vagrant.sh +++ b/sensor-iso/build_via_vagrant.sh @@ -28,7 +28,8 @@ function cleanup_shared_and_docs { "$SCRIPT_PATH"/_includes \ "$SCRIPT_PATH"/_layouts \ "$SCRIPT_PATH"/Gemfile \ - "$SCRIPT_PATH"/README.md + "$SCRIPT_PATH"/README.md \ + "$SCRIPT_PATH"/suricata } unset FORCE_PROVISION @@ -87,6 +88,8 @@ cp -r "$SCRIPT_PATH"/../shared \ "$SCRIPT_PATH"/../README.md "$SCRIPT_PATH"/ cp "$SCRIPT_PATH"/../scripts/documentation_build.sh "$SCRIPT_PATH"/docs/ cp "$SCRIPT_PATH"/../scripts/malcolm_utils.py "$SCRIPT_PATH"/shared/bin/ +mkdir "$SCRIPT_PATH"/suricata +cp -r "$SCRIPT_PATH"/../suricata/rules-default "$SCRIPT_PATH"/suricata/ YML_IMAGE_VERSION="$(grep -P "^\s+image:.*/malcolm/" "$SCRIPT_PATH"/../docker-compose-standalone.yml | awk '{print $2}' | cut -d':' -f2 | uniq -c | sort -nr | awk '{print $2}' | head -n 1)" [[ -n $YML_IMAGE_VERSION ]] && echo "$YML_IMAGE_VERSION" > "$SCRIPT_PATH"/shared/version.txt diff --git a/sensor-iso/config/hooks/normal/0168-firefox-install.hook.chroot b/sensor-iso/config/hooks/normal/0168-firefox-install.hook.chroot index 18c4941ab..98b7a4782 100755 --- a/sensor-iso/config/hooks/normal/0168-firefox-install.hook.chroot +++ b/sensor-iso/config/hooks/normal/0168-firefox-install.hook.chroot @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. export LC_ALL=C.UTF-8 export LANG=C.UTF-8 diff --git a/sensor-iso/config/hooks/normal/0169-pip-installs.hook.chroot b/sensor-iso/config/hooks/normal/0169-pip-installs.hook.chroot index a1b67dde1..3a217625c 100755 --- a/sensor-iso/config/hooks/normal/0169-pip-installs.hook.chroot +++ b/sensor-iso/config/hooks/normal/0169-pip-installs.hook.chroot @@ -1,6 +1,6 @@ #!/bin/sh -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. export LC_ALL=C.UTF-8 export LANG=C.UTF-8 diff --git a/sensor-iso/config/hooks/normal/0900-setup-rc-local.hook.chroot b/sensor-iso/config/hooks/normal/0900-setup-rc-local.hook.chroot index 7eeb00e8b..a97c39031 100755 --- a/sensor-iso/config/hooks/normal/0900-setup-rc-local.hook.chroot +++ b/sensor-iso/config/hooks/normal/0900-setup-rc-local.hook.chroot @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. sed -i 's/^exit 0//' /etc/rc.local 2>/dev/null diff --git a/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot b/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot index 161106cea..144e70778 100755 --- a/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot +++ b/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # some environment variables needed for build export CCACHE_DIR="/var/spool/ccache" @@ -224,9 +224,9 @@ freshclam --stdout --quiet --no-warnings # set up capabilities for network-related tools chown root:netdev /usr/sbin/netsniff-ng && \ - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip CAP_SYS_ADMIN+eip' /usr/sbin/netsniff-ng + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /usr/sbin/netsniff-ng chown root:netdev "${ZEEK_DIR}"/bin/zeek && \ - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' "${ZEEK_DIR}"/bin/zeek + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' "${ZEEK_DIR}"/bin/zeek chown root:netdev /sbin/ethtool && \ setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool chown root:netdev "${ZEEK_DIR}"/bin/capstats && \ @@ -236,7 +236,7 @@ chown root:netdev /usr/bin/tcpdump && \ chown root:netdev /usr/bin/suricata && \ setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /usr/bin/suricata chown root:netdev /opt/arkime/bin/capture && \ - setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /opt/arkime/bin/capture + setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /opt/arkime/bin/capture ln -s -f "${ZEEK_DIR}"/bin/zeek /usr/local/bin/ ln -s -f /usr/sbin/netsniff-ng /usr/local/bin/ diff --git a/sensor-iso/config/hooks/normal/0911-get-stig-scripts.hook.chroot b/sensor-iso/config/hooks/normal/0911-get-stig-scripts.hook.chroot index f5bc55347..2d9862133 100755 --- a/sensor-iso/config/hooks/normal/0911-get-stig-scripts.hook.chroot +++ b/sensor-iso/config/hooks/normal/0911-get-stig-scripts.hook.chroot @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # clone harbian-audit and clean up some stuff we don't need mkdir -p /opt diff --git a/sensor-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot b/sensor-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot index 09adb6273..ee8baa4ae 100755 --- a/sensor-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot +++ b/sensor-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # remove development packages not necessary for building dynamic Zeek plugins apt-get -y --purge remove \ diff --git a/sensor-iso/config/hooks/normal/0991-security-performance.hook.chroot b/sensor-iso/config/hooks/normal/0991-security-performance.hook.chroot index 2a462a350..93afb4f60 100755 --- a/sensor-iso/config/hooks/normal/0991-security-performance.hook.chroot +++ b/sensor-iso/config/hooks/normal/0991-security-performance.hook.chroot @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # configure firewall sed -i "s/LOGLEVEL=.*/LOGLEVEL=off/" /etc/ufw/ufw.conf @@ -96,14 +96,6 @@ echo "umask 077" >> /etc/profile echo "export UMASK=077" >> /etc/profile echo "export PYTHONDONTWRITEBYTECODE=1" >> /etc/profile -# enable cron logging -sed -r -i "s@^#(cron\.\*\s+.*/var/log/cron\.log)@\1@" /etc/rsyslog.conf - -# enable rsyslog forwarding to localhost:9514 over UDP (for filebeat syslog input) -echo >> /etc/rsyslog.conf -echo '*.* @127.0.0.1:9514' >> /etc/rsyslog.conf -echo >> /etc/rsyslog.conf - # put sudoers log into its own logfile awk 'FNR==NR{ if (/^Defaults/) p=NR; next} 1; FNR==p{ print "Defaults\t!syslog\nDefaults\tlogfile=/var/log/sudo.log" }' /etc/sudoers /etc/sudoers > /tmp/newsudoers mv /tmp/newsudoers /etc/sudoers && chmod 440 /etc/sudoers diff --git a/sensor-iso/config/hooks/normal/0992-login.hook.chroot b/sensor-iso/config/hooks/normal/0992-login.hook.chroot index d8dc2fff3..c8cd4a337 100755 --- a/sensor-iso/config/hooks/normal/0992-login.hook.chroot +++ b/sensor-iso/config/hooks/normal/0992-login.hook.chroot @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. sed -i 's/^#autologin-user=.*/autologin-user=sensor/' /etc/lightdm/lightdm.conf sed -i 's/^#autologin-user-timeout=.*/autologin-user-timeout=0/' /etc/lightdm/lightdm.conf diff --git a/sensor-iso/config/includes.binary/install/preseed_base.cfg b/sensor-iso/config/includes.binary/install/preseed_base.cfg index 81b5ba435..bc3c87ca2 100644 --- a/sensor-iso/config/includes.binary/install/preseed_base.cfg +++ b/sensor-iso/config/includes.binary/install/preseed_base.cfg @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. d-i netcfg/enable boolean false d-i netcfg/choose_interface select auto diff --git a/sensor-iso/config/includes.binary/install/preseed_multipar.cfg b/sensor-iso/config/includes.binary/install/preseed_multipar.cfg index 4f5c10499..9189b6932 100644 --- a/sensor-iso/config/includes.binary/install/preseed_multipar.cfg +++ b/sensor-iso/config/includes.binary/install/preseed_multipar.cfg @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. d-i debian-installer/locale string en_US.UTF-8 d-i console-setup/ask_detect boolean false diff --git a/sensor-iso/config/includes.binary/install/preseed_vmware.cfg b/sensor-iso/config/includes.binary/install/preseed_vmware.cfg index 7d433121d..792d0c7a6 100644 --- a/sensor-iso/config/includes.binary/install/preseed_vmware.cfg +++ b/sensor-iso/config/includes.binary/install/preseed_vmware.cfg @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. d-i debian-installer/locale string en_US.UTF-8 d-i console-setup/ask_detect boolean false diff --git a/sensor-iso/config/includes.chroot/opt/zeek/share/zeek/site/extractor.zeek b/sensor-iso/config/includes.chroot/opt/zeek/share/zeek/site/extractor.zeek index 0418f23e9..07e745368 100644 --- a/sensor-iso/config/includes.chroot/opt/zeek/share/zeek/site/extractor.zeek +++ b/sensor-iso/config/includes.chroot/opt/zeek/share/zeek/site/extractor.zeek @@ -1,6 +1,6 @@ #!/usr/bin/env zeek -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. @load ./extractor_params diff --git a/sensor-iso/config/includes.chroot/opt/zeek/share/zeek/site/extractor_params.zeek b/sensor-iso/config/includes.chroot/opt/zeek/share/zeek/site/extractor_params.zeek index a169bd0d9..2ffb8a8a2 100644 --- a/sensor-iso/config/includes.chroot/opt/zeek/share/zeek/site/extractor_params.zeek +++ b/sensor-iso/config/includes.chroot/opt/zeek/share/zeek/site/extractor_params.zeek @@ -1,6 +1,6 @@ #!/usr/bin/env zeek -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. export { const extractor_extract_none = "none" &redef; diff --git a/sensor-iso/interface/sensor_ctl/arkime/config.ini b/sensor-iso/interface/sensor_ctl/arkime/config.ini index 9b67ff43e..fd30ae188 100644 --- a/sensor-iso/interface/sensor_ctl/arkime/config.ini +++ b/sensor-iso/interface/sensor_ctl/arkime/config.ini @@ -2,49 +2,50 @@ # so you can (for the most part) ignore settings here that seem like dummy settings [default] +antiSynDrop=false +compressES=false +dropGroup=netdev +dropUser=sensor elasticsearch=http://192.168.0.1:9200 -rotateIndex=daily -passwordSecret=Malcolm +freeSpaceG=5% +geoLite2ASN=/dummy/GeoLite2-ASN.mmdb +geoLite2Country=/dummy/GeoLite2-Country.mmdb httpRealm=Arkime +icmpTimeout=10 +bpf= interface=enp0s1 -pcapDir=/tmp +logESRequests=false +logEveryXPackets=500000 +logFileCreation=true +logHTTPConnections=false +logUnknownProtocols=false +maxESConns=30 +maxESRequests=500 maxFileSizeG=4 maxFileTimeM=180 -tcpTimeout=600 -tcpSaveTimeout=720 -udpTimeout=30 -icmpTimeout=10 -maxStreams=1000000 maxPackets=10000 -freeSpaceG=5% -viewPort=8005 -geoLite2Country=/dummy/GeoLite2-Country.mmdb -geoLite2ASN=/dummy/GeoLite2-ASN.mmdb -rirFile=/dummy/ipv4-address-space.csv +maxReqBody=64 +maxStreams=1000000 ouiFile=/dummy/oui.txt -dropUser=sensor -dropGroup=netdev -parseSMTP=true -parseSMB=true +packetsPerPoll=50000 parseQSValue=false -supportSha256=false -maxReqBody=64 -reqBodyOnlyUtf8=true -smtpIpHeaders=X-Originating-IP:;X-Barracuda-Apparent-Source-IP: parsersDir=/dummy/parsers +parseSMB=true +parseSMTP=true +passwordSecret=Malcolm +pcapDir=/tmp pluginsDir=/dummy/plugins -spiDataMaxIndices=2 +reqBodyOnlyUtf8=true +rirFile=/dummy/ipv4-address-space.csv +rotateIndex=daily +smtpIpHeaders=X-Originating-IP:;X-Barracuda-Apparent-Source-IP: +spiDataMaxIndices=7 +supportSha256=false +tcpSaveTimeout=720 +tcpTimeout=600 +udpTimeout=30 uploadCommand= -compressES=false -maxESConns=30 -maxESRequests=500 -packetsPerPoll=50000 -antiSynDrop=false -logEveryXPackets=500000 -logUnknownProtocols=false -logESRequests=false -logFileCreation=true -logHTTPConnections=false +viewPort=8005 ### High Performance settings # https://github.com/arkime/arkime/wiki/Settings#High_Performance_Settings @@ -60,5 +61,4 @@ simpleGzipLevel=3 packetThreads=5 maxPacketsInQueue=300000 dbBulkSize=4000000 -#compressES=true rulesFiles=/dummy/rules.yml diff --git a/sensor-iso/interface/sensor_ctl/clean.sh b/sensor-iso/interface/sensor_ctl/clean.sh index e4f696d9c..047b1d220 100755 --- a/sensor-iso/interface/sensor_ctl/clean.sh +++ b/sensor-iso/interface/sensor_ctl/clean.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. set -e diff --git a/sensor-iso/interface/sensor_ctl/control.sh b/sensor-iso/interface/sensor_ctl/control.sh index eb020d25d..84a645b1f 100755 --- a/sensor-iso/interface/sensor_ctl/control.sh +++ b/sensor-iso/interface/sensor_ctl/control.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. set -e diff --git a/sensor-iso/interface/sensor_ctl/control_vars.conf b/sensor-iso/interface/sensor_ctl/control_vars.conf index 98301dae5..d821c64d8 100644 --- a/sensor-iso/interface/sensor_ctl/control_vars.conf +++ b/sensor-iso/interface/sensor_ctl/control_vars.conf @@ -89,6 +89,7 @@ export ZEEK_DISABLE_BEST_GUESS_ICS=true # Suricata export SURICATA_CUSTOM_RULES_ONLY=false +export SURICATA_DISABLE_ICS_ALL=false export SURICATA_RUNMODE=workers export SURICATA_AF_PACKET_BLOCK_SIZE=32768 export SURICATA_AF_PACKET_BLOCK_TIMEOUT=10 @@ -140,7 +141,7 @@ export AUTOSTART_FLUENTBIT_AIDE=false export AUTOSTART_FLUENTBIT_AUDITLOG=false export AUTOSTART_FLUENTBIT_KMSG=false export AUTOSTART_FLUENTBIT_METRICS=false -export AUTOSTART_FLUENTBIT_SYSLOG=false +export AUTOSTART_FLUENTBIT_SYSTEMD=false export AUTOSTART_FLUENTBIT_THERMAL=false export AUTOSTART_MISCBEAT=false export AUTOSTART_NETSNIFF=false diff --git a/sensor-iso/interface/sensor_ctl/filebeat/filebeat.yml b/sensor-iso/interface/sensor_ctl/filebeat/filebeat.yml index 6b6ec2fda..0816bb59f 100644 --- a/sensor-iso/interface/sensor_ctl/filebeat/filebeat.yml +++ b/sensor-iso/interface/sensor_ctl/filebeat/filebeat.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. logging.metrics.enabled: false diff --git a/sensor-iso/interface/sensor_ctl/filebeat/sensor_filebeat_local.sh b/sensor-iso/interface/sensor_ctl/filebeat/sensor_filebeat_local.sh index 47de02598..212a1d1ad 100755 --- a/sensor-iso/interface/sensor_ctl/filebeat/sensor_filebeat_local.sh +++ b/sensor-iso/interface/sensor_ctl/filebeat/sensor_filebeat_local.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. if [[ -z "$ZEEK_CAPTURE_PATH" ]]; then ZEEK_CAPTURE_PATH="$HOME/bro_logs" diff --git a/sensor-iso/interface/sensor_ctl/miscbeat/filebeat.yml b/sensor-iso/interface/sensor_ctl/miscbeat/filebeat.yml index de1d9eb84..f3a0dbae7 100644 --- a/sensor-iso/interface/sensor_ctl/miscbeat/filebeat.yml +++ b/sensor-iso/interface/sensor_ctl/miscbeat/filebeat.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. filebeat.inputs: - type: tcp diff --git a/sensor-iso/interface/sensor_ctl/miscbeat/sensor_miscbeat_local.sh b/sensor-iso/interface/sensor_ctl/miscbeat/sensor_miscbeat_local.sh index e650773dc..a62c80487 100755 --- a/sensor-iso/interface/sensor_ctl/miscbeat/sensor_miscbeat_local.sh +++ b/sensor-iso/interface/sensor_ctl/miscbeat/sensor_miscbeat_local.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # force-navigate to script directory (containing config file) [[ "$(uname -s)" = 'Darwin' ]] && REALPATH=grealpath || REALPATH=realpath diff --git a/sensor-iso/interface/sensor_ctl/scripts/log_disk_space.sh b/sensor-iso/interface/sensor_ctl/scripts/log_disk_space.sh index 28d09a96b..798987cde 100755 --- a/sensor-iso/interface/sensor_ctl/scripts/log_disk_space.sh +++ b/sensor-iso/interface/sensor_ctl/scripts/log_disk_space.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. SPACE_STRING="$(/bin/df -lh --output=source,target,avail,size,pcent | tail -n +2 | grep '^/dev' | tr -s ' ' ',' | cut -d, -f2,3,4,5 | sed 's/^/\[/' | sed 's/$/\]/' | tr '\n' '.')" logger "${SPACE_STRING}" diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-syslog.conf b/sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-syslog.conf deleted file mode 100644 index 0aaef35eb..000000000 --- a/sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-syslog.conf +++ /dev/null @@ -1,21 +0,0 @@ -[program:fluentbit-syslog] -command=/opt/fluent-bit/bin/fluent-bit - -R /etc/fluent-bit/parsers.conf - -i syslog - -p Mode=udp - -p Listen=127.0.0.1 - -p Port=9514 - -p Parser=syslog-rfc3164 - -p Buffer_Chunk_Size=32000 - -p Buffer_Max_Size=64000 - -o tcp://localhost:%(ENV_MISCBEAT_PORT)s - -p format=json_lines - -F nest -p Operation=nest -p Nested_under=syslog -p WildCard='*' -m '*' - -F record_modifier -p "Record=module syslog" -m '*' - -f 1 -startsecs=20 -startretries=3 -stopasgroup=true -killasgroup=true -autostart=%(ENV_AUTOSTART_FLUENTBIT_SYSLOG)s -autorestart=%(ENV_AUTOSTART_FLUENTBIT_SYSLOG)s diff --git a/sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-systemd.conf b/sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-systemd.conf new file mode 100644 index 000000000..d62057e30 --- /dev/null +++ b/sensor-iso/interface/sensor_ctl/supervisor.d/fluentbit-systemd.conf @@ -0,0 +1,18 @@ +[program:fluentbit-systemd] +command=/opt/fluent-bit/bin/fluent-bit + -R /etc/fluent-bit/parsers.conf + -i systemd + -p Read_From_Tail=On + -p Lowercase=On + -p Strip_Underscores=On + -o tcp://localhost:%(ENV_MISCBEAT_PORT)s + -p format=json_lines + -F nest -p Operation=nest -p Nested_under=systemd -p WildCard='*' -m '*' + -F record_modifier -p "Record=module systemd" -m '*' + -f 1 +startsecs=20 +startretries=3 +stopasgroup=true +killasgroup=true +autostart=%(ENV_AUTOSTART_FLUENTBIT_SYSTEMD)s +autorestart=%(ENV_AUTOSTART_FLUENTBIT_SYSTEMD)s diff --git a/sensor-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh b/sensor-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh index 048e2944e..fd7c32ce2 100644 --- a/sensor-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh +++ b/sensor-iso/interface/sensor_ctl/supervisor.init/arkime_config_populate.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. export ARKIME_HTTPS_FLAG="" @@ -38,6 +38,10 @@ if [[ -n $SUPERVISOR_PATH ]] && [[ -r "$SUPERVISOR_PATH"/arkime/config.ini ]]; t sed -r -i "s|(elasticsearch)\s*=\s*.*|\1=$ARKIME_ELASTICSEARCH|" "$ARKIME_CONFIG_FILE" fi + if [[ -n $ARKIME_VIEWER_PORT ]]; then + sed -r -i "s/(viewPort)\s*=\s*.*/\1=$ARKIME_VIEWER_PORT/" "$ARKIME_CONFIG_FILE" + fi + if [[ -n $ARKIME_PASSWORD_SECRET ]]; then # place the Arkime viewer cluster password hash in the config file sed -r -i "s|(passwordSecret)\s*=\s*.*|\1=$ARKIME_PASSWORD_SECRET|" "$ARKIME_CONFIG_FILE" @@ -64,7 +68,6 @@ if [[ -n $SUPERVISOR_PATH ]] && [[ -r "$SUPERVISOR_PATH"/arkime/config.ini ]]; t if [[ -n $ARKIME_FREESPACEG ]]; then sed -r -i "s/(freeSpaceG)\s*=\s*.*/\1=$ARKIME_FREESPACEG/" "$ARKIME_CONFIG_FILE" fi - # pcap compression COMPRESSION_TYPE="${ARKIME_COMPRESSION_TYPE:-none}" COMPRESSION_LEVEL="${ARKIME_COMPRESSION_LEVEL:-0}" diff --git a/sensor-iso/interface/sensor_ctl/supervisor.init/supercronic_populate.sh b/sensor-iso/interface/sensor_ctl/supervisor.init/supercronic_populate.sh index ee81c84ba..a4eb73573 100644 --- a/sensor-iso/interface/sensor_ctl/supervisor.init/supercronic_populate.sh +++ b/sensor-iso/interface/sensor_ctl/supervisor.init/supercronic_populate.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. if [[ -n $SUPERVISOR_PATH ]] && [[ -d "$SUPERVISOR_PATH"/supercronic ]]; then diff --git a/sensor-iso/interface/sensor_ctl/supervisor.init/suricata_config_populate.sh b/sensor-iso/interface/sensor_ctl/supervisor.init/suricata_config_populate.sh index 96473ab4b..63f020636 100644 --- a/sensor-iso/interface/sensor_ctl/supervisor.init/suricata_config_populate.sh +++ b/sensor-iso/interface/sensor_ctl/supervisor.init/suricata_config_populate.sh @@ -12,8 +12,10 @@ if [[ -n $SUPERVISOR_PATH ]] && [[ -r /usr/local/bin/suricata_config_populate.py [[ ! -f "$SUPERVISOR_PATH"/suricata/update.yaml ]] && cp "$(dpkg -L suricata-update | grep 'update\.yaml' | head -n 1)" "$SUPERVISOR_PATH"/suricata/update.yaml # specify the custom rules and configuration directories relative to the supervisor path + SURICATA_DEFAULT_RULES_DIR="$SUPERVISOR_PATH"/suricata/rules-default SURICATA_CUSTOM_RULES_DIR="$SUPERVISOR_PATH"/suricata/rules SURICATA_CUSTOM_CONFIG_DIR="$SUPERVISOR_PATH"/suricata/include-configs + [[ -d "$SURICATA_DEFAULT_RULES_DIR" ]] && export SURICATA_DEFAULT_RULES_DIR [[ -d "$SURICATA_CUSTOM_RULES_DIR" ]] && export SURICATA_CUSTOM_RULES_DIR [[ -d "$SURICATA_CUSTOM_CONFIG_DIR" ]] && export SURICATA_CUSTOM_CONFIG_DIR diff --git a/sensor-iso/interface/sensor_ctl/supervisor.sh b/sensor-iso/interface/sensor_ctl/supervisor.sh index 7f1dc9241..933baf5cf 100755 --- a/sensor-iso/interface/sensor_ctl/supervisor.sh +++ b/sensor-iso/interface/sensor_ctl/supervisor.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. set -e @@ -81,7 +81,11 @@ mkdir -p "$SUPERVISOR_PATH/"{log,run} rm -f "$SUPERVISOR_PATH/"/log/* rm -rf /opt/sensor/sensor_ctl/zeek/intel/lock || true -mkdir -p "$SUPERVISOR_PATH"/suricata/rules "$SUPERVISOR_PATH"/suricata/include-configs "$ZEEK_LOG_PATH"/suricata 2>/dev/null || true +mkdir -p "$SUPERVISOR_PATH"/suricata/rules \ + "$SUPERVISOR_PATH"/suricata/rules-default/OT \ + "$SUPERVISOR_PATH"/suricata/rules-default/IT \ + "$SUPERVISOR_PATH"/suricata/include-configs \ + "$ZEEK_LOG_PATH"/suricata 2>/dev/null || true mkdir -p "$PCAP_PATH"/ 2>/dev/null || true mkdir -p "$SUPERVISOR_PATH"/supercronic 2>/dev/null && touch "$SUPERVISOR_PATH"/supercronic/crontab || true diff --git a/sensor-iso/interface/sensor_ctl/zeek/extractor_override.interesting.zeek b/sensor-iso/interface/sensor_ctl/zeek/extractor_override.interesting.zeek index eaf220289..290a8948c 100644 --- a/sensor-iso/interface/sensor_ctl/zeek/extractor_override.interesting.zeek +++ b/sensor-iso/interface/sensor_ctl/zeek/extractor_override.interesting.zeek @@ -1,6 +1,6 @@ #!/usr/bin/env zeek -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. export { redef extractor_always_extract_unknown = F; diff --git a/sensor-iso/interface/sensor_interface/routes.py b/sensor-iso/interface/sensor_interface/routes.py index 2f4c3d66e..4a0b0c5ce 100644 --- a/sensor-iso/interface/sensor_interface/routes.py +++ b/sensor-iso/interface/sensor_interface/routes.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. import psutil import time diff --git a/sensor-iso/interface/sensor_interface/static/js/custom.js b/sensor-iso/interface/sensor_interface/static/js/custom.js index 592b584cb..9e7373347 100644 --- a/sensor-iso/interface/sensor_interface/static/js/custom.js +++ b/sensor-iso/interface/sensor_interface/static/js/custom.js @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +// Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. function start_all() { var xhttp = new XMLHttpRequest(); diff --git a/sensor-iso/interface/sensor_interface/sysquery/sys_service.py b/sensor-iso/interface/sensor_interface/sysquery/sys_service.py index a1a432bf2..781882084 100644 --- a/sensor-iso/interface/sensor_interface/sysquery/sys_service.py +++ b/sensor-iso/interface/sensor_interface/sysquery/sys_service.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. import subprocess import json diff --git a/sensor-iso/vagrant/Vagrantfile b/sensor-iso/vagrant/Vagrantfile index 500ab7083..04cfb86f6 100644 --- a/sensor-iso/vagrant/Vagrantfile +++ b/sensor-iso/vagrant/Vagrantfile @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. unless Vagrant.has_plugin?("vagrant-sshfs") raise 'vagrant-sshfs plugin is not installed!' diff --git a/sensor-iso/yara/Dockerfile b/sensor-iso/yara/Dockerfile index 4afe51c14..98b9c4674 100644 --- a/sensor-iso/yara/Dockerfile +++ b/sensor-iso/yara/Dockerfile @@ -1,6 +1,6 @@ FROM debian:12-slim -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" diff --git a/sensor-iso/yara/build-docker-image.sh b/sensor-iso/yara/build-docker-image.sh index 190dd2454..31060166b 100755 --- a/sensor-iso/yara/build-docker-image.sh +++ b/sensor-iso/yara/build-docker-image.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # force-navigate to script directory SCRIPT_PATH="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" diff --git a/sensor-iso/yara/build-yara-deb.sh b/sensor-iso/yara/build-yara-deb.sh index dd8415742..1fd9f7989 100755 --- a/sensor-iso/yara/build-yara-deb.sh +++ b/sensor-iso/yara/build-yara-deb.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. YARA_URL="https://github.com/VirusTotal/YARA" YARA_VER="$(curl -sqI "$YARA_URL/releases/latest" | awk -F '/' '/^location/ {print substr($NF, 1, length($NF)-1)}' | sed 's/^v//')" diff --git a/sensor-iso/zeek/Dockerfile b/sensor-iso/zeek/Dockerfile index 6086825ff..c4e69ce8e 100644 --- a/sensor-iso/zeek/Dockerfile +++ b/sensor-iso/zeek/Dockerfile @@ -1,6 +1,6 @@ FROM debian:12-slim -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" diff --git a/sensor-iso/zeek/build-docker-image.sh b/sensor-iso/zeek/build-docker-image.sh index 320226c35..3538212ff 100755 --- a/sensor-iso/zeek/build-docker-image.sh +++ b/sensor-iso/zeek/build-docker-image.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # force-navigate to script directory SCRIPT_PATH="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" diff --git a/sensor-iso/zeek/build-zeek-deb.sh b/sensor-iso/zeek/build-zeek-deb.sh index 4be53b1b8..0ea95d4dc 100755 --- a/sensor-iso/zeek/build-zeek-deb.sh +++ b/sensor-iso/zeek/build-zeek-deb.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. export CCACHE_DIR="/var/spool/ccache" export CCACHE_COMPRESS=1 diff --git a/shared/bin/agg-init.sh b/shared/bin/agg-init.sh index 3f8d0857f..b0215f97c 100755 --- a/shared/bin/agg-init.sh +++ b/shared/bin/agg-init.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. SCRIPT_PATH="$(dirname $(realpath -e "${BASH_SOURCE[0]}"))" diff --git a/shared/bin/capture-format-wait.sh b/shared/bin/capture-format-wait.sh index e2ba88095..1c4362e7c 100755 --- a/shared/bin/capture-format-wait.sh +++ b/shared/bin/capture-format-wait.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. grep -q boot=live /proc/cmdline && exit 0 diff --git a/shared/bin/common-init.sh b/shared/bin/common-init.sh index 645e0d181..19bbd3f9b 100755 --- a/shared/bin/common-init.sh +++ b/shared/bin/common-init.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. declare -A IFACES diff --git a/shared/bin/configure-capture.py b/shared/bin/configure-capture.py index e948c2bcb..c67729d81 100755 --- a/shared/bin/configure-capture.py +++ b/shared/bin/configure-capture.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # script for configuring sensor capture and forwarding parameters diff --git a/shared/bin/configure-interfaces.py b/shared/bin/configure-interfaces.py index f3f4c5bfc..56eafb508 100755 --- a/shared/bin/configure-interfaces.py +++ b/shared/bin/configure-interfaces.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # script for configuring sensor network interface controller(s) diff --git a/shared/bin/docker-load-wait.sh b/shared/bin/docker-load-wait.sh index c6cd55c8f..5f177c912 100755 --- a/shared/bin/docker-load-wait.sh +++ b/shared/bin/docker-load-wait.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. grep -q boot=live /proc/cmdline && exit 0 diff --git a/shared/bin/docker-uid-gid-setup.sh b/shared/bin/docker-uid-gid-setup.sh index 0022249ac..eb5189c2d 100755 --- a/shared/bin/docker-uid-gid-setup.sh +++ b/shared/bin/docker-uid-gid-setup.sh @@ -40,11 +40,25 @@ if [[ -n ${CONFIG_MAP_DIR} ]] && command -v rsync >/dev/null 2>&1; then awk '{print gsub("/","/"), $0}' | sort -n | cut -d' ' -f2- | \ while read CMDIR; do + DSTDIR="$(realpath "${CMDIR}"/../)" rsync --recursive --copy-links \ "--usermap=*:${PUID:-${DEFAULT_UID}}" \ "--groupmap=*:${PGID:-${DEFAULT_GID}}" \ --exclude='..*' --exclude="${MAP_DIR}"/ --exclude=.dockerignore --exclude=.gitignore \ - "${CMDIR}"/ "${CMDIR}"/../ + "${CMDIR}"/ "${DSTDIR}"/ + + # Additionally, files in these directories with _MALDIR_ in the name will be expanded out, + # creating the intermediate paths. For example: + # ./acid_MALDIR_ACID_MALDIR_s7comm_MALDIR_detect_MALDIR_copy.zeek + # will be renamed to + # ./acid/ACID/s7comm/detect/copy.zeek + find "${DSTDIR}" -type f -name '*_MALDIR_*' -print -o -path "${CMDIR}" -prune 2>/dev/null | \ + while read FLATTENED_FILE; do + EXPANDED_FILE="$(echo "${FLATTENED_FILE}" | sed 's@_MALDIR_@/@g')" + mkdir -p "$(dirname "${EXPANDED_FILE}")" && \ + mv "${FLATTENED_FILE}" "${EXPANDED_FILE}" || \ + true + done # loop over flattened filenames # TODO - regarding ownership and permissions: # diff --git a/shared/bin/fstab.py b/shared/bin/fstab.py index 7f13f45ce..b6750cd84 100644 --- a/shared/bin/fstab.py +++ b/shared/bin/fstab.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # fstab interpreter diff --git a/shared/bin/keystore-bootstrap.sh b/shared/bin/keystore-bootstrap.sh index f24243523..b519db285 100644 --- a/shared/bin/keystore-bootstrap.sh +++ b/shared/bin/keystore-bootstrap.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # make sure the keystore file used by the tool (e.g., foobar) is copied or created # into the correct location before the tool. starts up. diff --git a/shared/bin/malcolm-first-run-configure.sh b/shared/bin/malcolm-first-run-configure.sh index 1f82c876e..3a81f572f 100755 --- a/shared/bin/malcolm-first-run-configure.sh +++ b/shared/bin/malcolm-first-run-configure.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. grep -q boot=live /proc/cmdline && exit 0 diff --git a/shared/bin/manuf-oui-parse.py b/shared/bin/manuf-oui-parse.py index e37d65ed0..92920adff 100755 --- a/shared/bin/manuf-oui-parse.py +++ b/shared/bin/manuf-oui-parse.py @@ -17,7 +17,7 @@ except ImportError: import yaml -DEFAULT_MANUF_URL = "https://gitlab.com/wireshark/wireshark/raw/release-4.0/manuf" +DEFAULT_MANUF_URL = "https://www.wireshark.org/download/automated/data/manuf" padded_mac_low = '00:00:00:00:00:00' padded_mac_high = 'FF:FF:FF:FF:FF:FF' mac_pattern = re.compile(r"[-:\.]") diff --git a/shared/bin/opensearch_read_only.py b/shared/bin/opensearch_read_only.py index 7d28bf4b3..38dac8715 100755 --- a/shared/bin/opensearch_read_only.py +++ b/shared/bin/opensearch_read_only.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. import argparse import json diff --git a/shared/bin/opensearch_status.sh b/shared/bin/opensearch_status.sh index 47f7e5772..72883939b 100755 --- a/shared/bin/opensearch_status.sh +++ b/shared/bin/opensearch_status.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. set -e diff --git a/shared/bin/pcap_processor.py b/shared/bin/pcap_processor.py index 09ff58a00..8f54857c9 100755 --- a/shared/bin/pcap_processor.py +++ b/shared/bin/pcap_processor.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ################################################################################################### # Process queued files reported by pcap_watcher.py, using either arkime's capture or zeek to process @@ -47,14 +47,15 @@ PCAP_PROCESSING_MODE_ZEEK = "zeek" PCAP_PROCESSING_MODE_SURICATA = "suricata" -ARKIME_CAPTURE_PATH = "/opt/arkime/bin/capture" +ARKIME_CAPTURE_PATH = "/opt/arkime/bin/capture-offline" +ARKIME_AUTOARKIME_TAG = 'AUTOARKIME' -SURICATA_PATH = "/usr/bin/suricata" +SURICATA_PATH = "/usr/bin/suricata-offline" SURICATA_LOG_DIR = os.getenv('SURICATA_LOG_DIR', '/var/log/suricata') SURICATA_CONFIG_FILE = os.getenv('SURICATA_CONFIG_FILE', '/etc/suricata/suricata.yaml') SURICATA_AUTOSURICATA_TAG = 'AUTOSURICATA' -ZEEK_PATH = "/opt/zeek/bin/zeek" +ZEEK_PATH = "/opt/zeek/bin/zeek-offline" ZEEK_EXTRACTOR_MODE_INTERESTING = 'interesting' ZEEK_EXTRACTOR_MODE_MAPPED = 'mapped' ZEEK_EXTRACTOR_MODE_NONE = 'none' @@ -71,6 +72,7 @@ TAGS_NOSHOW = ( USERTAG_TAG, + ARKIME_AUTOARKIME_TAG, SURICATA_AUTOSURICATA_TAG, ZEEK_AUTOZEEK_TAG, ) @@ -112,7 +114,7 @@ def arkimeCaptureFileWorker(arkimeWorkerArgs): scanWorkerId = scanWorkersCount.increment() # unique ID for this thread - newFileQueue, pcapBaseDir, arkimeBin, nodeName, nodeHost, autoTag, notLocked, logger = ( + newFileQueue, pcapBaseDir, arkimeBin, nodeName, nodeHost, autoArkime, forceArkime, autoTag, notLocked, logger = ( arkimeWorkerArgs[0], arkimeWorkerArgs[1], arkimeWorkerArgs[2], @@ -121,6 +123,8 @@ def arkimeCaptureFileWorker(arkimeWorkerArgs): arkimeWorkerArgs[5], arkimeWorkerArgs[6], arkimeWorkerArgs[7], + arkimeWorkerArgs[8], + arkimeWorkerArgs[9], ) if not logger: @@ -141,49 +145,57 @@ def arkimeCaptureFileWorker(arkimeWorkerArgs): fileInfo[FILE_INFO_DICT_NAME] = os.path.join(pcapBaseDir, fileInfo[FILE_INFO_DICT_NAME]) if os.path.isfile(fileInfo[FILE_INFO_DICT_NAME]): - # finalize tags list - fileInfo[FILE_INFO_DICT_TAGS] = ( - [ - x - for x in fileInfo[FILE_INFO_DICT_TAGS] - if (x not in TAGS_NOSHOW) and (not x.startswith(ZEEK_AUTOCARVE_TAG_PREFIX)) - ] - if ((FILE_INFO_DICT_TAGS in fileInfo) and autoTag) - else list() - ) - logger.info(f"{scriptName}[{scanWorkerId}]:\t🔎\t{fileInfo}") - - # put together arkime execution command - cmd = [ - arkimeBin, - '--quiet', - '--insecure', - '--node', - fileInfo[FILE_INFO_DICT_NODE] if (FILE_INFO_DICT_NODE in fileInfo) else nodeName, - '-o', - f'ecsEventProvider={arkimeProvider}', - '-o', - f'ecsEventDataset={arkimeDataset}', - '-r', - fileInfo[FILE_INFO_DICT_NAME], - ] - if nodeHost: - cmd.append('--host') - cmd.append(nodeHost) - if notLocked: - cmd.append('--nolockpcap') - cmd.extend(list(chain.from_iterable(zip(repeat('-t'), fileInfo[FILE_INFO_DICT_TAGS])))) - - # execute capture for pcap file - retcode, output = run_process(cmd, logger=logger) - if retcode == 0: - logger.info( - f"{scriptName}[{scanWorkerId}]:\t✅\t{os.path.basename(fileInfo[FILE_INFO_DICT_NAME])}" + # Arkime this PCAP if it's tagged "AUTOARKIME" or if the global autoArkime flag is turned on. + if ( + forceArkime + or autoArkime + or ( + (FILE_INFO_DICT_TAGS in fileInfo) and ARKIME_AUTOARKIME_TAG in fileInfo[FILE_INFO_DICT_TAGS] ) - else: - logger.warning( - f"{scriptName}[{scanWorkerId}]:\t❗\t{arkimeBin} {os.path.basename(fileInfo[FILE_INFO_DICT_NAME])} returned {retcode} {output}" + ): + # finalize tags list + fileInfo[FILE_INFO_DICT_TAGS] = ( + [ + x + for x in fileInfo[FILE_INFO_DICT_TAGS] + if (x not in TAGS_NOSHOW) and (not x.startswith(ZEEK_AUTOCARVE_TAG_PREFIX)) + ] + if ((FILE_INFO_DICT_TAGS in fileInfo) and autoTag) + else list() ) + logger.info(f"{scriptName}[{scanWorkerId}]:\t🔎\t{fileInfo}") + + # put together arkime execution command + cmd = [ + arkimeBin, + '--quiet', + '--insecure', + '--node', + fileInfo[FILE_INFO_DICT_NODE] if (FILE_INFO_DICT_NODE in fileInfo) else nodeName, + '-o', + f'ecsEventProvider={arkimeProvider}', + '-o', + f'ecsEventDataset={arkimeDataset}', + '-r', + fileInfo[FILE_INFO_DICT_NAME], + ] + if nodeHost: + cmd.append('--host') + cmd.append(nodeHost) + if notLocked: + cmd.append('--nolockpcap') + cmd.extend(list(chain.from_iterable(zip(repeat('-t'), fileInfo[FILE_INFO_DICT_TAGS])))) + + # execute capture for pcap file + retcode, output = run_process(cmd, logger=logger) + if retcode == 0: + logger.info( + f"{scriptName}[{scanWorkerId}]:\t✅\t{os.path.basename(fileInfo[FILE_INFO_DICT_NAME])}" + ) + else: + logger.warning( + f"{scriptName}[{scanWorkerId}]:\t❗\t{arkimeBin} {os.path.basename(fileInfo[FILE_INFO_DICT_NAME])} returned {retcode} {output}" + ) logger.info(f"{scriptName}[{scanWorkerId}]:\tfinished") @@ -531,7 +543,7 @@ def main(): help="PCAP source node host (for Arkime viewer reachback)", metavar='', type=str, - default=os.getenv('PCAP_NODE_HOST', ''), + default='', ) requiredNamed = parser.add_argument_group('required arguments') requiredNamed.add_argument( @@ -543,6 +555,28 @@ def main(): required=True, ) if processingMode == PCAP_PROCESSING_MODE_ARKIME: + parser.add_argument( + '--autoarkime', + dest='autoArkime', + help="Autoanalyze all PCAP file with Arkime", + metavar='true|false', + type=str2bool, + nargs='?', + const=True, + default=False, + required=False, + ) + parser.add_argument( + '--forcearkime', + dest='forceArkime', + help="Force Arkime analysis even on rotated PCAPs", + metavar='true|false', + type=str2bool, + nargs='?', + const=True, + default=False, + required=False, + ) parser.add_argument( '--arkime', required=False, @@ -713,6 +747,8 @@ def main(): args.executable, args.nodeName, args.nodeHost, + args.autoArkime, + args.forceArkime, args.autoTag, args.notLocked, logging, diff --git a/shared/bin/pcap_utils.py b/shared/bin/pcap_utils.py index 2c9bf4f57..39fd5d313 100644 --- a/shared/bin/pcap_utils.py +++ b/shared/bin/pcap_utils.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. import os import re diff --git a/shared/bin/pcap_watcher.py b/shared/bin/pcap_watcher.py index 821986ffd..8d70f322e 100755 --- a/shared/bin/pcap_watcher.py +++ b/shared/bin/pcap_watcher.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ################################################################################################### # Monitor a directory for PCAP files for processing (by publishing their filenames to a ZMQ socket) diff --git a/shared/bin/preseed_late_user_config.sh b/shared/bin/preseed_late_user_config.sh index f5036608a..439c01a72 100755 --- a/shared/bin/preseed_late_user_config.sh +++ b/shared/bin/preseed_late_user_config.sh @@ -1,6 +1,6 @@ #!/bin/sh -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ################################################################################## # prompt whether to autologin or not diff --git a/shared/bin/prune_files.sh b/shared/bin/prune_files.sh index 56fe3a9dd..9d4c63f99 100755 --- a/shared/bin/prune_files.sh +++ b/shared/bin/prune_files.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # recursion depth (1 = not recursive) DEPTH=1 diff --git a/shared/bin/sensor-capture-disk-config.py b/shared/bin/sensor-capture-disk-config.py index b6ac3dd52..40c05b323 100755 --- a/shared/bin/sensor-capture-disk-config.py +++ b/shared/bin/sensor-capture-disk-config.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ################################################################################################### # Detect, partition, and format devices to be used for sensor packet/log captures. diff --git a/shared/bin/sensor-init.sh b/shared/bin/sensor-init.sh index 7a081f9ef..0f0ddaaa0 100755 --- a/shared/bin/sensor-init.sh +++ b/shared/bin/sensor-init.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. SCRIPT_PATH="$(dirname $(realpath -e "${BASH_SOURCE[0]}"))" @@ -66,7 +66,10 @@ if [[ -r "$SCRIPT_PATH"/common-init.sh ]]; then if dpkg -s suricata >/dev/null 2>&1 ; then mkdir -p /etc/suricata/rules /var/log/suricata /var/lib/suricata/rules if [[ -d /opt/sensor/sensor_ctl ]]; then - mkdir -p /opt/sensor/sensor_ctl/suricata/rules /opt/sensor/sensor_ctl/suricata/include-configs + mkdir -p /opt/sensor/sensor_ctl/suricata/rules \ + /opt/sensor/sensor_ctl/suricata/rules-default/OT \ + /opt/sensor/sensor_ctl/suricata/rules-default/IT \ + /opt/sensor/sensor_ctl/suricata/include-configs [[ ! -f /opt/sensor/sensor_ctl/suricata/suricata.yaml ]] && cp /etc/suricata/suricata.yaml /opt/sensor/sensor_ctl/suricata/suricata.yaml [[ ! -f /opt/sensor/sensor_ctl/suricata/update.yaml ]] && cp "$(dpkg -L suricata-update | grep 'update\.yaml' | head -n 1)" /opt/sensor/sensor_ctl/suricata/update.yaml fi diff --git a/shared/bin/sensorcommon.py b/shared/bin/sensorcommon.py index 19d97f3ff..8d5b3ebb3 100644 --- a/shared/bin/sensorcommon.py +++ b/shared/bin/sensorcommon.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. import argparse import ipaddress diff --git a/shared/bin/service_check_passthrough.sh b/shared/bin/service_check_passthrough.sh index 6b98aed5c..3e63a57a4 100755 --- a/shared/bin/service_check_passthrough.sh +++ b/shared/bin/service_check_passthrough.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # This script will check command-line parameters and environment variables to see # if the service (determined by the hostname, unless otherwise specified) is @@ -159,7 +159,7 @@ EOF fi # json vs http if command -v goStatic >/dev/null 2>&1; then - goStatic -path "$(pwd)" -fallback "index.html" -port $PORT + goStatic -vhost "" -path "$(pwd)" -fallback "index.html" -port $PORT elif command -v python3 >/dev/null 2>&1; then python3 -m http.server --bind 0.0.0.0 $PORT elif command -v python >/dev/null 2>&1; then diff --git a/shared/bin/set-dconf-screen-lock-defaults.sh b/shared/bin/set-dconf-screen-lock-defaults.sh index c5864c1fd..9ccb4c0c1 100755 --- a/shared/bin/set-dconf-screen-lock-defaults.sh +++ b/shared/bin/set-dconf-screen-lock-defaults.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. grep -q boot=live /proc/cmdline && exit 0 diff --git a/shared/bin/set-malcolm-gtk-bookmark.sh b/shared/bin/set-malcolm-gtk-bookmark.sh index f324b94dc..5b16afc88 100755 --- a/shared/bin/set-malcolm-gtk-bookmark.sh +++ b/shared/bin/set-malcolm-gtk-bookmark.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. if ! grep -q Malcolm$ "$HOME"/.config/gtk-3.0/bookmarks && [[ -d "$HOME"/Malcolm ]]; then mkdir -p "$HOME"/.config/gtk-3.0/ diff --git a/shared/bin/suricata_config_populate.py b/shared/bin/suricata_config_populate.py index 612e30a15..50c365304 100755 --- a/shared/bin/suricata_config_populate.py +++ b/shared/bin/suricata_config_populate.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # modify suricata.yaml according to many environment variables @@ -13,7 +13,7 @@ import argparse import contextlib -import fnmatch +import glob import logging import os import sys @@ -271,8 +271,8 @@ def __call__(self, repr, data): ), 'dnp3': ProtocolConfig( [], - val2bool(DEFAULT_VARS['DNP3_ENABLED']), - val2bool(DEFAULT_VARS['DNP3_EVE_ENABLED']), + (not val2bool(DEFAULT_VARS['DISABLE_ICS_ALL'])) and val2bool(DEFAULT_VARS['DNP3_ENABLED']), + (not val2bool(DEFAULT_VARS['DISABLE_ICS_ALL'])) and val2bool(DEFAULT_VARS['DNP3_EVE_ENABLED']), True, DEFAULT_VARS['DNP3_PORTS'], None, @@ -287,8 +287,8 @@ def __call__(self, repr, data): ), 'enip': ProtocolConfig( [], - val2bool(DEFAULT_VARS['ENIP_ENABLED']), - val2bool(DEFAULT_VARS['ENIP_EVE_ENABLED']), + (not val2bool(DEFAULT_VARS['DISABLE_ICS_ALL'])) and val2bool(DEFAULT_VARS['ENIP_ENABLED']), + (not val2bool(DEFAULT_VARS['DISABLE_ICS_ALL'])) and val2bool(DEFAULT_VARS['ENIP_EVE_ENABLED']), False, DEFAULT_VARS['ENIP_PORTS'], DEFAULT_VARS['ENIP_PORTS'], @@ -359,8 +359,8 @@ def __call__(self, repr, data): ), 'modbus': ProtocolConfig( [], - val2bool(DEFAULT_VARS['MODBUS_ENABLED']), - val2bool(DEFAULT_VARS['MODBUS_EVE_ENABLED']), + (not val2bool(DEFAULT_VARS['DISABLE_ICS_ALL'])) and val2bool(DEFAULT_VARS['MODBUS_ENABLED']), + (not val2bool(DEFAULT_VARS['DISABLE_ICS_ALL'])) and val2bool(DEFAULT_VARS['MODBUS_EVE_ENABLED']), False, DEFAULT_VARS['MODBUS_PORTS'], None, @@ -503,39 +503,83 @@ def __call__(self, repr, data): ################################################################################################### -def GetRuleSources(requireRulesExist=False): +def GetRuleFiles(): global DEFAULT_VARS - ruleSources = [] + ruleFiles = [] if not val2bool(DEFAULT_VARS['CUSTOM_RULES_ONLY']): - ruleSources.append('suricata.rules') + # built-in suricata rules + ruleFiles.append('suricata.rules') + + # Malcolm's default IT rules + ruleFiles.extend( + sorted( + list( + glob.iglob( + os.path.join( + os.path.join(os.path.join(DEFAULT_VARS['DEFAULT_RULES_DIR'], 'IT'), '**'), '*.rules' + ), + recursive=True, + ) + ) + ) + if os.path.isdir(str(DEFAULT_VARS['DEFAULT_RULES_DIR'])) + else [] + ) + + # Malcolm's default OT rules + ruleFiles.extend( + sorted( + list( + glob.iglob( + os.path.join( + os.path.join(os.path.join(DEFAULT_VARS['DEFAULT_RULES_DIR'], 'OT'), '**'), '*.rules' + ), + recursive=True, + ) + ) + ) + if ( + os.path.isdir(str(DEFAULT_VARS['DEFAULT_RULES_DIR'])) + and (not val2bool(DEFAULT_VARS['DISABLE_ICS_ALL'])) + ) + else [] + ) - customRuleFiles = ( - fnmatch.filter(os.listdir(DEFAULT_VARS['CUSTOM_RULES_DIR']), '*.rules') - if DEFAULT_VARS['CUSTOM_RULES_DIR'] is not None + # User's custom rules + ruleFiles.extend( + sorted( + list( + glob.iglob( + os.path.join(os.path.join(DEFAULT_VARS['CUSTOM_RULES_DIR'], '**'), '*.rules'), + recursive=True, + ) + ) + ) + if os.path.isdir(str(DEFAULT_VARS['CUSTOM_RULES_DIR'])) else [] ) - if (DEFAULT_VARS['CUSTOM_RULES_DIR'] is not None) and ((not requireRulesExist) or (len(customRuleFiles) > 0)): - ruleSources.append(os.path.join(DEFAULT_VARS['CUSTOM_RULES_DIR'], '*.rules')) - - return ruleSources + return ruleFiles ################################################################################################### def GetIncludeConfigSources(): global DEFAULT_VARS - configSources = list( - [ - os.path.join(DEFAULT_VARS['CUSTOM_CONFIG_DIR'], x) - for x in fnmatch.filter(os.listdir(DEFAULT_VARS['CUSTOM_CONFIG_DIR']), '*.yaml') - ] - if DEFAULT_VARS['CUSTOM_CONFIG_DIR'] is not None + configSources = ( + sorted( + list( + glob.iglob( + os.path.join(os.path.join(DEFAULT_VARS['CUSTOM_CONFIG_DIR'], '**'), '*.yaml'), + recursive=True, + ) + ) + ) + if os.path.isdir(str(DEFAULT_VARS['CUSTOM_CONFIG_DIR'])) else [] ) - return configSources @@ -627,7 +671,8 @@ def main(): if os.path.isfile(args.output) and os.path.samefile(args.input, args.output): backupFile = inFileParts[0] + "_bak_" + str(int(round(time.time()))) + inFileParts[1] CopyFile(args.input, backupFile) - backupFiles = sorted(fnmatch.filter(os.listdir(os.path.dirname(backupFile)), '*_bak_*')) + backupFiles = sorted(list(glob.glob(os.path.join(os.path.dirname(backupFile), '*_bak_*')))) + while len(backupFiles) > BACKUP_FILES_MAX: toDeleteFileName = os.path.join(os.path.dirname(backupFile), backupFiles.pop(0)) logging.debug(f'Removing old backup file "{toDeleteFileName}"') @@ -1080,7 +1125,7 @@ def main(): deep_set(cfg, ['stats', 'enabled'], True) cfg.pop('rule-files', None) - deep_set(cfg, ['rule-files'], GetRuleSources(requireRulesExist=True)) + deep_set(cfg, ['rule-files'], GetRuleFiles()) # Hackety-hack, don't talk back! Despite the "Including multiple files" section of # https://docs.suricata.io/en/latest/configuration/includes.html#including-multiple-files @@ -1127,7 +1172,7 @@ def main(): # final tweaks deep_set(cfg, ['stats', 'enabled'], False) cfg.pop('rule-files', None) - deep_set(cfg, ['rule-files'], GetRuleSources(requireRulesExist=False)) + deep_set(cfg, ['rule-files'], GetRuleFiles()) # see note on 'include' above cfg.pop('include', None) diff --git a/shared/bin/suricata_update_config_populate.py b/shared/bin/suricata_update_config_populate.py index edcad9155..52cf643f8 100755 --- a/shared/bin/suricata_update_config_populate.py +++ b/shared/bin/suricata_update_config_populate.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # modify suricata's update.yaml according to many environment variables @@ -13,7 +13,7 @@ import argparse import contextlib -import fnmatch +import glob import logging import os import sys @@ -157,7 +157,7 @@ def main(): if os.path.isfile(args.output) and os.path.samefile(args.input, args.output): backupFile = inFileParts[0] + "_bak_" + str(int(round(time.time()))) + inFileParts[1] CopyFile(args.input, backupFile) - backupFiles = sorted(fnmatch.filter(os.listdir(os.path.dirname(backupFile)), '*_bak_*')) + backupFiles = sorted(list(glob.glob(os.path.join(os.path.dirname(backupFile), '*_bak_*')))) while len(backupFiles) > BACKUP_FILES_MAX: toDeleteFileName = os.path.join(os.path.dirname(backupFile), backupFiles.pop(0)) logging.debug(f'Removing old backup file "{toDeleteFileName}"') diff --git a/shared/bin/therm-sensors-json.py b/shared/bin/therm-sensors-json.py index 48f7d6b2f..87ff591ea 100755 --- a/shared/bin/therm-sensors-json.py +++ b/shared/bin/therm-sensors-json.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. import argparse import json diff --git a/shared/bin/ufw_allow_viewer.sh b/shared/bin/ufw_allow_viewer.sh index ae581a314..90c06fbaf 100755 --- a/shared/bin/ufw_allow_viewer.sh +++ b/shared/bin/ufw_allow_viewer.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # manage a UFW rule for allowing a remote Arkime viewer instance (on the same host # to which arkime's capture is forwarding session logs) to connect to and diff --git a/shared/bin/zeek_carve_logger.py b/shared/bin/zeek_carve_logger.py index 0f2e0f884..2ab20dadd 100755 --- a/shared/bin/zeek_carve_logger.py +++ b/shared/bin/zeek_carve_logger.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ################################################################################################### # Monitor a directory for files extracted by zeek for processing diff --git a/shared/bin/zeek_carve_scanner.py b/shared/bin/zeek_carve_scanner.py index 35d56f7bd..c472b84b2 100755 --- a/shared/bin/zeek_carve_scanner.py +++ b/shared/bin/zeek_carve_scanner.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ################################################################################################### # Process queued files reported by zeek_carve_watcher.py, scanning them with the specified diff --git a/shared/bin/zeek_carve_utils.py b/shared/bin/zeek_carve_utils.py index 1ac1129ea..21550aeaa 100644 --- a/shared/bin/zeek_carve_utils.py +++ b/shared/bin/zeek_carve_utils.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. import clamd import logging diff --git a/shared/bin/zeek_carve_watcher.py b/shared/bin/zeek_carve_watcher.py index e2b93590a..960464905 100755 --- a/shared/bin/zeek_carve_watcher.py +++ b/shared/bin/zeek_carve_watcher.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. ################################################################################################### # Monitor a directory for files extracted by zeek for processing diff --git a/shared/bin/zeek_install_plugins.sh b/shared/bin/zeek_install_plugins.sh index d1588bf8b..70ce9b01f 100755 --- a/shared/bin/zeek_install_plugins.sh +++ b/shared/bin/zeek_install_plugins.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. if [ -z "$BASH_VERSION" ]; then echo "Wrong interpreter, please run \"$0\" with bash" diff --git a/shared/bin/zeek_intel_from_threat_feed.py b/shared/bin/zeek_intel_from_threat_feed.py index 45184e838..456387238 100755 --- a/shared/bin/zeek_intel_from_threat_feed.py +++ b/shared/bin/zeek_intel_from_threat_feed.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. from collections import deque from dateparser import parse as ParseDate diff --git a/shared/bin/zeek_intel_setup.sh b/shared/bin/zeek_intel_setup.sh index 077dade07..d4a8ad80b 100755 --- a/shared/bin/zeek_intel_setup.sh +++ b/shared/bin/zeek_intel_setup.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # set up intel files prior to running zeek # - https://idaholab.github.io/Malcolm/docs/zeek-intel.html#ZeekIntel diff --git a/shared/bin/zeek_threat_feed_utils.py b/shared/bin/zeek_threat_feed_utils.py index e5446a6aa..a7b0c21a7 100644 --- a/shared/bin/zeek_threat_feed_utils.py +++ b/shared/bin/zeek_threat_feed_utils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # adapted some code from tenzir/threatbus # - https://github.com/tenzir/threatbus diff --git a/shared/bin/zeekdeploy.sh b/shared/bin/zeekdeploy.sh index 0e2b2999e..f5a88d49f 100755 --- a/shared/bin/zeekdeploy.sh +++ b/shared/bin/zeekdeploy.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # get utilities for finding default zeek path and executable [[ "$(uname -s)" = 'Darwin' ]] && REALPATH=grealpath || REALPATH=realpath diff --git a/suricata/default-rules/IT/.gitignore b/suricata/rules-default/IT/.gitignore similarity index 100% rename from suricata/default-rules/IT/.gitignore rename to suricata/rules-default/IT/.gitignore diff --git a/suricata/default-rules/OT/.gitignore b/suricata/rules-default/OT/.gitignore similarity index 100% rename from suricata/default-rules/OT/.gitignore rename to suricata/rules-default/OT/.gitignore diff --git a/suricata/rules-default/OT/malcolm/CVE-2023-28771_Zyxel.rules b/suricata/rules-default/OT/malcolm/CVE-2023-28771_Zyxel.rules new file mode 100644 index 000000000..4d1d2db9a --- /dev/null +++ b/suricata/rules-default/OT/malcolm/CVE-2023-28771_Zyxel.rules @@ -0,0 +1,14 @@ +alert udp any any -> any 500 (msg:"Potential CVE-2023-28771 Exploit Detected"; content:"show users"; nocase; sid:1001001; rev:1;) +alert udp any any -> any 500 (msg:"Potential CVE-2023-28771 Exploit Detected"; content:"show sessions"; nocase; sid:1001002; rev:1;) +alert udp any any -> any 500 (msg:"Potential CVE-2023-28771 Exploit Detected"; content:"show configuration"; nocase; sid:1001003; rev:1;) +alert udp any any -> any 500 (msg:"Potential CVE-2023-28771 Exploit Detected"; content:"show running-config"; nocase; sid:1001004; rev:1;) +alert udp any any -> any 500 (msg:"Potential CVE-2023-28771 Exploit Detected"; content:"show firewall rule"; nocase; sid:1001005; rev:1;) +alert udp any any -> any 500 (msg:"Potential CVE-2023-28771 Exploit Detected"; content:"export config"; nocase; sid:1001006; rev:1;) +alert tcp any any -> any any (msg: "Potential Zyxel Payload connection"; content:"/mipskiller"; sid:1001007; rev:1;) +alert tcp any any -> any any (msg: "Potential Zyxel Payload connection"; content:"/proxy2"; sid:1001008; rev:1;) +alert tcp any any -> any 82 (msg: "Potential Zyxel Payload connection"; content:"/fuckjewishpeople.mips"; sid:1001009; rev:1;) +alert tcp any any -> any 8080 (msg: "Potential Zyxel Payload connection"; content:"/mips"; sid:1001010; rev:1;) +alert tcp any any -> any 8080 (msg: "Potential Zyxel Payload connection"; content:"/mpsl"; sid:1001011; rev:1;) +alert tcp any any -> any any (msg: "Potential Zyxel Payload connection"; content:"/bins/paraiso.mips"; sid:1001012; rev:1;) +alert tcp any any -> any any (msg: "Potential Zyxel Payload connection"; content:"/bins/libcurl1337.mips"; sid:1001013; rev:1;) +alert tcp any any -> any any (msg: "Potential Zyxel Payload connection"; content:"/proxy1"; sid:1001014; rev:1;) \ No newline at end of file diff --git a/suricata/default-rules/OT/nsacyber/ELITEWOLF/AllenBradley_RockwellAutomation.rules b/suricata/rules-default/OT/nsacyber/ELITEWOLF/AllenBradley_RockwellAutomation.rules similarity index 100% rename from suricata/default-rules/OT/nsacyber/ELITEWOLF/AllenBradley_RockwellAutomation.rules rename to suricata/rules-default/OT/nsacyber/ELITEWOLF/AllenBradley_RockwellAutomation.rules diff --git a/suricata/default-rules/OT/nsacyber/ELITEWOLF/SchweitzerEngineeringLaboratories.rules b/suricata/rules-default/OT/nsacyber/ELITEWOLF/SchweitzerEngineeringLaboratories.rules similarity index 100% rename from suricata/default-rules/OT/nsacyber/ELITEWOLF/SchweitzerEngineeringLaboratories.rules rename to suricata/rules-default/OT/nsacyber/ELITEWOLF/SchweitzerEngineeringLaboratories.rules diff --git a/suricata/default-rules/OT/nsacyber/ELITEWOLF/Siemens.rules b/suricata/rules-default/OT/nsacyber/ELITEWOLF/Siemens.rules similarity index 100% rename from suricata/default-rules/OT/nsacyber/ELITEWOLF/Siemens.rules rename to suricata/rules-default/OT/nsacyber/ELITEWOLF/Siemens.rules diff --git a/suricata/scripts/docker_entrypoint.sh b/suricata/scripts/docker_entrypoint.sh index a3d3e0200..b3044ed59 100755 --- a/suricata/scripts/docker_entrypoint.sh +++ b/suricata/scripts/docker_entrypoint.sh @@ -1,7 +1,6 @@ #!/bin/bash # ensure capabilities for capture -setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool || true setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' /usr/bin/suricata || true # modify suricata.yaml according to environment variables (as non-root) diff --git a/suricata/scripts/eve-clean-logs.sh b/suricata/scripts/eve-clean-logs.sh index e38a0e00b..33b0127e0 100755 --- a/suricata/scripts/eve-clean-logs.sh +++ b/suricata/scripts/eve-clean-logs.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. # Clean up suricata log files that have reached a certain age. If we can # verify they've been parsed and logged at least one event to the database, diff --git a/suricata/supervisord.conf b/suricata/supervisord.conf index 49e10393e..8376a91d9 100644 --- a/suricata/supervisord.conf +++ b/suricata/supervisord.conf @@ -1,4 +1,4 @@ -; Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +; Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. [unix_http_server] file=/tmp/supervisor.sock ; (the path to the socket file) @@ -24,7 +24,7 @@ command=python3 /usr/local/bin/pcap_suricata_processor.py %(ENV_PCAP_PIPELINE_VE --publisher "%(ENV_PCAP_MONITOR_HOST)s" --pcap-directory /data/pcap/processed --node "%(ENV_PCAP_NODE_NAME)s" - --suricata /usr/bin/suricata + --suricata /usr/bin/suricata-offline --autotag "%(ENV_AUTO_TAG)s" --autosuricata "%(ENV_SURICATA_AUTO_ANALYZE_PCAP_FILES)s" --forcesuricata "%(ENV_SURICATA_ROTATED_PCAP)s" diff --git a/zeek/config/extractor.zeek b/zeek/config/extractor.zeek index 0418f23e9..07e745368 100644 --- a/zeek/config/extractor.zeek +++ b/zeek/config/extractor.zeek @@ -1,6 +1,6 @@ #!/usr/bin/env zeek -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. @load ./extractor_params diff --git a/zeek/config/extractor_override.interesting.zeek b/zeek/config/extractor_override.interesting.zeek index eaf220289..290a8948c 100644 --- a/zeek/config/extractor_override.interesting.zeek +++ b/zeek/config/extractor_override.interesting.zeek @@ -1,6 +1,6 @@ #!/usr/bin/env zeek -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. export { redef extractor_always_extract_unknown = F; diff --git a/zeek/config/extractor_params.zeek b/zeek/config/extractor_params.zeek index 1d82162fe..653e91503 100644 --- a/zeek/config/extractor_params.zeek +++ b/zeek/config/extractor_params.zeek @@ -1,6 +1,6 @@ #!/usr/bin/env zeek -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +# Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. export { const extractor_extract_none = "none" &redef; diff --git a/zeek/scripts/docker_entrypoint.sh b/zeek/scripts/docker_entrypoint.sh index 2d9d4f972..0ed78c9b0 100755 --- a/zeek/scripts/docker_entrypoint.sh +++ b/zeek/scripts/docker_entrypoint.sh @@ -3,9 +3,8 @@ ZEEK_DIR=${ZEEK_DIR:-"/opt/zeek"} # ensure capabilities for capture -setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool || true -setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' "${ZEEK_DIR}"/bin/zeek || true -setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' "${ZEEK_DIR}"/bin/capstats || true +setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' "${ZEEK_DIR}"/bin/zeek || true +setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' "${ZEEK_DIR}"/bin/capstats || true if [[ "${ZEEK_LIVE_CAPTURE:-false}" != "true" ]] && [[ -x "${ZEEK_DIR}"/bin/zeek_intel_setup.sh ]]; then sleep 15 # give the "live" instance, if there is one, a chance to go first diff --git a/zeek/supervisord.conf b/zeek/supervisord.conf index 23db15fbe..5ad1d49b7 100644 --- a/zeek/supervisord.conf +++ b/zeek/supervisord.conf @@ -1,4 +1,4 @@ -; Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +; Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. [unix_http_server] file=/tmp/supervisor.sock ; (the path to the socket file) @@ -24,7 +24,7 @@ command=python3 /usr/local/bin/pcap_zeek_processor.py %(ENV_PCAP_PIPELINE_VERBOS --publisher "%(ENV_PCAP_MONITOR_HOST)s" --pcap-directory /pcap/processed --node "%(ENV_PCAP_NODE_NAME)s" - --zeek /opt/zeek/bin/zeek + --zeek /opt/zeek/bin/zeek-offline --autotag "%(ENV_AUTO_TAG)s" --autozeek "%(ENV_ZEEK_AUTO_ANALYZE_PCAP_FILES)s" --forcezeek "%(ENV_ZEEK_ROTATED_PCAP)s"